Merge branch 'archive-policy-fix' into 'master'

Archive policy overhaul

Closes #323, #292, and #166

See merge request !291

.gitlab-ci.yml

@@ -2,6 +2,7 @@ image: registry.gitlab.com/fdroid/ci-images:server-latest
 
 test:
   script:
+    - apt-get -qq update && apt-get -y dist-upgrade
     - mkdir -p /usr/lib/python3.4/site-packages/
     # workaround https://github.com/pypa/setuptools/issues/937
     - pip3 install setuptools==33.1.1

examples/config.py

@@ -71,6 +71,15 @@ archive_description = """
 The repository of older versions of applications from the main demo repository.
 """
 
+# This allows a specific kind of insecure APK to be included in the
+# 'repo' section.  Since April 2017, APK signatures that use MD5 are
+# no longer considered valid, jarsigner and apksigner will return an
+# error when verifying.  `fdroid update` will move APKs with these
+# disabled signatures to the archive.  This option stops that
+# behavior, and lets those APKs stay part of 'repo'.
+#
+# allow_disabled_algorithms = True
+
 # Normally, all apps are collected into a single app repository, like on
 # https://f-droid.org. For certain situations, it is better to make a repo
 # that is made up of APKs only from a single app. For example, an automated

fdroidserver/common.py

@@ -85,6 +85,7 @@ default_config = {
     'gradle': 'gradle',
     'accepted_formats': ['txt', 'yml'],
     'sync_from_local_copy_dir': False,
+    'allow_disabled_algorithms': False,
     'per_app_repos': False,
     'make_current_version_link': True,
     'current_version_name_source': 'Name',
@@ -2041,6 +2042,26 @@ def verify_apk_signature(apk, jar=False):
         return subprocess.call([config['jarsigner'], '-strict', '-verify', apk]) == 4
 
 
+def verify_old_apk_signature(apk):
+    """verify the signature on an archived APK, supporting deprecated algorithms
+
+    F-Droid aims to keep every single binary that it ever published.  Therefore,
+    it needs to be able to verify APK signatures that include deprecated/removed
+    algorithms.  For example, jarsigner treats an MD5 signature as unsigned.
+
+    jarsigner passes unsigned APKs as "verified"! So this has to turn
+    on -strict then check for result 4.
+
+    """
+
+    _java_security = os.path.join(os.getcwd(), '.java.security')
+    with open(_java_security, 'w') as fp:
+        fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
+
+    return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
+                            '-strict', '-verify', apk]) == 4
+
+
 apk_badchars = re.compile('''[/ :;'"]''')
 
 

fdroidserver/update.py

@@ -423,20 +423,35 @@ def get_cache_file():
 
 
 def get_cache():
-    """
+    """Get the cached dict of the APK index
+
     Gather information about all the apk files in the repo directory,
-    using cached data if possible.
+    using cached data if possible. Some of the index operations take a
+    long time, like calculating the SHA-256 and verifying the APK
+    signature.
+
+    The cache is invalidated if the metadata version is different, or
+    the 'allow_disabled_algorithms' config/option is different.  In
+    those cases, there is no easy way to know what has changed from
+    the cache, so just rerun the whole thing.
+
     :return: apkcache
+
     """
     apkcachefile = get_cache_file()
+    ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
     if not options.clean and os.path.exists(apkcachefile):
         with open(apkcachefile, 'rb') as cf:
             apkcache = pickle.load(cf, encoding='utf-8')
-        if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
+        if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
+           or apkcache.get('allow_disabled_algorithms') != ada:
             apkcache = {}
     else:
         apkcache = {}
 
+    apkcache["METADATA_VERSION"] = METADATA_VERSION
+    apkcache['allow_disabled_algorithms'] = ada
+
     return apkcache
 
 
@@ -445,7 +460,6 @@ def write_cache(apkcache):
     cache_path = os.path.dirname(apkcachefile)
     if not os.path.exists(cache_path):
         os.makedirs(cache_path)
-    apkcache["METADATA_VERSION"] = METADATA_VERSION
     with open(apkcachefile, 'wb') as cf:
         pickle.dump(apkcache, cf)
 
@@ -1082,7 +1096,8 @@ def scan_apk_androguard(apk, apkfile):
         apk['features'].append(feature)
 
 
-def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
+def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
+             allow_disabled_algorithms=False, archive_bad_sig=False):
     """Scan the apk with the given filename in the given repo directory.
 
     This also extracts the icons.
@@ -1093,6 +1108,9 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
     :param knownapks: known apks info
     :param use_date_from_apk: use date from APK (instead of current date)
                               for newly added APKs
+    :param allow_disabled_algorithms: allow APKs with valid signatures that include
+                                      disabled algorithms in the signature (e.g. MD5)
+    :param archive_bad_sig: move APKs with a bad signature to the archive
     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
     """
@@ -1184,12 +1202,29 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
             apk['srcname'] = srcfilename
         apk['size'] = os.path.getsize(apkfile)
 
-        # verify the jar signature is correct
+        # verify the jar signature is correct, allow deprecated
+        # algorithms only if the APK is in the archive.
+        skipapk = False
         if not common.verify_apk_signature(apkfile):
+            if repodir == 'archive' or allow_disabled_algorithms:
+                if common.verify_old_apk_signature(apkfile):
+                    apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
+                else:
+                    skipapk = True
+            else:
+                skipapk = True
+
+        if skipapk:
+            if archive_bad_sig:
+                logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
+                move_apk_between_sections(repodir, 'archive', apk)
+            else:
+                logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
             return True, None, False
 
-        if has_known_vulnerability(apkfile):
+        if 'KnownVuln' not in apk['antiFeatures']:
-            apk['antiFeatures'].add('KnownVuln')
+            if has_known_vulnerability(apkfile):
+                apk['antiFeatures'].add('KnownVuln')
 
         apkzip = zipfile.ZipFile(apkfile, 'r')
 
@@ -1363,10 +1398,13 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
     apks = []
     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
         apkfilename = apkfile[len(repodir) + 1:]
-        (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk)
+        ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
+        (skip, apk, cachethis) = scan_apk(apkcache, apkfilename, repodir, knownapks,
+                                          use_date_from_apk, ada, True)
         if skip:
             continue
         apks.append(apk)
+        cachechanged = cachechanged or cachethis
 
     return apks, cachechanged
 
@@ -1421,6 +1459,15 @@ def make_categories_txt(repodir, categories):
 
 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 
+    def filter_apk_list_sorted(apk_list):
+        res = []
+        for apk in apk_list:
+            if apk['packageName'] == appid:
+                res.append(apk)
+
+        # Sort the apk list by version code. First is highest/newest.
+        return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
+
     for appid, app in apps.items():
 
         if app.ArchivePolicy:
@@ -1428,60 +1475,57 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi
         else:
             keepversions = defaultkeepversions
 
-        def filter_apk_list_sorted(apk_list):
-            res = []
-            for apk in apk_list:
-                if apk['packageName'] == appid:
-                    res.append(apk)
-
-            # Sort the apk list by version code. First is highest/newest.
-            return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
-
-        def move_file(from_dir, to_dir, filename, ignore_missing):
-            from_path = os.path.join(from_dir, filename)
-            if ignore_missing and not os.path.exists(from_path):
-                return
-            to_path = os.path.join(to_dir, filename)
-            shutil.move(from_path, to_path)
-
         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
                       .format(appid, len(apks), keepversions, len(archapks)))
 
-        if len(apks) > keepversions:
+        current_app_apks = filter_apk_list_sorted(apks)
-            apklist = filter_apk_list_sorted(apks)
+        if len(current_app_apks) > keepversions:
             # Move back the ones we don't want.
-            for apk in apklist[keepversions:]:
+            for apk in current_app_apks[keepversions:]:
-                logging.info("Moving " + apk['apkName'] + " to archive")
+                move_apk_between_sections(repodir, archivedir, apk)
-                move_file(repodir, archivedir, apk['apkName'], False)
-                move_file(repodir, archivedir, apk['apkName'] + '.asc', True)
-                for density in all_screen_densities:
-                    repo_icon_dir = get_icon_dir(repodir, density)
-                    archive_icon_dir = get_icon_dir(archivedir, density)
-                    if density not in apk['icons']:
-                        continue
-                    move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
-                if 'srcname' in apk:
-                    move_file(repodir, archivedir, apk['srcname'], False)
                 archapks.append(apk)
                 apks.remove(apk)
-        elif len(apks) < keepversions and len(archapks) > 0:
+
-            required = keepversions - len(apks)
+        current_app_archapks = filter_apk_list_sorted(archapks)
-            archapklist = filter_apk_list_sorted(archapks)
+        if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
-            # Move forward the ones we want again.
+            kept = 0
-            for apk in archapklist[:required]:
+            # Move forward the ones we want again, except DisableAlgorithm
-                logging.info("Moving " + apk['apkName'] + " from archive")
+            for apk in current_app_archapks:
-                move_file(archivedir, repodir, apk['apkName'], False)
+                if 'DisabledAlgorithm' not in apk['antiFeatures']:
-                move_file(archivedir, repodir, apk['apkName'] + '.asc', True)
+                    move_apk_between_sections(archivedir, repodir, apk)
-                for density in all_screen_densities:
+                    archapks.remove(apk)
-                    repo_icon_dir = get_icon_dir(repodir, density)
+                    apks.append(apk)
-                    archive_icon_dir = get_icon_dir(archivedir, density)
+                    kept += 1
-                    if density not in apk['icons']:
+                if kept == keepversions:
-                        continue
+                    break
-                    move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
+
-                if 'srcname' in apk:
+
-                    move_file(archivedir, repodir, apk['srcname'], False)
+def move_apk_between_sections(from_dir, to_dir, apk):
-                archapks.remove(apk)
+    """move an APK from repo to archive or vice versa"""
-                apks.append(apk)
+
+    def _move_file(from_dir, to_dir, filename, ignore_missing):
+        from_path = os.path.join(from_dir, filename)
+        if ignore_missing and not os.path.exists(from_path):
+            return
+        to_path = os.path.join(to_dir, filename)
+        if not os.path.exists(to_dir):
+            os.mkdir(to_dir)
+        shutil.move(from_path, to_path)
+
+    if from_dir == to_dir:
+        return
+
+    logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
+    _move_file(from_dir, to_dir, apk['apkName'], False)
+    _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
+    for density in all_screen_densities:
+        from_icon_dir = get_icon_dir(from_dir, density)
+        to_icon_dir = get_icon_dir(to_dir, density)
+        if density not in apk['icons']:
+            continue
+        _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
+    if 'srcname' in apk:
+        _move_file(from_dir, to_dir, apk['srcname'], False)
 
 
 def add_apks_to_per_app_repos(repodir, apks):
@@ -1544,6 +1588,8 @@ def main():
                         help="Use date from apk instead of current time for newly added apks")
     parser.add_argument("--rename-apks", action="store_true", default=False,
                         help="Rename APK files that do not match package.name_123.apk")
+    parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
+                        help="Include APKs that are signed with disabled algorithms like MD5")
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
     metadata.warnings_action = options.W

tests/metadata/com.politedroid.txt

@@ -0,0 +1,36 @@
+Categories:Time
+License:GPL-3.0
+Web Site:
+Source Code:https://github.com/miguelvps/PoliteDroid
+Issue Tracker:https://github.com/miguelvps/PoliteDroid/issues
+
+Auto Name:Polite Droid
+Summary:Calendar tool
+Description:
+Activates silent mode during calendar events.
+.
+
+Repo Type:git
+Repo:https://github.com/miguelvps/PoliteDroid.git
+
+Build:1.2,3
+    commit=6a548e4b19
+    target=android-10
+
+Build:1.3,4
+    commit=ad865b57bf3ac59580f38485608a9b1dda4fa7dc
+    target=android-15
+
+Build:1.4,5
+    commit=456bd615f3fbe6dff06433928cf7ea20073601fb
+    target=android-10
+
+Build:1.5,6
+    commit=v1.5
+    gradle=yes
+
+Archive Policy:4 versions
+Auto Update Mode:Version v%v
+Update Check Mode:Tags
+Current Version:1.5
+Current Version Code:6

tests/repo/categories.txt

@@ -7,3 +7,4 @@ None
 Phone & SMS
 Security
 System
+Time

tests/repo/index.xml

@@ -159,6 +159,71 @@
 			<sig>b4964fd759edaa54e65bb476d0276880</sig>
 		</package>
 	</application>
+	<application id="com.politedroid">
+		<id>com.politedroid</id>
+		<added>2017-06-23</added>
+		<lastupdated>2017-06-23</lastupdated>
+		<name>Polite Droid</name>
+		<summary>Calendar tool</summary>
+		<icon>com.politedroid.6.png</icon>
+		<desc>&lt;p&gt;Activates silent mode during calendar events.&lt;/p&gt;</desc>
+		<license>GPL-3.0</license>
+		<categories>Time</categories>
+		<category>Time</category>
+		<web></web>
+		<source>https://github.com/miguelvps/PoliteDroid</source>
+		<tracker>https://github.com/miguelvps/PoliteDroid/issues</tracker>
+		<marketversion>1.5</marketversion>
+		<marketvercode>6</marketvercode>
+		<package>
+			<version>1.5</version>
+			<versioncode>6</versioncode>
+			<apkname>com.politedroid_6.apk</apkname>
+			<hash type="sha256">70c2f776a2bac38a58a7d521f96ee0414c6f0fb1de973c3ca8b10862a009247d</hash>
+			<size>16578</size>
+			<sdkver>14</sdkver>
+			<targetSdkVersion>21</targetSdkVersion>
+			<added>2017-06-23</added>
+			<sig>b4964fd759edaa54e65bb476d0276880</sig>
+			<permissions>READ_CALENDAR,RECEIVE_BOOT_COMPLETED</permissions>
+		</package>
+		<package>
+			<version>1.4</version>
+			<versioncode>5</versioncode>
+			<apkname>com.politedroid_5.apk</apkname>
+			<hash type="sha256">5bdbfa071cca4b8d05ced41d6b28763595d6e8096cca5bbf0f9253c9a2622e5d</hash>
+			<size>18817</size>
+			<sdkver>3</sdkver>
+			<targetSdkVersion>10</targetSdkVersion>
+			<added>2017-06-23</added>
+			<sig>b4964fd759edaa54e65bb476d0276880</sig>
+			<permissions>READ_CALENDAR,RECEIVE_BOOT_COMPLETED</permissions>
+		</package>
+		<package>
+			<version>1.3</version>
+			<versioncode>4</versioncode>
+			<apkname>com.politedroid_4.apk</apkname>
+			<hash type="sha256">c809bdff83715fbf919f3840ee09869b038e209378b906e135ee40d3f0e1f075</hash>
+			<size>18489</size>
+			<sdkver>3</sdkver>
+			<targetSdkVersion>3</targetSdkVersion>
+			<added>2017-06-23</added>
+			<sig>b4964fd759edaa54e65bb476d0276880</sig>
+			<permissions>READ_CALENDAR,READ_EXTERNAL_STORAGE,READ_PHONE_STATE,RECEIVE_BOOT_COMPLETED,WRITE_EXTERNAL_STORAGE</permissions>
+		</package>
+		<package>
+			<version>1.2</version>
+			<versioncode>3</versioncode>
+			<apkname>com.politedroid_3.apk</apkname>
+			<hash type="sha256">665d03d61ebc642289fda697f71a59305b0202b16cafc5ffdae91cbe91f0b25d</hash>
+			<size>17552</size>
+			<sdkver>3</sdkver>
+			<targetSdkVersion>3</targetSdkVersion>
+			<added>2017-06-23</added>
+			<sig>b4964fd759edaa54e65bb476d0276880</sig>
+			<permissions>READ_CALENDAR,READ_EXTERNAL_STORAGE,READ_PHONE_STATE,RECEIVE_BOOT_COMPLETED,WRITE_EXTERNAL_STORAGE</permissions>
+		</package>
+	</application>
 	<application id="info.guardianproject.urzip">
 		<id>info.guardianproject.urzip</id>
 		<added>2016-06-23</added>

tests/run-tests

@@ -141,6 +141,8 @@ $fdroid signindex --verbose
 test -e repo/index.xml
 test -e repo/index.jar
 test -e repo/index-v1.jar
+test -e tmp/apkcache
+! test -z tmp/apkcache
 test -L urzip.apk
 grep -F '<application id=' repo/index.xml > /dev/null
 
@@ -237,7 +239,250 @@ test -e repo/obb.main.twoversions_1101617_src.tar.gz.asc
 
 # we can't easily reproduce the timestamps for things, so just hardcode them
 sed -i --expression='s,timestamp="[0-9]*",timestamp="1480431575",' repo/index.xml
-diff $WORKSPACE/tests/repo/index.xml repo/index.xml
+diff -uw $WORKSPACE/tests/repo/index.xml repo/index.xml
+
+
+#------------------------------------------------------------------------------#
+echo_header 'test moving lots of APKs to the archive'
+
+REPOROOT=`create_test_dir`
+cd $REPOROOT
+cp $WORKSPACE/tests/keystore.jks $REPOROOT/
+$fdroid init --keystore keystore.jks --repo-keyalias=sova
+echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo "accepted_formats = ['txt']" >> config.py
+sed -i '/allow_disabled_algorithms/d' config.py
+test -d metadata || mkdir metadata
+cp $WORKSPACE/tests/metadata/*.txt metadata/
+echo 'Summary:good test version of urzip' > metadata/info.guardianproject.urzip.txt
+echo 'Summary:good MD5 sig, which is disabled algorithm' > metadata/org.bitbucket.tickytacky.mirrormirror.txt
+sed -i '/Archive Policy:/d' metadata/*.txt
+test -d repo || mkdir repo
+cp $WORKSPACE/tests/urzip.apk \
+   $WORKSPACE/tests/org.bitbucket.tickytacky.mirrormirror_[0-9].apk \
+   $WORKSPACE/tests/repo/com.politedroid_[0-9].apk \
+   $WORKSPACE/tests/repo/obb.main.twoversions_110161[357].apk \
+   repo/
+sed -i 's,archive_older = [0-9],archive_older = 3,' config.py
+
+$fdroid update --pretty --nosign
+test `grep '<package>' archive/index.xml | wc -l` -eq 5
+test `grep '<package>' repo/index.xml | wc -l` -eq 7
+
+
+#------------------------------------------------------------------------------#
+echo_header 'test per-app "Archive Policy"'
+
+REPOROOT=`create_test_dir`
+cd $REPOROOT
+cp $WORKSPACE/tests/keystore.jks $REPOROOT/
+$fdroid init --keystore keystore.jks --repo-keyalias=sova
+echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo "accepted_formats = ['txt']" >> config.py
+test -d metadata || mkdir metadata
+cp $WORKSPACE/tests/metadata/com.politedroid.txt metadata/
+test -d repo || mkdir repo
+cp $WORKSPACE/tests/repo/com.politedroid_[0-9].apk repo/
+sed -i 's,archive_older = [0-9],archive_older = 3,' config.py
+
+$fdroid update --pretty --nosign
+test `grep '<package>' archive/index.xml | wc -l` -eq 0
+test `grep '<package>' repo/index.xml | wc -l` -eq 4
+grep -F com.politedroid_3.apk repo/index.xml
+grep -F com.politedroid_4.apk repo/index.xml
+grep -F com.politedroid_5.apk repo/index.xml
+grep -F com.politedroid_6.apk repo/index.xml
+test -e repo/com.politedroid_3.apk
+test -e repo/com.politedroid_4.apk
+test -e repo/com.politedroid_5.apk
+test -e repo/com.politedroid_6.apk
+
+echo "enable one app in the repo"
+sed -i 's,^Archive Policy:4,Archive Policy:1,' metadata/com.politedroid.txt
+$fdroid update --pretty --nosign
+test `grep '<package>' archive/index.xml | wc -l` -eq 3
+test `grep '<package>' repo/index.xml | wc -l` -eq 1
+grep -F com.politedroid_3.apk archive/index.xml
+grep -F com.politedroid_4.apk archive/index.xml
+grep -F com.politedroid_5.apk archive/index.xml
+grep -F com.politedroid_6.apk repo/index.xml
+test -e archive/com.politedroid_3.apk
+test -e archive/com.politedroid_4.apk
+test -e archive/com.politedroid_5.apk
+test -e repo/com.politedroid_6.apk
+
+echo "remove all apps from the repo"
+sed -i 's,^Archive Policy:1,Archive Policy:0,' metadata/com.politedroid.txt
+$fdroid update --pretty --nosign
+test `grep '<package>' archive/index.xml | wc -l` -eq 4
+test `grep '<package>' repo/index.xml | wc -l` -eq 0
+grep -F com.politedroid_3.apk archive/index.xml
+grep -F com.politedroid_4.apk archive/index.xml
+grep -F com.politedroid_5.apk archive/index.xml
+grep -F com.politedroid_6.apk archive/index.xml
+test -e archive/com.politedroid_3.apk
+test -e archive/com.politedroid_4.apk
+test -e archive/com.politedroid_5.apk
+test -e archive/com.politedroid_6.apk
+! test -e repo/com.politedroid_6.apk
+
+echo "move back one from archive to the repo"
+sed -i 's,^Archive Policy:0,Archive Policy:1,' metadata/com.politedroid.txt
+$fdroid update --pretty --nosign
+test `grep '<package>' archive/index.xml | wc -l` -eq 3
+test `grep '<package>' repo/index.xml | wc -l` -eq 1
+grep -F com.politedroid_3.apk archive/index.xml
+grep -F com.politedroid_4.apk archive/index.xml
+grep -F com.politedroid_5.apk archive/index.xml
+grep -F com.politedroid_6.apk repo/index.xml
+test -e archive/com.politedroid_3.apk
+test -e archive/com.politedroid_4.apk
+test -e archive/com.politedroid_5.apk
+! test -e archive/com.politedroid_6.apk
+test -e repo/com.politedroid_6.apk
+
+
+
+#------------------------------------------------------------------------------#
+echo_header 'test moving old APKs to and from the archive'
+
+REPOROOT=`create_test_dir`
+cd $REPOROOT
+cp $WORKSPACE/tests/keystore.jks $REPOROOT/
+$fdroid init --keystore keystore.jks --repo-keyalias=sova
+echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo "accepted_formats = ['txt']" >> config.py
+test -d metadata || mkdir metadata
+cp $WORKSPACE/tests/metadata/com.politedroid.txt metadata/
+sed -i '/Archive Policy:/d' metadata/com.politedroid.txt
+test -d repo || mkdir repo
+cp $WORKSPACE/tests/repo/com.politedroid_[0-9].apk repo/
+sed -i 's,archive_older = [0-9],archive_older = 3,' config.py
+
+$fdroid update --pretty --nosign
+test `grep '<package>' archive/index.xml | wc -l` -eq 1
+test `grep '<package>' repo/index.xml | wc -l` -eq 3
+grep -F com.politedroid_3.apk archive/index.xml
+grep -F com.politedroid_4.apk repo/index.xml
+grep -F com.politedroid_5.apk repo/index.xml
+grep -F com.politedroid_6.apk repo/index.xml
+test -e archive/com.politedroid_3.apk
+test -e repo/com.politedroid_4.apk
+test -e repo/com.politedroid_5.apk
+test -e repo/com.politedroid_6.apk
+
+sed -i 's,archive_older = 3,archive_older = 1,' config.py
+$fdroid update --pretty --nosign
+test `grep '<package>' archive/index.xml | wc -l` -eq 3
+test `grep '<package>' repo/index.xml | wc -l` -eq 1
+grep -F com.politedroid_3.apk archive/index.xml
+grep -F com.politedroid_4.apk archive/index.xml
+grep -F com.politedroid_5.apk archive/index.xml
+grep -F com.politedroid_6.apk repo/index.xml
+test -e archive/com.politedroid_3.apk
+test -e archive/com.politedroid_4.apk
+test -e archive/com.politedroid_5.apk
+test -e repo/com.politedroid_6.apk
+
+# disabling deletes from the archive
+sed -i 's/Build:1.3,4/Build:1.3,4\n    disable=testing deletion/' metadata/com.politedroid.txt
+$fdroid update --pretty --nosign
+test `grep '<package>' archive/index.xml | wc -l` -eq 2
+test `grep '<package>' repo/index.xml | wc -l` -eq 1
+grep -F com.politedroid_3.apk archive/index.xml
+! grep -F com.politedroid_4.apk archive/index.xml
+grep -F com.politedroid_5.apk archive/index.xml
+grep -F com.politedroid_6.apk repo/index.xml
+test -e archive/com.politedroid_3.apk
+! test -e archive/com.politedroid_4.apk
+test -e archive/com.politedroid_5.apk
+test -e repo/com.politedroid_6.apk
+
+# disabling deletes from the repo, and promotes one from the archive
+sed -i 's/Build:1.5,6/Build:1.5,6\n    disable=testing deletion/' metadata/com.politedroid.txt
+$fdroid update --pretty --nosign
+test `grep '<package>' archive/index.xml | wc -l` -eq 1
+test `grep '<package>' repo/index.xml | wc -l` -eq 1
+grep -F com.politedroid_3.apk archive/index.xml
+grep -F com.politedroid_5.apk repo/index.xml
+! grep -F com.politedroid_6.apk repo/index.xml
+test -e archive/com.politedroid_3.apk
+test -e repo/com.politedroid_5.apk
+! test -e repo/com.politedroid_6.apk
+
+
+#------------------------------------------------------------------------------#
+echo_header 'test allowing disabled signatures in repo and archive'
+
+REPOROOT=`create_test_dir`
+cd $REPOROOT
+cp $WORKSPACE/tests/keystore.jks $REPOROOT/
+$fdroid init --keystore keystore.jks --repo-keyalias=sova
+echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo "accepted_formats = ['txt']" >> config.py
+echo 'allow_disabled_algorithms = True' >> config.py
+sed -i 's,archive_older = [0-9],archive_older = 3,' config.py
+test -d metadata || mkdir metadata
+cp $WORKSPACE/tests/metadata/com.politedroid.txt metadata/
+echo 'Summary:good test version of urzip' > metadata/info.guardianproject.urzip.txt
+echo 'Summary:good MD5 sig, disabled algorithm' > metadata/org.bitbucket.tickytacky.mirrormirror.txt
+sed -i '/Archive Policy:/d' metadata/*.txt
+test -d repo || mkdir repo
+cp $WORKSPACE/tests/repo/com.politedroid_[0-9].apk \
+   $WORKSPACE/tests/org.bitbucket.tickytacky.mirrormirror_[0-9].apk \
+   $WORKSPACE/tests/urzip-badsig.apk \
+   repo/
+
+$fdroid update --pretty --nosign
+test `grep '<package>' archive/index.xml | wc -l` -eq 2
+test `grep '<package>' repo/index.xml | wc -l` -eq 6
+grep -F com.politedroid_3.apk archive/index.xml
+grep -F com.politedroid_4.apk repo/index.xml
+grep -F com.politedroid_5.apk repo/index.xml
+grep -F com.politedroid_6.apk repo/index.xml
+grep -F org.bitbucket.tickytacky.mirrormirror_1.apk archive/index.xml
+grep -F org.bitbucket.tickytacky.mirrormirror_2.apk repo/index.xml
+grep -F org.bitbucket.tickytacky.mirrormirror_3.apk repo/index.xml
+grep -F org.bitbucket.tickytacky.mirrormirror_4.apk repo/index.xml
+! grep -F urzip-badsig.apk repo/index.xml
+! grep -F urzip-badsig.apk archive/index.xml
+test -e archive/com.politedroid_3.apk
+test -e repo/com.politedroid_4.apk
+test -e repo/com.politedroid_5.apk
+test -e repo/com.politedroid_6.apk
+test -e archive/org.bitbucket.tickytacky.mirrormirror_1.apk
+test -e repo/org.bitbucket.tickytacky.mirrormirror_2.apk
+test -e repo/org.bitbucket.tickytacky.mirrormirror_3.apk
+test -e repo/org.bitbucket.tickytacky.mirrormirror_4.apk
+test -e archive/urzip-badsig.apk
+
+sed -i '/allow_disabled_algorithms/d' config.py
+$fdroid update --pretty --nosign
+test `grep '<package>' archive/index.xml | wc -l` -eq 5
+test `grep '<package>' repo/index.xml | wc -l` -eq 3
+grep -F org.bitbucket.tickytacky.mirrormirror_1.apk archive/index.xml
+grep -F org.bitbucket.tickytacky.mirrormirror_2.apk archive/index.xml
+grep -F org.bitbucket.tickytacky.mirrormirror_3.apk archive/index.xml
+grep -F org.bitbucket.tickytacky.mirrormirror_4.apk archive/index.xml
+grep -F com.politedroid_3.apk archive/index.xml
+grep -F com.politedroid_4.apk repo/index.xml
+grep -F com.politedroid_5.apk repo/index.xml
+grep -F com.politedroid_6.apk repo/index.xml
+! grep -F urzip-badsig.apk repo/index.xml
+! grep -F urzip-badsig.apk archive/index.xml
+test -e archive/org.bitbucket.tickytacky.mirrormirror_1.apk
+test -e archive/org.bitbucket.tickytacky.mirrormirror_2.apk
+test -e archive/org.bitbucket.tickytacky.mirrormirror_3.apk
+test -e archive/org.bitbucket.tickytacky.mirrormirror_4.apk
+test -e archive/com.politedroid_3.apk
+test -e archive/urzip-badsig.apk
+test -e repo/com.politedroid_4.apk
+test -e repo/com.politedroid_5.apk
+test -e repo/com.politedroid_6.apk
 
 
 #------------------------------------------------------------------------------#
@@ -517,6 +762,8 @@ grep -F '<application id=' repo/index.xml > /dev/null
 test -e repo/index.xml
 test -e repo/index.jar
 test -e repo/index-v1.jar
+test -e tmp/apkcache
+! test -z tmp/apkcache
 export ANDROID_HOME=$STORED_ANDROID_HOME
 
 
@@ -570,6 +817,8 @@ $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
 test -e repo/index-v1.jar
+test -e tmp/apkcache
+! test -z tmp/apkcache
 grep -F '<application id=' repo/index.xml > /dev/null
 
 
@@ -599,6 +848,8 @@ $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
 test -e repo/index-v1.jar
+test -e tmp/apkcache
+! test -z tmp/apkcache
 grep -F '<application id=' repo/index.xml > /dev/null
 
 
@@ -624,6 +875,8 @@ $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
 test -e repo/index-v1.jar
+test -e tmp/apkcache
+! test -z tmp/apkcache
 grep -F '<application id=' repo/index.xml > /dev/null
 
 
@@ -717,6 +970,8 @@ $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
 test -e repo/index-v1.jar
+test -e tmp/apkcache
+! test -z tmp/apkcache
 grep -F '<application id=' repo/index.xml > /dev/null
 
 # now set fake repo_keyalias

tests/stats/known_apks.txt

@@ -1,3 +1,7 @@
+com.politedroid_3.apk repo/com.politedroid 2017-06-23
+com.politedroid_4.apk repo/com.politedroid 2017-06-23
+com.politedroid_5.apk repo/com.politedroid 2017-06-23
+com.politedroid_6.apk repo/com.politedroid 2017-06-23
 fake.ota.update_1234.zip fake.ota.update 2016-03-10
 obb.main.oldversion_1444412523.apk obb.main.oldversion 2013-12-31
 obb.main.twoversions_1101613.apk obb.main.twoversions 2015-10-12

tests/update.TestCase

@@ -201,12 +201,21 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options.clean = True
         fdroidserver.update.options.delete_unknown = True
         fdroidserver.update.options.rename_apks = False
+        fdroidserver.update.options.allow_disabled_algorithms = False
 
         apps = fdroidserver.metadata.read_metadata(xref=True)
         knownapks = fdroidserver.common.KnownApks()
         apks, cachechanged = fdroidserver.update.scan_apks({}, 'repo', knownapks, False)
-        self.assertEqual(len(apks), 7)
+        self.assertEqual(len(apks), 11)
         apk = apks[0]
+        self.assertEqual(apk['packageName'], 'com.politedroid')
+        self.assertEqual(apk['versionCode'], 3)
+        self.assertEqual(apk['minSdkVersion'], '3')
+        self.assertEqual(apk['targetSdkVersion'], '3')
+        self.assertFalse('maxSdkVersion' in apk)
+        apk = apks[4]
+        self.assertEqual(apk['packageName'], 'obb.main.oldversion')
+        self.assertEqual(apk['versionCode'], 1444412523)
         self.assertEqual(apk['minSdkVersion'], '4')
         self.assertEqual(apk['targetSdkVersion'], '18')
         self.assertFalse('maxSdkVersion' in apk)
@@ -242,7 +251,7 @@ class UpdateTest(unittest.TestCase):
         config = dict()
         fdroidserver.common.fill_config_defaults(config)
         fdroidserver.update.config = config
-        os.chdir(os.path.dirname(__file__))
+        os.chdir(os.path.join(localmodule, 'tests'))
         if os.path.basename(os.getcwd()) != 'tests':
             raise Exception('This test must be run in the "tests/" subdir')
 
@@ -255,6 +264,7 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options.clean = True
         fdroidserver.update.options.rename_apks = False
         fdroidserver.update.options.delete_unknown = True
+        fdroidserver.update.options.allow_disabled_algorithms = False
 
         for icon_dir in fdroidserver.update.get_all_icon_dirs('repo'):
             if not os.path.exists(icon_dir):
@@ -282,6 +292,87 @@ class UpdateTest(unittest.TestCase):
             self.maxDiff = None
             self.assertEqual(apk, frompickle)
 
+    def test_scan_apk_signed_by_disabled_algorithms(self):
+        os.chdir(os.path.join(localmodule, 'tests'))
+        if os.path.basename(os.getcwd()) != 'tests':
+            raise Exception('This test must be run in the "tests/" subdir')
+
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.update.config = config
+
+        config['ndk_paths'] = dict()
+        config['accepted_formats'] = ['json', 'txt', 'yml']
+        fdroidserver.common.config = config
+        fdroidserver.update.config = config
+
+        fdroidserver.update.options = type('', (), {})()
+        fdroidserver.update.options.clean = True
+        fdroidserver.update.options.verbose = True
+        fdroidserver.update.options.rename_apks = False
+        fdroidserver.update.options.delete_unknown = True
+        fdroidserver.update.options.allow_disabled_algorithms = False
+
+        knownapks = fdroidserver.common.KnownApks()
+        apksourcedir = os.getcwd()
+        tmpdir = os.path.join(localmodule, '.testfiles')
+        if not os.path.exists(tmpdir):
+            os.makedirs(tmpdir)
+        tmptestsdir = tempfile.mkdtemp(prefix='test_scan_apk_signed_by_disabled_algorithms-', dir=tmpdir)
+        print('tmptestsdir', tmptestsdir)
+        os.chdir(tmptestsdir)
+        os.mkdir('repo')
+        os.mkdir('archive')
+        # setup the repo, create icons dirs, etc.
+        fdroidserver.update.scan_apks({}, 'repo', knownapks)
+        fdroidserver.update.scan_apks({}, 'archive', knownapks)
+
+        disabledsigs = ['org.bitbucket.tickytacky.mirrormirror_2.apk', ]
+        for apkName in disabledsigs:
+            shutil.copy(os.path.join(apksourcedir, apkName),
+                        os.path.join(tmptestsdir, 'repo'))
+
+            skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'repo', knownapks,
+                                                                   allow_disabled_algorithms=True,
+                                                                   archive_bad_sig=False)
+            self.assertFalse(skip)
+            self.assertIsNotNone(apk)
+            self.assertTrue(cachechanged)
+            self.assertFalse(os.path.exists(os.path.join('archive', apkName)))
+            self.assertTrue(os.path.exists(os.path.join('repo', apkName)))
+
+            # this test only works on systems with fully updated Java/jarsigner
+            # that has MD5 listed in jdk.jar.disabledAlgorithms in java.security
+            skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'repo', knownapks,
+                                                                   allow_disabled_algorithms=False,
+                                                                   archive_bad_sig=True)
+            self.assertTrue(skip)
+            self.assertIsNone(apk)
+            self.assertFalse(cachechanged)
+            self.assertTrue(os.path.exists(os.path.join('archive', apkName)))
+            self.assertFalse(os.path.exists(os.path.join('repo', apkName)))
+
+            skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'archive', knownapks,
+                                                                   allow_disabled_algorithms=False,
+                                                                   archive_bad_sig=False)
+            self.assertFalse(skip)
+            self.assertIsNotNone(apk)
+            self.assertTrue(cachechanged)
+            self.assertTrue(os.path.exists(os.path.join('archive', apkName)))
+            self.assertFalse(os.path.exists(os.path.join('repo', apkName)))
+
+        badsigs = ['urzip-badcert.apk', 'urzip-badsig.apk', 'urzip-release-unsigned.apk', ]
+        for apkName in badsigs:
+            shutil.copy(os.path.join(apksourcedir, apkName),
+                        os.path.join(tmptestsdir, 'repo'))
+
+            skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'repo', knownapks,
+                                                                   allow_disabled_algorithms=False,
+                                                                   archive_bad_sig=False)
+            self.assertTrue(skip)
+            self.assertIsNone(apk)
+            self.assertFalse(cachechanged)
+
     def test_scan_invalid_apk(self):
         os.chdir(os.path.join(localmodule, 'tests'))
         if os.path.basename(os.getcwd()) != 'tests':