update: allow_disabled_algorithms option to keep MD5 sigs in repo

The new policy is to move APKs with invalid signatures to the archive,
and only add those APKs to the archive's index if they have valid MD5
signatures.

closes #323
closes #292

tests/run-tests

@@ -240,6 +240,35 @@ sed -i --expression='s,timestamp="[0-9]*",timestamp="1480431575",' repo/index.xm
 diff -uw $WORKSPACE/tests/repo/index.xml repo/index.xml
 
 
+#------------------------------------------------------------------------------#
+echo_header 'test moving lots of APKs to the archive'
+
+REPOROOT=`create_test_dir`
+cd $REPOROOT
+cp $WORKSPACE/tests/keystore.jks $REPOROOT/
+$fdroid init --keystore keystore.jks --repo-keyalias=sova
+echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo "accepted_formats = ['txt']" >> config.py
+sed -i '/allow_disabled_algorithms/d' config.py
+test -d metadata || mkdir metadata
+cp $WORKSPACE/tests/metadata/*.txt metadata/
+echo 'Summary:good test version of urzip' > metadata/info.guardianproject.urzip.txt
+echo 'Summary:good MD5 sig, which is disabled algorithm' > metadata/org.bitbucket.tickytacky.mirrormirror.txt
+sed -i '/Archive Policy:/d' metadata/*.txt
+test -d repo || mkdir repo
+cp $WORKSPACE/tests/urzip.apk \
+   $WORKSPACE/tests/org.bitbucket.tickytacky.mirrormirror_[0-9].apk \
+   $WORKSPACE/tests/repo/com.politedroid_[0-9].apk \
+   $WORKSPACE/tests/repo/obb.main.twoversions_110161[357].apk \
+   repo/
+sed -i 's,archive_older = [0-9],archive_older = 3,' config.py
+
+$fdroid update --pretty --nosign
+test `grep '<package>' archive/index.xml | wc -l` -eq 5
+test `grep '<package>' repo/index.xml | wc -l` -eq 7
+
+
 #------------------------------------------------------------------------------#
 echo_header 'test per-app "Archive Policy"'
 
@@ -383,6 +412,77 @@ test -e repo/com.politedroid_5.apk
 ! test -e repo/com.politedroid_6.apk
 
 
+#------------------------------------------------------------------------------#
+echo_header 'test allowing disabled signatures in repo and archive'
+
+REPOROOT=`create_test_dir`
+cd $REPOROOT
+cp $WORKSPACE/tests/keystore.jks $REPOROOT/
+$fdroid init --keystore keystore.jks --repo-keyalias=sova
+echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo "accepted_formats = ['txt']" >> config.py
+echo 'allow_disabled_algorithms = True' >> config.py
+sed -i 's,archive_older = [0-9],archive_older = 3,' config.py
+test -d metadata || mkdir metadata
+cp $WORKSPACE/tests/metadata/com.politedroid.txt metadata/
+echo 'Summary:good test version of urzip' > metadata/info.guardianproject.urzip.txt
+echo 'Summary:good MD5 sig, disabled algorithm' > metadata/org.bitbucket.tickytacky.mirrormirror.txt
+sed -i '/Archive Policy:/d' metadata/*.txt
+test -d repo || mkdir repo
+cp $WORKSPACE/tests/repo/com.politedroid_[0-9].apk \
+   $WORKSPACE/tests/org.bitbucket.tickytacky.mirrormirror_[0-9].apk \
+   $WORKSPACE/tests/urzip-badsig.apk \
+   repo/
+
+$fdroid update --pretty --nosign
+test `grep '<package>' archive/index.xml | wc -l` -eq 2
+test `grep '<package>' repo/index.xml | wc -l` -eq 6
+grep -F com.politedroid_3.apk archive/index.xml
+grep -F com.politedroid_4.apk repo/index.xml
+grep -F com.politedroid_5.apk repo/index.xml
+grep -F com.politedroid_6.apk repo/index.xml
+grep -F org.bitbucket.tickytacky.mirrormirror_1.apk archive/index.xml
+grep -F org.bitbucket.tickytacky.mirrormirror_2.apk repo/index.xml
+grep -F org.bitbucket.tickytacky.mirrormirror_3.apk repo/index.xml
+grep -F org.bitbucket.tickytacky.mirrormirror_4.apk repo/index.xml
+! grep -F urzip-badsig.apk repo/index.xml
+! grep -F urzip-badsig.apk archive/index.xml
+test -e archive/com.politedroid_3.apk
+test -e repo/com.politedroid_4.apk
+test -e repo/com.politedroid_5.apk
+test -e repo/com.politedroid_6.apk
+test -e archive/org.bitbucket.tickytacky.mirrormirror_1.apk
+test -e repo/org.bitbucket.tickytacky.mirrormirror_2.apk
+test -e repo/org.bitbucket.tickytacky.mirrormirror_3.apk
+test -e repo/org.bitbucket.tickytacky.mirrormirror_4.apk
+test -e archive/urzip-badsig.apk
+
+sed -i '/allow_disabled_algorithms/d' config.py
+$fdroid update --pretty --nosign
+test `grep '<package>' archive/index.xml | wc -l` -eq 5
+test `grep '<package>' repo/index.xml | wc -l` -eq 3
+grep -F org.bitbucket.tickytacky.mirrormirror_1.apk archive/index.xml
+grep -F org.bitbucket.tickytacky.mirrormirror_2.apk archive/index.xml
+grep -F org.bitbucket.tickytacky.mirrormirror_3.apk archive/index.xml
+grep -F org.bitbucket.tickytacky.mirrormirror_4.apk archive/index.xml
+grep -F com.politedroid_3.apk archive/index.xml
+grep -F com.politedroid_4.apk repo/index.xml
+grep -F com.politedroid_5.apk repo/index.xml
+grep -F com.politedroid_6.apk repo/index.xml
+! grep -F urzip-badsig.apk repo/index.xml
+! grep -F urzip-badsig.apk archive/index.xml
+test -e archive/org.bitbucket.tickytacky.mirrormirror_1.apk
+test -e archive/org.bitbucket.tickytacky.mirrormirror_2.apk
+test -e archive/org.bitbucket.tickytacky.mirrormirror_3.apk
+test -e archive/org.bitbucket.tickytacky.mirrormirror_4.apk
+test -e archive/com.politedroid_3.apk
+test -e archive/urzip-badsig.apk
+test -e repo/com.politedroid_4.apk
+test -e repo/com.politedroid_5.apk
+test -e repo/com.politedroid_6.apk
+
+
 #------------------------------------------------------------------------------#
 echo_header 'rename apks with `fdroid update --rename-apks`, --nosign for speed'
 

tests/update.TestCase

@@ -201,6 +201,7 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options.clean = True
         fdroidserver.update.options.delete_unknown = True
         fdroidserver.update.options.rename_apks = False
+        fdroidserver.update.options.allow_disabled_algorithms = False
 
         apps = fdroidserver.metadata.read_metadata(xref=True)
         knownapks = fdroidserver.common.KnownApks()
@@ -250,7 +251,7 @@ class UpdateTest(unittest.TestCase):
         config = dict()
         fdroidserver.common.fill_config_defaults(config)
         fdroidserver.update.config = config
-        os.chdir(os.path.dirname(__file__))
+        os.chdir(os.path.join(localmodule, 'tests'))
         if os.path.basename(os.getcwd()) != 'tests':
             raise Exception('This test must be run in the "tests/" subdir')
 
@@ -263,6 +264,7 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options.clean = True
         fdroidserver.update.options.rename_apks = False
         fdroidserver.update.options.delete_unknown = True
+        fdroidserver.update.options.allow_disabled_algorithms = False
 
         for icon_dir in fdroidserver.update.get_all_icon_dirs('repo'):
             if not os.path.exists(icon_dir):
@@ -290,6 +292,87 @@ class UpdateTest(unittest.TestCase):
             self.maxDiff = None
             self.assertEqual(apk, frompickle)
 
+    def test_scan_apk_signed_by_disabled_algorithms(self):
+        os.chdir(os.path.join(localmodule, 'tests'))
+        if os.path.basename(os.getcwd()) != 'tests':
+            raise Exception('This test must be run in the "tests/" subdir')
+
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.update.config = config
+
+        config['ndk_paths'] = dict()
+        config['accepted_formats'] = ['json', 'txt', 'yml']
+        fdroidserver.common.config = config
+        fdroidserver.update.config = config
+
+        fdroidserver.update.options = type('', (), {})()
+        fdroidserver.update.options.clean = True
+        fdroidserver.update.options.verbose = True
+        fdroidserver.update.options.rename_apks = False
+        fdroidserver.update.options.delete_unknown = True
+        fdroidserver.update.options.allow_disabled_algorithms = False
+
+        knownapks = fdroidserver.common.KnownApks()
+        apksourcedir = os.getcwd()
+        tmpdir = os.path.join(localmodule, '.testfiles')
+        if not os.path.exists(tmpdir):
+            os.makedirs(tmpdir)
+        tmptestsdir = tempfile.mkdtemp(prefix='test_scan_apk_signed_by_disabled_algorithms-', dir=tmpdir)
+        print('tmptestsdir', tmptestsdir)
+        os.chdir(tmptestsdir)
+        os.mkdir('repo')
+        os.mkdir('archive')
+        # setup the repo, create icons dirs, etc.
+        fdroidserver.update.scan_apks({}, 'repo', knownapks)
+        fdroidserver.update.scan_apks({}, 'archive', knownapks)
+
+        disabledsigs = ['org.bitbucket.tickytacky.mirrormirror_2.apk', ]
+        for apkName in disabledsigs:
+            shutil.copy(os.path.join(apksourcedir, apkName),
+                        os.path.join(tmptestsdir, 'repo'))
+
+            skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'repo', knownapks,
+                                                                   allow_disabled_algorithms=True,
+                                                                   archive_bad_sig=False)
+            self.assertFalse(skip)
+            self.assertIsNotNone(apk)
+            self.assertTrue(cachechanged)
+            self.assertFalse(os.path.exists(os.path.join('archive', apkName)))
+            self.assertTrue(os.path.exists(os.path.join('repo', apkName)))
+
+            # this test only works on systems with fully updated Java/jarsigner
+            # that has MD5 listed in jdk.jar.disabledAlgorithms in java.security
+            skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'repo', knownapks,
+                                                                   allow_disabled_algorithms=False,
+                                                                   archive_bad_sig=True)
+            self.assertTrue(skip)
+            self.assertIsNone(apk)
+            self.assertFalse(cachechanged)
+            self.assertTrue(os.path.exists(os.path.join('archive', apkName)))
+            self.assertFalse(os.path.exists(os.path.join('repo', apkName)))
+
+            skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'archive', knownapks,
+                                                                   allow_disabled_algorithms=False,
+                                                                   archive_bad_sig=False)
+            self.assertFalse(skip)
+            self.assertIsNotNone(apk)
+            self.assertTrue(cachechanged)
+            self.assertTrue(os.path.exists(os.path.join('archive', apkName)))
+            self.assertFalse(os.path.exists(os.path.join('repo', apkName)))
+
+        badsigs = ['urzip-badcert.apk', 'urzip-badsig.apk', 'urzip-release-unsigned.apk', ]
+        for apkName in badsigs:
+            shutil.copy(os.path.join(apksourcedir, apkName),
+                        os.path.join(tmptestsdir, 'repo'))
+
+            skip, apk, cachechanged = fdroidserver.update.scan_apk({}, apkName, 'repo', knownapks,
+                                                                   allow_disabled_algorithms=False,
+                                                                   archive_bad_sig=False)
+            self.assertTrue(skip)
+            self.assertIsNone(apk)
+            self.assertFalse(cachechanged)
+
     def test_scan_invalid_apk(self):
         os.chdir(os.path.join(localmodule, 'tests'))
         if os.path.basename(os.getcwd()) != 'tests':