Merge branch 'v2-btlog-sign' into 'master'

update signing and btlog for index-v2

Closes #1005

See merge request fdroid/fdroidserver!1133

.gitlab-ci.yml

@@ -235,6 +235,7 @@ black:
         examples/fdroid_extract_repo_pubkey.py
         examples/makebuildserver.config.py
         fdroid
+        fdroidserver/btlog.py
         fdroidserver/exception.py
         fdroidserver/gpgsign.py
         fdroidserver/lint.py

fdroidserver/btlog.py

@@ -70,7 +70,8 @@ def make_binary_transparency_log(
         if not url:
             url = common.config['repo_url'].rstrip('/')
         with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
-            fp.write("""
+            fp.write(
+                """
 # Binary Transparency Log for %s
 
 This is a log of the signed app index metadata.  This is stored in a
@@ -80,8 +81,10 @@ F-Droid repository was a publicly released file.
 
 For more info on this idea:
 * https://wiki.mozilla.org/Security/Binary_Transparency
-""" % url[:url.rindex('/')])  # strip '/repo'
-        gitrepo.index.add(['README.md', ])
+"""
+                % url[: url.rindex('/')]  # strip '/repo'
+            )
+        gitrepo.index.add(['README.md'])
         gitrepo.index.commit('add README')
 
     for repodir in repodirs:
@@ -150,13 +153,22 @@ def main():
 
     parser = ArgumentParser()
     common.setup_global_opts(parser)
-    parser.add_argument("--git-repo",
-                        default=os.path.join(os.getcwd(), 'binary_transparency'),
-                        help=_("Path to the git repo to use as the log"))
-    parser.add_argument("-u", "--url", default='https://f-droid.org',
-                        help=_("The base URL for the repo to log (default: https://f-droid.org)"))
-    parser.add_argument("--git-remote", default=None,
-                        help=_("Push the log to this git remote repository"))
+    parser.add_argument(
+        "--git-repo",
+        default=os.path.join(os.getcwd(), 'binary_transparency'),
+        help=_("Path to the git repo to use as the log"),
+    )
+    parser.add_argument(
+        "-u",
+        "--url",
+        default='https://f-droid.org',
+        help=_("The base URL for the repo to log (default: https://f-droid.org)"),
+    )
+    parser.add_argument(
+        "--git-remote",
+        default=None,
+        help=_("Push the log to this git remote repository"),
+    )
     options = parser.parse_args()
 
     if options.verbose:
@@ -182,7 +194,15 @@ def main():
         os.makedirs(tempdir, exist_ok=True)
         gitrepodir = os.path.join(options.git_repo, repodir)
         os.makedirs(gitrepodir, exist_ok=True)
-        for f in ('index.jar', 'index.xml', 'index-v1.jar', 'index-v1.json'):
+        for f in (
+            'entry.jar',
+            'entry.json',
+            'index-v1.jar',
+            'index-v1.json',
+            'index-v2.json',
+            'index.jar',
+            'index.xml',
+        ):
             dlfile = os.path.join(tempdir, f)
             dlurl = options.url + '/' + repodir + '/' + f
             http_headers_file = os.path.join(gitrepodir, f + '.HTTP-headers.json')
@@ -196,7 +216,7 @@ def main():
             r = session.head(dlurl, headers=headers, allow_redirects=False)
             if r.status_code != 200:
                 logging.debug(
-                    'HTTP Response (' + str(r.status_code) + '), did not download ' + dlurl
+                    'HTTP Response (%d), did not download %s' % (r.status_code, dlurl)
                 )
                 continue
             if etag and etag == r.headers.get('ETag'):
@@ -218,7 +238,9 @@ def main():
 
     if new_files:
         os.chdir(tempdirbase)
-        make_binary_transparency_log(repodirs, options.git_repo, options.url, 'fdroid btlog')
+        make_binary_transparency_log(
+            repodirs, options.git_repo, options.url, 'fdroid btlog'
+        )
     if options.git_remote:
         deploy.push_binary_transparency(options.git_repo, options.git_remote)
     shutil.rmtree(tempdirbase, ignore_errors=True)

fdroidserver/common.py

@@ -2592,6 +2592,7 @@ def use_androguard():
             use_androguard.show_path = False
         if options and options.verbose:
             logging.getLogger("androguard.axml").setLevel(logging.INFO)
+        logging.getLogger("androguard.core.api_specific_resources").setLevel(logging.ERROR)
         return True
     except ImportError:
         return False

tests/signindex.TestCase

@@ -76,7 +76,15 @@ class SignindexTest(unittest.TestCase):
             fp.write('# placeholder')
         shutil.copy(str(self.basedir / 'urzip.apk'), 'repo')
         index_files = []
-        for f in ('index.xml', 'index.jar', 'index-v1.json', 'index-v1.jar'):
+        for f in (
+            'entry.jar',
+            'entry.json',
+            'index-v1.jar',
+            'index-v1.json',
+            'index-v2.json',
+            'index.jar',
+            'index.xml',
+        ):
             for section in (Path('repo'), Path('archive')):
                 path = section / f
                 self.assertFalse(path.exists(), '%s should not exist yet!' % path)