Merge branch 'repomaker-api-fixes' into 'master'

repomaker/issuebot API fixes

See merge request fdroid/fdroidserver!807

fdroidserver/index.py

@@ -61,7 +61,7 @@ def make(apps, apks, repodir, archive):
         common.assert_config_keystore(common.config)
 
     # Historically the index has been sorted by App Name, so we enforce this ordering here
-    sortedids = sorted(apps, key=lambda appid: apps[appid].Name.upper())
+    sortedids = sorted(apps, key=lambda appid: apps[appid]['Name'].upper())
     sortedapps = collections.OrderedDict()
     for appid in sortedids:
         sortedapps[appid] = apps[appid]
@@ -593,8 +593,21 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
 
     # Copy the repo icon into the repo directory...
     icon_dir = os.path.join(repodir, 'icons')
-    iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
+    repo_icon = common.config.get('repo_icon', common.default_config['repo_icon'])
-    shutil.copyfile(common.config['repo_icon'], iconfilename)
+    iconfilename = os.path.join(icon_dir, os.path.basename(repo_icon))
+    if os.path.exists(repo_icon):
+        shutil.copyfile(common.config['repo_icon'], iconfilename)
+    else:
+        logging.warning(_('repo_icon %s does not exist, generating placeholder.')
+                        % repo_icon)
+        os.makedirs(os.path.dirname(iconfilename), exist_ok=True)
+        try:
+            import qrcode
+            qrcode.make(common.config['repo_url']).save(iconfilename)
+        except Exception:
+            exampleicon = os.path.join(common.get_examples_dir(),
+                                       common.default_config['repo_icon'])
+            shutil.copy(exampleicon, iconfilename)
 
 
 def extract_pubkey():

fdroidserver/scanner.py

@@ -40,6 +40,47 @@ json_per_build = DEFAULT_JSON_PER_BUILD
 MAVEN_URL_REGEX = re.compile(r"""\smaven\s*{.*?(?:setUrl|url)\s*=?\s*(?:uri)?\(?\s*["']?([^\s"']+)["']?[^}]*}""",
                              re.DOTALL)
 
+CODE_SIGNATURES = {
+    # The `apkanalyzer dex packages` output looks like this:
+    # M d 1   1       93      <packagename> <other stuff>
+    # The first column has P/C/M/F for package, class, method or field
+    # The second column has x/k/r/d for removed, kept, referenced and defined.
+    # We already filter for defined only in the apkanalyzer call. 'r' will be
+    # for things referenced but not distributed in the apk.
+    exp: re.compile(r'.[\s]*d[\s]*[0-9]*[\s]*[0-9*][\s]*[0-9]*[\s]*' + exp, re.IGNORECASE) for exp in [
+        r'(com\.google\.firebase[^\s]*)',
+        r'(com\.google\.android\.gms[^\s]*)',
+        r'(com\.google\.tagmanager[^\s]*)',
+        r'(com\.google\.analytics[^\s]*)',
+        r'(com\.android\.billing[^\s]*)',
+    ]
+}
+
+# Common known non-free blobs (always lower case):
+NON_FREE_GRADLE_LINES = {
+    exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [
+        r'flurryagent',
+        r'paypal.*mpl',
+        r'admob.*sdk.*android',
+        r'google.*ad.*view',
+        r'google.*admob',
+        r'google.*play.*services',
+        r'crittercism',
+        r'heyzap',
+        r'jpct.*ae',
+        r'youtube.*android.*player.*api',
+        r'bugsense',
+        r'crashlytics',
+        r'ouya.*sdk',
+        r'libspen23',
+        r'firebase',
+        r'''["']com.facebook.android['":]''',
+        r'cloudrail',
+        r'com.tencent.bugly',
+        r'appcenter-push',
+    ]
+}
+
 
 def get_gradle_compile_commands(build):
     compileCommands = ['compile',
@@ -59,25 +100,10 @@ def get_gradle_compile_commands(build):
 
 
 def scan_binary(apkfile):
-    usual_suspects = {
-        # The `apkanalyzer dex packages` output looks like this:
-        # M d 1   1       93      <packagename> <other stuff>
-        # The first column has P/C/M/F for package, class, method or field
-        # The second column has x/k/r/d for removed, kept, referenced and defined.
-        # We already filter for defined only in the apkanalyzer call. 'r' will be
-        # for things referenced but not distributed in the apk.
-        exp: re.compile(r'.[\s]*d[\s]*[0-9]*[\s]*[0-9*][\s]*[0-9]*[\s]*' + exp, re.IGNORECASE) for exp in [
-            r'(com\.google\.firebase[^\s]*)',
-            r'(com\.google\.android\.gms[^\s]*)',
-            r'(com\.google\.tagmanager[^\s]*)',
-            r'(com\.google\.analytics[^\s]*)',
-            r'(com\.android\.billing[^\s]*)',
-        ]
-    }
     logging.info("Scanning APK for known non-free classes.")
     result = common.SdkToolsPopen(["apkanalyzer", "dex", "packages", "--defined-only", apkfile], output=False)
     problems = 0
-    for suspect, regexp in usual_suspects.items():
+    for suspect, regexp in CODE_SIGNATURES.items():
         matches = regexp.findall(result.output)
         if matches:
             for m in set(matches):
@@ -95,31 +121,6 @@ def scan_source(build_dir, build=metadata.Build()):
 
     count = 0
 
-    # Common known non-free blobs (always lower case):
-    usual_suspects = {
-        exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [
-            r'flurryagent',
-            r'paypal.*mpl',
-            r'admob.*sdk.*android',
-            r'google.*ad.*view',
-            r'google.*admob',
-            r'google.*play.*services',
-            r'crittercism',
-            r'heyzap',
-            r'jpct.*ae',
-            r'youtube.*android.*player.*api',
-            r'bugsense',
-            r'crashlytics',
-            r'ouya.*sdk',
-            r'libspen23',
-            r'firebase',
-            r'''["']com.facebook.android['":]''',
-            r'cloudrail',
-            r'com.tencent.bugly',
-            r'appcenter-push',
-        ]
-    }
-
     whitelisted = [
         'firebase-jobdispatcher',  # https://github.com/firebase/firebase-jobdispatcher-android/blob/master/LICENSE
         'com.firebaseui',          # https://github.com/firebase/FirebaseUI-Android/blob/master/LICENSE
@@ -130,7 +131,7 @@ def scan_source(build_dir, build=metadata.Build()):
         return any(wl in s for wl in whitelisted)
 
     def suspects_found(s):
-        for n, r in usual_suspects.items():
+        for n, r in NON_FREE_GRADLE_LINES.items():
             if r.match(s) and not is_whitelisted(s):
                 yield n
 

fdroidserver/update.py

@@ -123,7 +123,9 @@ def get_all_icon_dirs(repodir):
 
 
 def disabled_algorithms_allowed():
-    return options.allow_disabled_algorithms or config['allow_disabled_algorithms']
+    return ((options is not None and options.allow_disabled_algorithms)
+            or (config is not None and config['allow_disabled_algorithms'])
+            or common.default_config['allow_disabled_algorithms'])
 
 
 def status_update_json(apps, apks):
@@ -147,7 +149,7 @@ def status_update_json(apps, apks):
             antiFeatures = output['antiFeatures']  # JSON camelCase
             if af not in antiFeatures:
                 antiFeatures[af] = dict()
-            if appid not in antiFeatures[af]:
+            if 'apps' not in antiFeatures[af]:
                 antiFeatures[af]['apps'] = set()
             antiFeatures[af]['apps'].add(appid)
 
@@ -521,7 +523,7 @@ def get_cache():
     """
     apkcachefile = get_cache_file()
     ada = disabled_algorithms_allowed()
-    if not options.clean and os.path.exists(apkcachefile):
+    if options is not None and not options.clean and os.path.exists(apkcachefile):
         with open(apkcachefile) as fp:
             apkcache = json.load(fp, object_pairs_hook=collections.OrderedDict)
         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
@@ -1778,7 +1780,7 @@ def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
 
     for icon_dir in get_all_icon_dirs(repodir):
         if os.path.exists(icon_dir):
-            if options.clean:
+            if options is not None and options.clean:
                 shutil.rmtree(icon_dir)
                 os.makedirs(icon_dir)
         else:
@@ -1971,26 +1973,26 @@ def apply_info_from_latest_apk(apps, apks):
                     bestver = apk['versionCode']
                     bestapk = apk
 
-                if app.NoSourceSince:
+                if app['NoSourceSince']:
                     apk['antiFeatures'].add('NoSourceSince')
 
-        if not app.added:
+        if not app['added']:
             logging.debug("Don't know when " + appid + " was added")
-        if not app.lastUpdated:
+        if not app['lastUpdated']:
             logging.debug("Don't know when " + appid + " was last updated")
 
         if bestver == UNSET_VERSION_CODE:
 
-            if app.Name is None:
+            if app['Name'] is None:
-                app.Name = app.AutoName or appid
+                app['Name'] = app['AutoName'] or appid
-            app.icon = None
+            app['icon'] = None
             logging.debug("Application " + appid + " has no packages")
         else:
-            if app.Name is None:
+            if app['Name'] is None:
-                app.Name = bestapk['name']
+                app['Name'] = bestapk['name']
-            app.icon = bestapk['icon'] if 'icon' in bestapk else None
+            app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
-            if app.CurrentVersionCode is None:
+            if app['CurrentVersionCode'] is None:
-                app.CurrentVersionCode = str(bestver)
+                app['CurrentVersionCode'] = str(bestver)
 
 
 def make_categories_txt(repodir, categories):

tests/index.TestCase

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import datetime
 import inspect
 import logging
 import optparse
@@ -29,17 +30,25 @@ from testcommon import TmpCwd
 GP_FINGERPRINT = 'B7C2EEFD8DAC7806AF67DFCD92EB18126BC08312A7F2D6F3862E46013C7A6135'
 
 
+class Options:
+    nosign = True
+    pretty = False
+    verbose = False
+
+
 class IndexTest(unittest.TestCase):
 
     def setUp(self):
         logging.basicConfig(level=logging.DEBUG)
         self.basedir = os.path.join(localmodule, 'tests')
+        os.chmod(os.path.join(self.basedir, 'config.py'), 0o600)
         self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
         if not os.path.exists(self.tmpdir):
             os.makedirs(self.tmpdir)
         os.chdir(self.basedir)
 
         fdroidserver.common.config = None
+        fdroidserver.common.options = Options
         config = fdroidserver.common.read_config(fdroidserver.common.options)
         config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner')
         fdroidserver.common.config = config
@@ -215,6 +224,29 @@ class IndexTest(unittest.TestCase):
             self.maxDiff = None
             self.assertEqual(json.dumps(i, indent=2), json.dumps(o, indent=2))
 
+    def test_make_v0(self):
+        tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name,
+                                       dir=self.tmpdir)
+        os.chdir(tmptestsdir)
+        os.mkdir('repo')
+        repo_icons_dir = os.path.join('repo', 'icons')
+        self.assertFalse(os.path.isdir(repo_icons_dir))
+        repodict = {
+            'address': 'https://example.com/fdroid/repo',
+            'description': 'This is just a test',
+            'icon': 'blahblah',
+            'name': 'test',
+            'timestamp': datetime.datetime.now(),
+            'version': 12,
+        }
+        requestsdict = {'install': [], 'uninstall': []}
+        fdroidserver.common.config['repo_pubkey'] = 'ffffffffffffffffffffffffffffffffff'
+        fdroidserver.index.make_v0({}, [], 'repo', repodict, requestsdict, [])
+        self.assertTrue(os.path.isdir(repo_icons_dir))
+        self.assertTrue(os.path.exists(os.path.join(repo_icons_dir,
+                                                    fdroidserver.common.default_config['repo_icon'])))
+        self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml')))
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))
@@ -222,7 +254,8 @@ if __name__ == "__main__":
     parser = optparse.OptionParser()
     parser.add_option("-v", "--verbose", action="store_true", default=False,
                       help="Spew out even more information than normal")
-    (fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
+    (options, args) = parser.parse_args()
+    Options.verbose = options.verbose
 
     newSuite = unittest.TestSuite()
     newSuite.addTest(unittest.makeSuite(IndexTest))