Merge branch 'random-bag-of-fies' into 'master'

random bag of fixes

Closes #719

See merge request fdroid/fdroidserver!710

.gitlab-ci.yml

@@ -1,8 +1,12 @@
 
+variables:
+  pip: pip3 --timeout 100 --retries 10
+
+
 test:
   image: registry.gitlab.com/fdroid/ci-images-server:latest
   script:
-    - pip3 install -e .
+    - $pip install -e .
     - cd tests
     - ./complete-ci-tests
 
@@ -104,8 +108,8 @@ ubuntu_xenial_pip:
     - rm -rf env
     - pyvenv env
     - . env/bin/activate
-    - pip3 install --upgrade babel pip setuptools
+    - $pip install --upgrade babel pip setuptools
-    - pip3 install -e .
+    - $pip install -e .
     - ./setup.py compile_catalog
     - ./tests/run-tests
 
@@ -135,13 +139,12 @@ lint_format_safety_bandit_checks:
   script:
     - apk add --no-cache bash dash ca-certificates python3
     - python3 -m ensurepip
-    - pip3 install Babel 'bandit<1.6.0' pycodestyle pyflakes 'pylint<2.0' safety
+    - $pip install Babel 'bandit<1.6.0' pycodestyle pyflakes 'pylint<2.0' safety
     - export EXITVALUE=0
     - ./hooks/pre-commit || export EXITVALUE=1
     - bandit
         -ii
-        -s B110,B310,B322,B404,B408,B410,B603,B607
+        -s B110,B322,B404,B408,B410,B603,B607
-        -x fdroidserver/dscanner.py,docker/install_agent.py,docker/drozer.py
         -r $CI_PROJECT_DIR fdroid
         || export EXITVALUE=1
     - safety check --full-report || export EXITVALUE=1
@@ -185,7 +188,7 @@ fedora_latest:
                      which
     - ./setup.py compile_catalog sdist
     - useradd -m -c "test account" --password "fakepassword"  testuser
-    - su testuser --login --command "cd `pwd`; pip3 install --user dist/fdroidserver-*.tar.gz"
+    - su testuser --login --command "cd `pwd`; $pip install --user dist/fdroidserver-*.tar.gz"
     - test -e ~testuser/.local/share/locale/de/LC_MESSAGES/fdroidserver.mo
     - wget --no-verbose -O tools.zip https://dl.google.com/android/repository/tools_r25.2.5-linux.zip
     - unzip -q tools.zip
@@ -223,5 +226,5 @@ gradle:
         test -z "$CHANGED" && exit;
       fi
     - python3 -m ensurepip
-    - pip3 install beautifulsoup4 requests
+    - $pip install beautifulsoup4 requests
     - ./tests/gradle-release-checksums.py

completion/bash-completion

@@ -73,7 +73,7 @@ __vercode() {
 			line="${line#*,}"
 			printf "${line%%,*} "
 		fi
-		done < "metadata/${p}.txt" )" -- $cur ) )
+		done < "metadata/${p}.yml" )" -- $cur ) )
 }
 
 __complete_options() {
@@ -253,7 +253,7 @@ __complete_btlog() {
 
 __complete_mirror() {
 	opts="-v"
-	lopts="--archive --output-dir"
+	lopts="--all --archive --build-logs --pgp-signatures --src-tarballs --output-dir"
 	__complete_options
 }
 

examples/template.yml

@@ -2,7 +2,7 @@ AuthorName: .
 WebSite: ''
 Bitcoin: null
 Litecoin: null
-Donation: null
+Donate: null
 
 License: Unknown
 Categories:
@@ -17,5 +17,5 @@ Summary: .
 Description: |
     .
 
-Archive Policy: 2 versions
+ArchivePolicy: 2 versions
-Requires Root: No
+RequiresRoot: No

fdroidserver/checkupdates.py

@@ -64,7 +64,7 @@ def check_http(app):
         if len(urlcode) > 0:
             logging.debug("...requesting {0}".format(urlcode))
             req = urllib.request.Request(urlcode, None)
-            resp = urllib.request.urlopen(req, None, 20)
+            resp = urllib.request.urlopen(req, None, 20)  # nosec B310 scheme is filtered above
             page = resp.read().decode('utf-8')
 
             m = re.search(codeex, page)
@@ -77,7 +77,7 @@ def check_http(app):
             if urlver != '.':
                 logging.debug("...requesting {0}".format(urlver))
                 req = urllib.request.Request(urlver, None)
-                resp = urllib.request.urlopen(req, None, 20)
+                resp = urllib.request.urlopen(req, None, 20)  # nosec B310 scheme is filtered above
                 page = resp.read().decode('utf-8')
 
             m = re.search(verex, page)
@@ -295,7 +295,7 @@ def check_gplay(app):
     headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
     req = urllib.request.Request(url, None, headers)
     try:
-        resp = urllib.request.urlopen(req, None, 20)
+        resp = urllib.request.urlopen(req, None, 20)  # nosec B310 URL base is hardcoded above
         page = resp.read().decode()
     except urllib.error.HTTPError as e:
         return (None, str(e.code))
@@ -358,6 +358,18 @@ def possible_subdirs(app):
             yield subdir
 
 
+def _getappname(app):
+    if app.Name:
+        return app.Name
+    if app.AutoName:
+        return app.AutoName
+    return app.id
+
+
+def _getcvname(app):
+    return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
+
+
 def fetch_autoname(app, tag):
 
     if not app.RepoType or app.UpdateCheckMode in ('None', 'Static') \
@@ -393,7 +405,7 @@ def fetch_autoname(app, tag):
         if new_name != app.AutoName:
             app.AutoName = new_name
             if not commitmsg:
-                commitmsg = "Set autoname of {0}".format(common.getappname(app))
+                commitmsg = "Set autoname of {0}".format(_getappname(app))
     else:
         logging.debug("...couldn't get autoname")
 
@@ -472,8 +484,8 @@ def checkupdates_app(app):
     commitmsg = fetch_autoname(app, tag)
 
     if updating:
-        name = common.getappname(app)
+        name = _getappname(app)
-        ver = common.getcvname(app)
+        ver = _getcvname(app)
         logging.info('...updating to version %s' % ver)
         commitmsg = 'Update CV of %s to %s' % (name, ver)
 
@@ -513,8 +525,8 @@ def checkupdates_app(app):
                 commit = commit.replace('%c', newbuild.versionCode)
                 newbuild.commit = commit
                 app.builds.append(newbuild)
-                name = common.getappname(app)
+                name = _getappname(app)
-                ver = common.getcvname(app)
+                ver = _getcvname(app)
                 commitmsg = "Update %s to %s" % (name, ver)
         else:
             logging.warning('Invalid auto update mode "' + mode + '" on ' + app.id)
@@ -610,24 +622,24 @@ def main():
             version, reason = check_gplay(app)
             if version is None:
                 if reason == '404':
-                    logging.info("{0} is not in the Play Store".format(common.getappname(app)))
+                    logging.info("{0} is not in the Play Store".format(_getappname(app)))
                 else:
-                    logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
+                    logging.info("{0} encountered a problem: {1}".format(_getappname(app), reason))
             if version is not None:
                 stored = app.CurrentVersion
                 if not stored:
                     logging.info("{0} has no Current Version but has version {1} on the Play Store"
-                                 .format(common.getappname(app), version))
+                                 .format(_getappname(app), version))
                 elif LooseVersion(stored) < LooseVersion(version):
                     logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
-                                 .format(common.getappname(app), version, stored))
+                                 .format(_getappname(app), version, stored))
                 else:
                     if stored != version:
                         logging.info("{0} has version {1} on the Play Store, which differs from {2}"
-                                     .format(common.getappname(app), version, stored))
+                                     .format(_getappname(app), version, stored))
                     else:
                         logging.info("{0} has the same version {1} on the Play Store"
-                                     .format(common.getappname(app), version))
+                                     .format(_getappname(app), version))
         update_wiki(gplaylog, None)
         return
 

fdroidserver/common.py

@@ -664,18 +664,6 @@ def getsrcname(app, build):
     return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
 
 
-def getappname(app):
-    if app.Name:
-        return app.Name
-    if app.AutoName:
-        return app.AutoName
-    return app.id
-
-
-def getcvname(app):
-    return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
-
-
 def get_build_dir(app):
     '''get the dir that this app will be built in'''
 

fdroidserver/import.py

@@ -40,8 +40,9 @@ SETTINGS_GRADLE = re.compile(r'''include\s+['"]:([^'"]*)['"]''')
 # when one of these is found it's assumed that's the information we want.
 # Returns repotype, address, or None, reason
 def getrepofrompage(url):
-
+    if not url.startswith('http'):
-    req = urllib.request.urlopen(url)
+        return (None, _('{url} does not start with "http"!'.format(url=url)))
+    req = urllib.request.urlopen(url)  # nosec B310 non-http URLs are filtered out
     if req.getcode() != 200:
         return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
     page = req.read().decode(req.headers.get_content_charset())

fdroidserver/mirror.py

@@ -46,16 +46,26 @@ def main():
     parser.add_argument("url", nargs='?',
                         help=_('Base URL to mirror, can include the index signing key '
                                + 'using the query string: ?fingerprint='))
+    parser.add_argument("--all", action='store_true', default=False,
+                        help=_("Mirror the full repo and archive, all file types."))
     parser.add_argument("--archive", action='store_true', default=False,
                         help=_("Also mirror the full archive section"))
     parser.add_argument("--build-logs", action='store_true', default=False,
                         help=_("Include the build logs in the mirror"))
+    parser.add_argument("--pgp-signatures", action='store_true', default=False,
+                        help=_("Include the PGP signature .asc files in the mirror"))
     parser.add_argument("--src-tarballs", action='store_true', default=False,
                         help=_("Include the source tarballs in the mirror"))
     parser.add_argument("--output-dir", default=None,
                         help=_("The directory to write the mirror to"))
     options = parser.parse_args()
 
+    if options.all:
+        options.archive = True
+        options.build_logs = True
+        options.pgp_signatures = True
+        options.src_tarballs = True
+
     if options.url is None:
         logging.error(_('A URL is required as an argument!') + '\n')
         parser.print_help()
@@ -152,7 +162,8 @@ def main():
                     if not os.path.exists(f) \
                        or (f.endswith('.apk') and os.path.getsize(f) != package['size']):
                         urls.append(_append_to_url_path(section, f))
-                        urls.append(_append_to_url_path(section, f + '.asc'))
+                        if options.pgp_signatures:
+                            urls.append(_append_to_url_path(section, f + '.asc'))
                         if options.build_logs and f.endswith('.apk'):
                             urls.append(_append_to_url_path(section, f[:-4] + '.log.gz'))
 

fdroidserver/update.py

@@ -714,7 +714,10 @@ def _set_localized_text_entry(app, locale, key, f):
     with open(f, errors='replace') as fp:
         text = fp.read()[:limit]
         if len(text) > 0:
-            localized[key] = text
+            if key in ('name', 'summary', 'video'):  # hardcoded as a single line
+                localized[key] = text.strip('\n')
+            else:
+                localized[key] = text
 
 
 def _set_author_entry(app, key, f):

makebuildserver

@@ -362,8 +362,8 @@ CACHE_FILES = [
      '1f3067073041bc44554d0efe5d402a33bc3d3c93cc39ab684f308586d732a80d'),
     ('https://services.gradle.org/distributions/gradle-6.0.1-bin.zip',
      'd364b7098b9f2e58579a3603dc0a12a1991353ac58ed339316e6762b21efba44'),
-    ('https://services.gradle.org/distributions/gradle-6.1-bin.zip',
+    ('https://services.gradle.org/distributions/gradle-6.1.1-bin.zip',
-     'd0c43d14e1c70a48b82442f435d06186351a2d290d72afd5b8866f15e6d7038a'),
+     '9d94e6e4a28ad328072ef6e56bce79a810494ae756751fdcedffdeaf27c093b1'),
     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
@@ -531,6 +531,10 @@ def main():
         open(logfilename, 'a').close()  # create blank file
     log_cm = vagrant.make_file_cm(logfilename)
     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
+    # https://phoenhex.re/2018-03-25/not-a-vagrant-bug
+    os_env = os.environ.copy()
+    os_env['VAGRANT_DISABLE_VBOXSYMLINKCREATE'] = '1'
+    v.env = os_env
 
     if options.verbosity >= 2:
         tail = fdroidserver.tail.Tail(logfilename)

tests/checkupdates.TestCase

@@ -19,10 +19,11 @@ if localmodule not in sys.path:
 
 import fdroidserver.checkupdates
 import fdroidserver.metadata
+from fdroidserver.exception import FDroidException
 
 
-class CommonTest(unittest.TestCase):
+class CheckupdatesTest(unittest.TestCase):
-    '''fdroidserver/common.py'''
+    '''fdroidserver/checkupdates.py'''
 
     def setUp(self):
         logging.basicConfig(level=logging.DEBUG)
@@ -123,6 +124,17 @@ class CommonTest(unittest.TestCase):
             self.assertEqual(vername, '1.1.9')
             self.assertEqual(vercode, '10109')
 
+    def test_check_http_blocks_unknown_schemes(self):
+        app = fdroidserver.metadata.App()
+        for scheme in ('file', 'ssh', 'http', ';pwn'):
+            app.id = scheme
+            faked = scheme + '://fake.url/for/testing/scheme'
+            app.UpdateCheckData = faked + '|ignored|' + faked + '|ignored'
+            app.metadatapath = 'metadata/' + app.id + '.yml'
+            vername, vercode = fdroidserver.checkupdates.check_http(app)
+            self.assertIsNone(vername)
+            self.assertTrue(FDroidException.__name__ in vercode)
+
     def test_check_http_ignore(self):
         fdroidserver.checkupdates.options = mock.Mock()
 
@@ -151,5 +163,5 @@ if __name__ == "__main__":
     (fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
 
     newSuite = unittest.TestSuite()
-    newSuite.addTest(unittest.makeSuite(CommonTest))
+    newSuite.addTest(unittest.makeSuite(CheckupdatesTest))
     unittest.main(failfast=False)

tests/config.py

@@ -26,8 +26,8 @@ keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="
 keydname = "CN=sova, OU=F-Droid"
 
 mirrors = (
-    'https://foo.bar/fdroid',
     'http://foobarfoobarfoobar.onion/fdroid',
+    'https://foo.bar/fdroid',
 )
 
 update_stats = True

tests/repo/index-v1.json

@@ -198,9 +198,9 @@
           "description": "full description\n",
           "featureGraphic": "featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png",
           "icon": "icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png",
-          "name": "title\n",
+          "name": "title",
-          "summary": "short description\n",
+          "summary": "short description",
-          "video": "video\n"
+          "video": "video"
         }
       }
     }

tests/update.TestCase

@@ -114,9 +114,9 @@ class UpdateTest(unittest.TestCase):
             if packageName == 'info.guardianproject.urzip':
                 self.assertEqual(7, len(app['localized']['en-US']))
                 self.assertEqual('full description\n', app['localized']['en-US']['description'])
-                self.assertEqual('title\n', app['localized']['en-US']['name'])
+                self.assertEqual('title', app['localized']['en-US']['name'])
-                self.assertEqual('short description\n', app['localized']['en-US']['summary'])
+                self.assertEqual('short description', app['localized']['en-US']['summary'])
-                self.assertEqual('video\n', app['localized']['en-US']['video'])
+                self.assertEqual('video', app['localized']['en-US']['video'])
                 self.assertEqual('icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png',
                                  app['localized']['en-US']['icon'])
                 self.assertEqual('featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png',
@@ -140,7 +140,7 @@ class UpdateTest(unittest.TestCase):
             elif packageName == 'com.nextcloud.client.dev':
                 self.assertEqual('Nextcloud Dev', app['localized']['en-US']['name'])
                 self.assertEqual(586, len(app['localized']['en-US']['description']))
-                self.assertEqual(79, len(app['localized']['en-US']['summary']))
+                self.assertEqual(78, len(app['localized']['en-US']['summary']))
             elif packageName == 'eu.siacs.conversations':
                 self.assertEqual('Conversations', app['localized']['en-US']['name'])