Merge branch 'master' into 'master'

modernize PEP8 checks to prepare for Python 3.7

See merge request fdroid/fdroidserver!512

fdroidserver/build.py

@@ -105,14 +105,14 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
             logging.debug("rsyncing " + path + " to " + ftp.getcwd())
             # TODO this should move to `vagrant rsync` from >= v1.5
             try:
-                subprocess.check_output(['rsync', '--recursive', '--perms', '--links', '--quiet', '--rsh=' +
+                subprocess.check_output(['rsync', '--recursive', '--perms', '--links', '--quiet', '--rsh='
-                                         'ssh -o StrictHostKeyChecking=no' +
+                                         + 'ssh -o StrictHostKeyChecking=no'
-                                         ' -o UserKnownHostsFile=/dev/null' +
+                                         + ' -o UserKnownHostsFile=/dev/null'
-                                         ' -o LogLevel=FATAL' +
+                                         + ' -o LogLevel=FATAL'
-                                         ' -o IdentitiesOnly=yes' +
+                                         + ' -o IdentitiesOnly=yes'
-                                         ' -o PasswordAuthentication=no' +
+                                         + ' -o PasswordAuthentication=no'
-                                         ' -p ' + str(sshinfo['port']) +
+                                         + ' -p ' + str(sshinfo['port'])
-                                         ' -i ' + sshinfo['idfile'],
+                                         + ' -i ' + sshinfo['idfile'],
                                          path,
                                          sshinfo['user'] + "@" + sshinfo['hostname'] + ":" + ftp.getcwd()],
                                         stderr=subprocess.STDOUT)

fdroidserver/common.py

@@ -352,8 +352,8 @@ def assert_config_keystore(config):
         nosigningkey = True
         logging.critical(_("'keypass' not found in config.py!"))
     if nosigningkey:
-        raise FDroidException("This command requires a signing key, " +
+        raise FDroidException("This command requires a signing key, "
-                              "you can create one using: fdroid update --create-key")
+                              + "you can create one using: fdroid update --create-key")
 
 
 def find_sdk_tools_cmd(cmd):
@@ -560,7 +560,7 @@ def clean_description(description):
         paragraph = re.sub('\r', '', paragraph)
         paragraph = re.sub('\n', ' ', paragraph)
         paragraph = re.sub(' {2,}', ' ', paragraph)
-        paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
+        paragraph = re.sub(r'^\s*(\w)', r'\1', paragraph)
         returnstring += paragraph + '\n\n'
     return returnstring.rstrip('\n')
 
@@ -575,8 +575,8 @@ def publishednameinfo(filename):
     return result
 
 
-apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
+apk_release_filename = re.compile(r'(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
-apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
+apk_release_filename_with_sigfp = re.compile(r'(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
 
 
 def apk_parse_release_filename(apkname):
@@ -2148,8 +2148,8 @@ def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdou
                              stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
                              stderr=stderr_param)
     except OSError as e:
-        raise BuildException("OSError while trying to execute " +
+        raise BuildException("OSError while trying to execute "
-                             ' '.join(commands) + ': ' + str(e))
+                             + ' '.join(commands) + ': ' + str(e))
 
     # TODO are these AsynchronousFileReader threads always exiting?
     if not stderr_to_stdout and options.verbose:
@@ -2479,7 +2479,7 @@ def metadata_find_signing_files(appid, vercode):
     sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
         glob.glob(os.path.join(sigdir, '*.EC')) + \
         glob.glob(os.path.join(sigdir, '*.RSA'))
-    extre = re.compile('(\.DSA|\.EC|\.RSA)$')
+    extre = re.compile(r'(\.DSA|\.EC|\.RSA)$')
     for sig in sigs:
         sf = extre.sub('.SF', sig)
         if os.path.isfile(sf):
@@ -3011,9 +3011,9 @@ def write_to_config(thisconfig, key, value=None, config_file=None):
 
     # regex for finding and replacing python string variable
     # definitions/initializations
-    pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
+    pattern = re.compile(r'^[\s#]*' + key + r'\s*=\s*"[^"]*"')
     repl = key + ' = "' + value + '"'
-    pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
+    pattern2 = re.compile(r'^[\s#]*' + key + r"\s*=\s*'[^']*'")
     repl2 = key + " = '" + value + "'"
 
     # If we replaced this line once, we make sure won't be a

fdroidserver/dscanner.py

@@ -179,8 +179,8 @@ class DockerDriver(object):
             )['Id']
         output = self.cli.exec_start(exec_id).decode('utf-8')
 
-        if ("Could not access the Package Manager" in output or
+        if ("Could not access the Package Manager" in output
-                "device offline" in output):
+                or "device offline" in output):
             logging.info("Device or package manager isn't up")
 
         if app_id.split('_')[0] in output:   # TODO: this is a temporary fix

fdroidserver/import.py

@@ -32,7 +32,7 @@ from . import metadata
 from .exception import FDroidException
 
 
-SETTINGS_GRADLE = re.compile('''include\s+['"]:([^'"]*)['"]''')
+SETTINGS_GRADLE = re.compile(r'''include\s+['"]:([^'"]*)['"]''')
 
 
 # Get the repo type and address from the given web page. The page is scanned
@@ -146,18 +146,18 @@ def get_metadata_from_url(app, url):
         app.SourceCode = ""
         app.WebSite = ""
     if not projecttype:
-        raise FDroidException("Unable to determine the project type. " +
+        raise FDroidException("Unable to determine the project type. "
-                              "The URL you supplied was not in one of the supported formats. " +
+                              + "The URL you supplied was not in one of the supported formats. "
-                              "Please consult the manual for a list of supported formats, " +
+                              + "Please consult the manual for a list of supported formats, "
-                              "and supply one of those.")
+                              + "and supply one of those.")
 
     # Ensure we have a sensible-looking repo address at this point. If not, we
     # might have got a page format we weren't expecting. (Note that we
     # specifically don't want git@...)
-    if ((repotype != 'bzr' and (not repo.startswith('http://') and
+    if ((repotype != 'bzr' and (not repo.startswith('http://')
-        not repo.startswith('https://') and
+        and not repo.startswith('https://')
-        not repo.startswith('git://'))) or
+        and not repo.startswith('git://')))
-            ' ' in repo):
+            or ' ' in repo):
         raise FDroidException("Repo address '{0}' does not seem to be valid".format(repo))
 
     # Get a copy of the source so we can extract some info...

fdroidserver/init.py

@@ -39,7 +39,7 @@ def disable_in_config(key, value):
     '''write a key/value to the local config.py, then comment it out'''
     with open('config.py', 'r', encoding='utf8') as f:
         data = f.read()
-    pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
+    pattern = r'\n[\s#]*' + key + r'\s*=\s*"[^"]*"'
     repl = '\n#' + key + ' = "' + value + '"'
     data = re.sub(pattern, repl, data)
     with open('config.py', 'w', encoding='utf8') as f:
@@ -106,7 +106,7 @@ def main():
                 except KeyboardInterrupt:
                     print('')
                     sys.exit(1)
-                if re.match('^\s*$', s) is not None:
+                if re.match(r'^\s*$', s) is not None:
                     test_config['sdk_path'] = default_sdk_path
                 else:
                     test_config['sdk_path'] = s
@@ -209,8 +209,8 @@ def main():
                     opensc_so = files[0]
                 else:
                     opensc_so = '/usr/lib/opensc-pkcs11.so'
-                    logging.warn('No OpenSC PKCS#11 module found, ' +
+                    logging.warn('No OpenSC PKCS#11 module found, '
-                                 'install OpenSC then edit "opensc-fdroid.cfg"!')
+                                 + 'install OpenSC then edit "opensc-fdroid.cfg"!')
             with open(os.path.join(examplesdir, 'opensc-fdroid.cfg'), 'r') as f:
                 opensc_fdroid = f.read()
             opensc_fdroid = re.sub('^library.*', 'library = ' + opensc_so, opensc_fdroid,

fdroidserver/metadata.py

@@ -811,7 +811,7 @@ def get_default_app_info(metadatapath=None):
         if os.path.exists('AndroidManifest.xml'):
             manifestroot = fdroidserver.common.parse_xml('AndroidManifest.xml')
         else:
-            pattern = re.compile(""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""")
+            pattern = re.compile(r""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""")
             for root, dirs, files in os.walk(os.getcwd()):
                 if 'build.gradle' in files:
                     p = os.path.join(root, 'build.gradle')
@@ -1036,7 +1036,7 @@ def write_yaml(mf, app):
         raise FDroidException('ruamel.yaml not instlled, can not write metadata.') from e
     if not ruamel.yaml.__version__:
         raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..')
-    m = re.match('(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
+    m = re.match(r'(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
                  ruamel.yaml.__version__)
     if not m:
         raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml')

fdroidserver/publish.py

@@ -85,7 +85,7 @@ def read_fingerprints_from_keystore():
         raise FDroidException('could not read keysotre {}'.format(config['keystore']))
 
     realias = re.compile('Alias name: (?P<alias>.+)\n')
-    resha256 = re.compile('\s+SHA256: (?P<sha256>[:0-9A-F]{95})\n')
+    resha256 = re.compile(r'\s+SHA256: (?P<sha256>[:0-9A-F]{95})\n')
     fps = {}
     for block in p.output.split(('*' * 43) + '\n' + '*' * 43):
         s_alias = realias.search(block)

fdroidserver/server.py

@@ -110,20 +110,20 @@ def update_awsbucket_s3cmd(repo_section):
     s3url = s3bucketurl + '/fdroid/'
     logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
     logging.debug(_('Running first pass with MD5 checking disabled'))
-    if subprocess.call(s3cmd_sync +
+    if subprocess.call(s3cmd_sync
-                       ['--no-check-md5', '--skip-existing',
+                       + ['--no-check-md5', '--skip-existing',
-                        '--exclude', indexxml,
+                          '--exclude', indexxml,
-                        '--exclude', indexjar,
+                          '--exclude', indexjar,
-                        '--exclude', indexv1jar,
+                          '--exclude', indexv1jar,
-                        repo_section, s3url]) != 0:
+                          repo_section, s3url]) != 0:
         raise FDroidException()
     logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
-    if subprocess.call(s3cmd_sync +
+    if subprocess.call(s3cmd_sync
-                       ['--no-check-md5',
+                       + ['--no-check-md5',
-                        '--exclude', indexxml,
+                          '--exclude', indexxml,
-                        '--exclude', indexjar,
+                          '--exclude', indexjar,
-                        '--exclude', indexv1jar,
+                          '--exclude', indexv1jar,
-                        repo_section, s3url]) != 0:
+                          repo_section, s3url]) != 0:
         raise FDroidException()
 
     logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
@@ -256,10 +256,11 @@ def update_serverwebroot(serverwebroot, repo_section):
     # the one rsync command that is allowed to run in ~/.ssh/authorized_keys.
     # (serverwebroot is guaranteed to have a trailing slash in common.py)
     logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
-    if subprocess.call(rsyncargs +
+    if subprocess.call(rsyncargs
-                       ['--exclude', indexxml, '--exclude', indexjar,
+                       + ['--exclude', indexxml,
-                        '--exclude', indexv1jar,
+                          '--exclude', indexjar,
-                        repo_section, serverwebroot]) != 0:
+                          '--exclude', indexv1jar,
+                          repo_section, serverwebroot]) != 0:
         raise FDroidException()
     if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0:
         raise FDroidException()

fdroidserver/signatures.py

@@ -57,7 +57,7 @@ def extract(options):
         sys.exit(1)
 
     # iterate over supplied APKs downlaod and extract them…
-    httpre = re.compile('https?:\/\/')
+    httpre = re.compile(r'https?:\/\/')
     for apk in options.APK:
         try:
             if os.path.isfile(apk):

fdroidserver/stats.py

@@ -113,8 +113,8 @@ def main():
 
                     destpath = os.path.join(logsdir, f)
                     destsize = ftp.stat(f).st_size
-                    if (not os.path.exists(destpath) or
+                    if not os.path.exists(destpath) \
-                            os.path.getsize(destpath) != destsize):
+                       or os.path.getsize(destpath) != destsize:
                         logging.debug("...retrieving " + f)
                         ftp.get(f, destpath)
         except Exception:
@@ -135,9 +135,9 @@ def main():
         logging.info('Processing logs...')
         appscount = Counter()
         appsvercount = Counter()
-        logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] ' + \
+        logexpr = r'(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] ' \
-            '"GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) ' + \
+            + r'"GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) ' \
-            '\d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
+            + r'\d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
         logsearch = re.compile(logexpr).search
         for logfile in glob.glob(os.path.join(logsdir, 'access-*.log.gz')):
             logging.debug('...' + logfile)

fdroidserver/update.py

@@ -53,7 +53,7 @@ UNSET_VERSION_CODE = -0x100000000
 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
-APK_LABEL_ICON_PAT = re.compile(".*\s+label='(.*)'\s+icon='(.*?)'")
+APK_LABEL_ICON_PAT = re.compile(r".*\s+label='(.*)'\s+icon='(.*?)'")
 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
 APK_PERMISSION_PAT = \
     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
@@ -1083,7 +1083,7 @@ def _get_apk_icons_src(apkfile, icon_name):
 
     """
     icons_src = dict()
-    density_re = re.compile('^res/(.*)/{}\.(png|xml)$'.format(icon_name))
+    density_re = re.compile(r'^res/(.*)/{}\.(png|xml)$'.format(icon_name))
     with zipfile.ZipFile(apkfile) as zf:
         for filename in zf.namelist():
             m = density_re.match(filename)
@@ -1853,8 +1853,8 @@ def main():
     parser.add_argument("-I", "--icons", action="store_true", default=False,
                         help=_("Resize all the icons exceeding the max pixel size and exit"))
     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
-                        help=_("Specify editor to use in interactive mode. Default " +
+                        help=_("Specify editor to use in interactive mode. Default is {path}")
-                               "is {path}").format(path='/etc/alternatives/editor'))
+                        .format(path='/etc/alternatives/editor'))
     parser.add_argument("-w", "--wiki", default=False, action="store_true",
                         help=_("Update the wiki"))
     parser.add_argument("--pretty", action="store_true", default=False,

fdroidserver/vmtools.py

@@ -368,8 +368,8 @@ class LibvirtBuildVm(FDroidBuildVm):
     def package(self, output=None, keep_box_file=False):
         if not output:
             output = "buildserver.box"
-            logger.debug('no output name set for packaging \'%s\',' +
+            logger.debug('no output name set for packaging \'%s\','
-                         'defaulting to %s', self.srvname, output)
+                         + 'defaulting to %s', self.srvname, output)
         storagePool = self.conn.storagePoolLookupByName('default')
         domainInfo = self.conn.lookupByName(self.srvname).info()
         if storagePool:
@@ -434,8 +434,8 @@ class LibvirtBuildVm(FDroidBuildVm):
                 os.remove('box.img')
 
         else:
-            logger.warn('could not connect to storage-pool \'default\',' +
+            logger.warn('could not connect to storage-pool \'default\','
-                        'skipping packaging buildserver box')
+                        + 'skipping packaging buildserver box')
 
     def box_add(self, boxname, boxfile, force=True):
         boximg = '%s_vagrant_box_image_0.img' % (boxname)

makebuildserver

@@ -602,8 +602,7 @@ def main():
         sys.exit(1)
 
     if not options.keep_box_file:
-        logger.debug('box added to vagrant, ' +
+        logger.debug("""box added to vagrant, removing generated box file '%s'""",
-                     'removing generated box file \'%s\'',
                      boxfile)
         os.remove(boxfile)
 

tests/build.TestCase

@@ -61,13 +61,13 @@ class BuildTest(unittest.TestCase):
                         os.path.join(testdir, 'source-files'))
         teststring = 'FAKE_VERSION_FOR_TESTING'
         fdroidserver.build.force_gradle_build_tools(testdir, teststring)
-        pattern = re.compile(bytes("buildToolsVersion[\s=]+'%s'\s+" % teststring, 'utf8'))
+        pattern = re.compile(r"buildToolsVersion[\s=]+'%s'\s+" % teststring)
         for p in ('source-files/fdroid/fdroidclient/build.gradle',
                   'source-files/Zillode/syncthing-silk/build.gradle',
                   'source-files/open-keychain/open-keychain/build.gradle',
                   'source-files/osmandapp/osmand/build.gradle',
                   'source-files/open-keychain/open-keychain/OpenKeychain/build.gradle'):
-            with open(os.path.join(testdir, p), 'rb') as f:
+            with open(os.path.join(testdir, p), 'r') as f:
                 filedata = f.read()
             self.assertIsNotNone(pattern.search(filedata))
 

tests/common.TestCase

@@ -204,7 +204,7 @@ class CommonTest(unittest.TestCase):
 
         with open(os.path.join(fdroidclient_testdir, 'build.gradle'), 'r') as f:
             filedata = f.read()
-        self.assertIsNotNone(re.search("\s+compileSdkVersion %s\s+" % testint, filedata))
+        self.assertIsNotNone(re.search(r"\s+compileSdkVersion %s\s+" % testint, filedata))
 
         with open(os.path.join(fdroidclient_testdir, 'AndroidManifest.xml')) as f:
             filedata = f.read()

tests/index.TestCase

@@ -53,9 +53,9 @@ class IndexTest(unittest.TestCase):
             _, fingerprint = fdroidserver.index.get_public_key_from_jar(jar)
             # comparing fingerprints should be sufficient
             if f == 'testy.jar':
-                self.assertTrue(fingerprint ==
+                self.assertEqual(fingerprint,
-                                '818E469465F96B704E27BE2FEE4C63AB' +
+                                 '818E469465F96B704E27BE2FEE4C63AB'
-                                '9F83DDF30E7A34C7371A4728D83B0BC1')
+                                 + '9F83DDF30E7A34C7371A4728D83B0BC1')
             if f == 'guardianproject.jar':
                 self.assertTrue(fingerprint == GP_FINGERPRINT)