Merge branch 'py3' into 'master'

Python 3

I tried to keep commits separate, so if anything causes trouble, it can be reverted or changed easily.

* pre-commit hooks pass
* all tests pass
* My use of `build`, `checkupdates`, `lint`, `import`, `publish` and `update` work as usual
* 2to3 does not report anything useful anymore (only useless parentheses and list() encapsulation of iterators)
* rewritemeta works exactly as usual

CC @eighthave

See merge request !88

.gitignore

@@ -15,4 +15,4 @@ pylint.parseable
 docs/html/
 
 # files generated by tests
-tests/getsig/tmp/
+tmp/

.gitlab-ci.yml

@@ -3,6 +3,7 @@ image: mvdan/fdroid-ci:latest
 test:
   script:
     - apt-get update
-    - apt-get install -y python-dev gcc
+    - apt-get install -y python3-dev gcc
+    - pip3 install -e .
     - cd tests
     - ./complete-ci-tests

README.md

@@ -23,8 +23,7 @@ install, and keep track of updates on your device.
 
 ### Installing
 
-Note that only Python 2 is supported. We recommend version 2.7.7 or
+Note that only Python 3 is supported. We recommend version 3.4 or later.
-later.
 
 The easiest way to install the `fdroidserver` tools is on Ubuntu, Mint or other
 Ubuntu based distributions, you can install using:
@@ -56,7 +55,7 @@ or Cygwin, you can use it:
 
 Python's `pip` also works:
 
-	sudo pip install fdroidserver
+	sudo pip3 install fdroidserver
 
 The combination of `virtualenv` and `pip` is great for testing out the
 latest versions of `fdroidserver`. Using `pip`, `fdroidserver` can
@@ -67,7 +66,7 @@ via other mechanisms like Brew/dnf/pacman/emerge/Fink/MacPorts.
 
 For Debian based distributions:
 
-	apt-get install python-dev python-pip python-virtualenv
+	apt-get install python3-dev python3-pip virtualenv
 
 Then here's how to install:
 
@@ -75,5 +74,5 @@ Then here's how to install:
 	cd fdroidserver
 	virtualenv env/
 	source env/bin/activate
-	pip install -e .
+	pip3 install -e .
-	python2 setup.py install
+	python3 setup.py install

docs/fdroid.texi

@@ -83,9 +83,7 @@ intended usage. At the very least, you'll need:
 @item
 GNU/Linux
 @item
-Python 2.x
+Python 3.4 or later
-To be sure of being able to process all apk files without error, you need
-2.7.7 or later. See @code{http://bugs.python.org/issue14315}.
 @item
 The Android SDK Tools and Build-tools.
 Note that F-Droid does not assume that you have the Android SDK in your

examples/config.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 # Copy this file to config.py, then amend the settings below according to
 # your system configuration.

examples/makebuildserver.config.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 #
 # You may want to alter these before running ./makebuildserver
 

fdroid

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # fdroid.py - part of the FDroid server tools
 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com

fdroidserver/build.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # build.py - part of the FDroid server tools
 # Copyright (C) 2010-2014, Ciaran Gultnieks, ciaran@ciarang.com
@@ -28,15 +27,15 @@ import tarfile
 import traceback
 import time
 import json
-from ConfigParser import ConfigParser
+from configparser import ConfigParser
 from argparse import ArgumentParser
 import logging
 
-import common
+from . import common
-import net
+from . import net
-import metadata
+from . import metadata
-import scanner
+from . import scanner
-from common import FDroidException, BuildException, VCSException, FDroidPopen, SdkToolsPopen
+from .common import FDroidException, BuildException, VCSException, FDroidPopen, SdkToolsPopen
 
 try:
     import paramiko
@@ -463,7 +462,7 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir,
         if not ndk_path:
             logging.critical("Android NDK version '%s' could not be found!" % build.ndk or 'r10e')
             logging.critical("Configured versions:")
-            for k, v in config['ndk_paths'].iteritems():
+            for k, v in config['ndk_paths'].items():
                 if k.endswith("_orig"):
                     continue
                 logging.critical("  %s: %s" % (k, v))
@@ -1071,7 +1070,7 @@ def main():
         raise FDroidException("No apps to process.")
 
     if options.latest:
-        for app in apps.itervalues():
+        for app in apps.values():
             for build in reversed(app.builds):
                 if build.disable and not options.force:
                     continue
@@ -1087,7 +1086,7 @@ def main():
     # Build applications...
     failed_apps = {}
     build_succeeded = []
-    for appid, app in apps.iteritems():
+    for appid, app in apps.items():
 
         first = True
 

fdroidserver/checkupdates.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # checkupdates.py - part of the FDroid server tools
 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
@@ -21,20 +20,21 @@
 import sys
 import os
 import re
-import urllib2
+import urllib.request
+import urllib.error
 import time
 import subprocess
 from argparse import ArgumentParser
 import traceback
-import HTMLParser
+from html.parser import HTMLParser
 from distutils.version import LooseVersion
 import logging
 import copy
 
-import common
+from . import common
-import metadata
+from . import metadata
-from common import VCSException, FDroidException
+from .common import VCSException, FDroidException
-from metadata import MetaDataException
+from .metadata import MetaDataException
 
 
 # Check for a new version by looking at a document retrieved via HTTP.
@@ -52,8 +52,8 @@ def check_http(app):
         vercode = "99999999"
         if len(urlcode) > 0:
             logging.debug("...requesting {0}".format(urlcode))
-            req = urllib2.Request(urlcode, None)
+            req = urllib.request.Request(urlcode, None)
-            resp = urllib2.urlopen(req, None, 20)
+            resp = urllib.request.urlopen(req, None, 20)
             page = resp.read()
 
             m = re.search(codeex, page)
@@ -65,8 +65,8 @@ def check_http(app):
         if len(urlver) > 0:
             if urlver != '.':
                 logging.debug("...requesting {0}".format(urlver))
-                req = urllib2.Request(urlver, None)
+                req = urllib.request.Request(urlver, None)
-                resp = urllib2.urlopen(req, None, 20)
+                resp = urllib.request.urlopen(req, None, 20)
                 page = resp.read()
 
             m = re.search(verex, page)
@@ -280,11 +280,11 @@ def check_gplay(app):
     time.sleep(15)
     url = 'https://play.google.com/store/apps/details?id=' + app.id
     headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
-    req = urllib2.Request(url, None, headers)
+    req = urllib.request.Request(url, None, headers)
     try:
-        resp = urllib2.urlopen(req, None, 20)
+        resp = urllib.request.urlopen(req, None, 20)
         page = resp.read()
-    except urllib2.HTTPError as e:
+    except urllib.error.HTTPError as e:
         return (None, str(e.code))
     except Exception as e:
         return (None, 'Failed:' + str(e))
@@ -293,7 +293,7 @@ def check_gplay(app):
 
     m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*</div>', page)
     if m:
-        html_parser = HTMLParser.HTMLParser()
+        html_parser = HTMLParser()
         version = html_parser.unescape(m.group(1))
 
     if version == 'Varies with device':
@@ -559,7 +559,7 @@ def main():
                                      .format(common.getappname(app), version))
         return
 
-    for appid, app in apps.iteritems():
+    for appid, app in apps.items():
 
         if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
             logging.debug("Nothing to do for {0}...".format(appid))

fdroidserver/common.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 #
 # common.py - part of the FDroid server tools
 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
@@ -20,6 +20,7 @@
 # common.py is imported by all modules, so do not import third-party
 # libraries here as they will become a requirement for all commands.
 
+import io
 import os
 import sys
 import re
@@ -32,19 +33,15 @@ import operator
 import logging
 import hashlib
 import socket
+import base64
 import xml.etree.ElementTree as XMLElementTree
 
-try:
+from queue import Queue
-    # Python 2
-    from Queue import Queue
-except ImportError:
-    # Python 3
-    from queue import Queue
 
 from zipfile import ZipFile
 
-import metadata
+import fdroidserver.metadata
-from fdroidserver.asynchronousfilereader import AsynchronousFileReader
+from .asynchronousfilereader import AsynchronousFileReader
 
 
 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
@@ -206,7 +203,9 @@ def read_config(opts, config_file='config.py'):
     config = {}
 
     logging.debug("Reading %s" % config_file)
-    execfile(config_file, config)
+    with io.open(config_file, "rb") as f:
+        code = compile(f.read(), config_file, 'exec')
+        exec(code, None, config)
 
     # smartcardoptions must be a list since its command line args for Popen
     if 'smartcardoptions' in config:
@@ -244,9 +243,9 @@ def read_config(opts, config_file='config.py'):
             config[k] = clean_description(config[k])
 
     if 'serverwebroot' in config:
-        if isinstance(config['serverwebroot'], basestring):
+        if isinstance(config['serverwebroot'], str):
             roots = [config['serverwebroot']]
-        elif all(isinstance(item, basestring) for item in config['serverwebroot']):
+        elif all(isinstance(item, str) for item in config['serverwebroot']):
             roots = config['serverwebroot']
         else:
             raise TypeError('only accepts strings, lists, and tuples')
@@ -339,9 +338,9 @@ def write_password_file(pwtype, password=None):
     filename = '.fdroid.' + pwtype + '.txt'
     fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
     if password is None:
-        os.write(fd, config[pwtype])
+        os.write(fd, config[pwtype].encode('utf-8'))
     else:
-        os.write(fd, password)
+        os.write(fd, password.encode('utf-8'))
     os.close(fd)
     config[pwtype + 'file'] = filename
 
@@ -378,7 +377,7 @@ def read_app_args(args, allapps, allow_vercodes=False):
         return allapps
 
     apps = {}
-    for appid, app in allapps.iteritems():
+    for appid, app in allapps.items():
         if appid in vercodes:
             apps[appid] = app
 
@@ -391,7 +390,7 @@ def read_app_args(args, allapps, allow_vercodes=False):
         raise FDroidException("No packages specified")
 
     error = False
-    for appid, app in apps.iteritems():
+    for appid, app in apps.items():
         vc = vercodes[appid]
         if not vc:
             continue
@@ -486,9 +485,9 @@ def getvcs(vcstype, remote, local):
 
 
 def getsrclibvcs(name):
-    if name not in metadata.srclibs:
+    if name not in fdroidserver.metadata.srclibs:
         raise VCSException("Missing srclib " + name)
-    return metadata.srclibs[name]['Repo Type']
+    return fdroidserver.metadata.srclibs[name]['Repo Type']
 
 
 class vcs:
@@ -532,6 +531,7 @@ class vcs:
         # automatically if either of those things changes.
         fdpath = os.path.join(self.local, '..',
                               '.fdroidvcs-' + os.path.basename(self.local))
+        fdpath = os.path.normpath(fdpath)
         cdata = self.repotype() + ' ' + self.remote
         writeback = True
         deleterepo = False
@@ -563,7 +563,8 @@ class vcs:
 
         # If necessary, write the .fdroidvcs file.
         if writeback and not self.clone_failed:
-            with open(fdpath, 'w') as f:
+            os.makedirs(os.path.dirname(fdpath), exist_ok=True)
+            with open(fdpath, 'w+') as f:
                 f.write(cdata)
 
         if exc is not None:
@@ -947,7 +948,7 @@ def retrieve_string(app_dir, string, xmlfiles=None):
         if element.text is None:
             return ""
         s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
-        return s.strip()
+        return s.decode('utf-8').strip()
 
     for path in xmlfiles:
         if not os.path.isfile(path):
@@ -995,7 +996,7 @@ def fetch_real_name(app_dir, flavours):
             continue
         if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
             continue
-        label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
+        label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
         result = retrieve_string_singleline(app_dir, label)
         if result:
             result = result.strip()
@@ -1008,15 +1009,16 @@ def get_library_references(root_dir):
     proppath = os.path.join(root_dir, 'project.properties')
     if not os.path.isfile(proppath):
         return libraries
-    for line in file(proppath):
+    with open(proppath, 'r') as f:
-        if not line.startswith('android.library.reference.'):
+        for line in f:
-            continue
+            if not line.startswith('android.library.reference.'):
-        path = line.split('=')[1].strip()
+                continue
-        relpath = os.path.join(root_dir, path)
+            path = line.split('=')[1].strip()
-        if not os.path.isdir(relpath):
+            relpath = os.path.join(root_dir, path)
-            continue
+            if not os.path.isdir(relpath):
-        logging.debug("Found subproject at %s" % path)
+                continue
-        libraries.append(path)
+            logging.debug("Found subproject at %s" % path)
+            libraries.append(path)
     return libraries
 
 
@@ -1082,38 +1084,39 @@ def parse_androidmanifests(paths, app):
         package = None
 
         if gradle:
-            for line in file(path):
+            with open(path, 'r') as f:
-                if gradle_comment.match(line):
+                for line in f:
-                    continue
+                    if gradle_comment.match(line):
-                # Grab first occurence of each to avoid running into
+                        continue
-                # alternative flavours and builds.
+                    # Grab first occurence of each to avoid running into
-                if not package:
+                    # alternative flavours and builds.
-                    matches = psearch_g(line)
+                    if not package:
-                    if matches:
+                        matches = psearch_g(line)
-                        s = matches.group(2)
+                        if matches:
-                        if app_matches_packagename(app, s):
+                            s = matches.group(2)
-                            package = s
+                            if app_matches_packagename(app, s):
-                if not version:
+                                package = s
-                    matches = vnsearch_g(line)
+                    if not version:
-                    if matches:
+                        matches = vnsearch_g(line)
-                        version = matches.group(2)
+                        if matches:
-                if not vercode:
+                            version = matches.group(2)
-                    matches = vcsearch_g(line)
+                    if not vercode:
-                    if matches:
+                        matches = vcsearch_g(line)
-                        vercode = matches.group(1)
+                        if matches:
+                            vercode = matches.group(1)
         else:
             try:
                 xml = parse_xml(path)
                 if "package" in xml.attrib:
-                    s = xml.attrib["package"].encode('utf-8')
+                    s = xml.attrib["package"]
                     if app_matches_packagename(app, s):
                         package = s
                 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
-                    version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
+                    version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
                     base_dir = os.path.dirname(path)
                     version = retrieve_string_singleline(base_dir, version)
                 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
-                    a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
+                    a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
                     if string_is_integer(a):
                         vercode = a
             except Exception:
@@ -1209,10 +1212,10 @@ def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
         if '/' in name:
             name, subdir = name.split('/', 1)
 
-    if name not in metadata.srclibs:
+    if name not in fdroidserver.metadata.srclibs:
         raise VCSException('srclib ' + name + ' not found.')
 
-    srclib = metadata.srclibs[name]
+    srclib = fdroidserver.metadata.srclibs[name]
 
     sdir = os.path.join(srclib_dir, name)
 
@@ -1510,7 +1513,7 @@ def getpaths_map(build_dir, globpaths):
 def getpaths(build_dir, globpaths):
     paths_map = getpaths_map(build_dir, globpaths)
     paths = set()
-    for k, v in paths_map.iteritems():
+    for k, v in paths_map.items():
         for p in v:
             paths.add(p)
     return paths
@@ -1526,12 +1529,13 @@ class KnownApks:
         self.path = os.path.join('stats', 'known_apks.txt')
         self.apks = {}
         if os.path.isfile(self.path):
-            for line in file(self.path):
+            with open(self.path, 'r') as f:
-                t = line.rstrip().split(' ')
+                for line in f:
-                if len(t) == 2:
+                    t = line.rstrip().split(' ')
-                    self.apks[t[0]] = (t[1], None)
+                    if len(t) == 2:
-                else:
+                        self.apks[t[0]] = (t[1], None)
-                    self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
+                    else:
+                        self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
         self.changed = False
 
     def writeifchanged(self):
@@ -1542,7 +1546,7 @@ class KnownApks:
             os.mkdir('stats')
 
         lst = []
-        for apk, app in self.apks.iteritems():
+        for apk, app in self.apks.items():
             appid, added = app
             line = apk + ' ' + appid
             if added:
@@ -1573,7 +1577,7 @@ class KnownApks:
     # with the most recent first.
     def getlatest(self, num):
         apps = {}
-        for apk, app in self.apks.iteritems():
+        for apk, app in self.apks.items():
             appid, added = app
             if added:
                 if appid in apps:
@@ -1581,7 +1585,7 @@ class KnownApks:
                         apps[appid] = added
                 else:
                     apps[appid] = added
-        sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
+        sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
         lst = [app for app, _ in sortedapps]
         lst.reverse()
         return lst
@@ -1604,8 +1608,9 @@ def isApkDebuggable(apkfile, config):
 
 
 class PopenResult:
-    returncode = None
+    def __init__(self):
-    output = ''
+        self.returncode = None
+        self.output = None
 
 
 def SdkToolsPopen(commands, cwd=None, output=True):
@@ -1620,9 +1625,9 @@ def SdkToolsPopen(commands, cwd=None, output=True):
                        cwd=cwd, output=output)
 
 
-def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
+def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
     """
-    Run a command and capture the possibly huge output.
+    Run a command and capture the possibly huge output as bytes.
 
     :param commands: command and argument list like in subprocess.Popen
     :param cwd: optionally specifies a working directory
@@ -1653,13 +1658,14 @@ def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
         while not stderr_reader.eof():
             while not stderr_queue.empty():
                 line = stderr_queue.get()
-                sys.stderr.write(line)
+                sys.stderr.buffer.write(line)
                 sys.stderr.flush()
 
             time.sleep(0.1)
 
     stdout_queue = Queue()
     stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
+    buf = io.BytesIO()
 
     # Check the queue for output (until there is no more to get)
     while not stdout_reader.eof():
@@ -1667,13 +1673,28 @@ def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
             line = stdout_queue.get()
             if output and options.verbose:
                 # Output directly to console
-                sys.stderr.write(line)
+                sys.stderr.buffer.write(line)
                 sys.stderr.flush()
-            result.output += line
+            buf.write(line)
 
         time.sleep(0.1)
 
     result.returncode = p.wait()
+    result.output = buf.getvalue()
+    buf.close()
+    return result
+
+
+def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
+    """
+    Run a command and capture the possibly huge output as a str.
+
+    :param commands: command and argument list like in subprocess.Popen
+    :param cwd: optionally specifies a working directory
+    :returns: A PopenResult.
+    """
+    result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
+    result.output = result.output.decode('utf-8')
     return result
 
 
@@ -1919,8 +1940,9 @@ def genpassword():
     '''generate a random password for when generating keys'''
     h = hashlib.sha256()
     h.update(os.urandom(16))  # salt
-    h.update(bytes(socket.getfqdn()))
+    h.update(socket.getfqdn().encode('utf-8'))
-    return h.digest().encode('base64').strip()
+    passwd = base64.b64encode(h.digest()).strip()
+    return passwd.decode('utf-8')
 
 
 def genkeystore(localconfig):

fdroidserver/gpgsign.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # gpgsign.py - part of the FDroid server tools
 # Copyright (C) 2014, Ciaran Gultnieks, ciaran@ciarang.com
@@ -23,8 +22,8 @@ import glob
 from argparse import ArgumentParser
 import logging
 
-import common
+from . import common
-from common import FDroidPopen
+from .common import FDroidPopen
 
 config = None
 options = None

fdroidserver/import.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # import.py - part of the FDroid server tools
 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
@@ -21,12 +20,13 @@
 import sys
 import os
 import shutil
-import urllib
+import urllib.request
 from argparse import ArgumentParser
-from ConfigParser import ConfigParser
+from configparser import ConfigParser
 import logging
-import common
+
-import metadata
+from . import common
+from . import metadata
 
 
 # Get the repo type and address from the given web page. The page is scanned
@@ -35,7 +35,7 @@ import metadata
 # Returns repotype, address, or None, reason
 def getrepofrompage(url):
 
-    req = urllib.urlopen(url)
+    req = urllib.request.urlopen(url)
     if req.getcode() != 200:
         return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
     page = req.read()

fdroidserver/init.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # update.py - part of the FDroid server tools
 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
@@ -28,7 +27,7 @@ import sys
 from argparse import ArgumentParser
 import logging
 
-import common
+from . import common
 
 config = {}
 options = None
@@ -103,8 +102,8 @@ def main():
             default_sdk_path = '/opt/android-sdk'
             while not options.no_prompt:
                 try:
-                    s = raw_input('Enter the path to the Android SDK ('
+                    s = input('Enter the path to the Android SDK ('
-                                  + default_sdk_path + ') here:\n> ')
+                              + default_sdk_path + ') here:\n> ')
                 except KeyboardInterrupt:
                     print('')
                     sys.exit(1)

fdroidserver/install.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # install.py - part of the FDroid server tools
 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
@@ -24,8 +23,8 @@ import glob
 from argparse import ArgumentParser
 import logging
 
-import common
+from . import common
-from common import SdkToolsPopen, FDroidException
+from .common import SdkToolsPopen, FDroidException
 
 options = None
 config = None
@@ -82,7 +81,7 @@ def main():
                 continue
             apks[appid] = apkfile
 
-        for appid, apk in apks.iteritems():
+        for appid, apk in apks.items():
             if not apk:
                 raise FDroidException("No signed apk available for %s" % appid)
 
@@ -91,7 +90,7 @@ def main():
         apks = {common.apknameinfo(apkfile)[0]: apkfile for apkfile in
                 sorted(glob.glob(os.path.join(output_dir, '*.apk')))}
 
-    for appid, apk in apks.iteritems():
+    for appid, apk in apks.items():
         # Get device list each time to avoid device not found errors
         devs = devices()
         if not devs:

fdroidserver/lint.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # lint.py - part of the FDroid server tool
 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
@@ -20,11 +19,10 @@
 from argparse import ArgumentParser
 import re
 import sys
-from sets import Set
 
-import common
+from . import common
-import metadata
+from . import metadata
-import rewritemeta
+from . import rewritemeta
 
 config = None
 options = None
@@ -118,7 +116,7 @@ regex_checks = {
 
 
 def check_regexes(app):
-    for f, checks in regex_checks.iteritems():
+    for f, checks in regex_checks.items():
         for m, r in checks:
             v = app.get_field(f)
             t = metadata.fieldtype(f)
@@ -205,7 +203,7 @@ def check_empty_fields(app):
     if not app.Categories:
         yield "Categories are not set"
 
-all_categories = Set([
+all_categories = set([
     "Connectivity",
     "Development",
     "Games",
@@ -333,7 +331,7 @@ def main():
     allapps = metadata.read_metadata(xref=True)
     apps = common.read_app_args(options.appid, allapps, False)
 
-    for appid, app in apps.iteritems():
+    for appid, app in apps.items():
         if app.Disabled:
             continue
 

fdroidserver/metadata.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 #
 # metadata.py - part of the FDroid server tools
 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
@@ -23,11 +23,7 @@ import re
 import glob
 import cgi
 import textwrap
-
+import io
-try:
-    from cStringIO import StringIO
-except:
-    from StringIO import StringIO
 
 import yaml
 # use libyaml if it is available
@@ -41,7 +37,7 @@ except ImportError:
 # use the C implementation when available
 import xml.etree.cElementTree as ElementTree
 
-import common
+import fdroidserver.common
 
 srclibs = None
 
@@ -162,11 +158,11 @@ class App():
     # names. Should only be used for tests.
     def field_dict(self):
         d = {}
-        for k, v in self.__dict__.iteritems():
+        for k, v in self.__dict__.items():
             if k == 'builds':
                 d['builds'] = []
                 for build in v:
-                    b = {k: v for k, v in build.__dict__.iteritems() if not k.startswith('_')}
+                    b = {k: v for k, v in build.__dict__.items() if not k.startswith('_')}
                     d['builds'].append(b)
             elif not k.startswith('_'):
                 f = App.attr_to_field(k)
@@ -200,7 +196,7 @@ class App():
 
     # Like dict.update(), but using human-readable field names
     def update_fields(self, d):
-        for f, v in d.iteritems():
+        for f, v in d.items():
             if f == 'builds':
                 for b in v:
                     build = Build()
@@ -352,13 +348,13 @@ class Build():
         version = self.ndk
         if not version:
             version = 'r10e'  # falls back to latest
-        paths = common.config['ndk_paths']
+        paths = fdroidserver.common.config['ndk_paths']
         if version not in paths:
             return ''
         return paths[version]
 
     def update_flags(self, d):
-        for f, v in d.iteritems():
+        for f, v in d.items():
             self.set_flag(f, v)
 
 flagtypes = {
@@ -513,8 +509,8 @@ class DescriptionFormatter:
         self.laststate = self.stNONE
         self.text_html = ''
         self.text_txt = ''
-        self.html = StringIO()
+        self.html = io.StringIO()
-        self.text = StringIO()
+        self.text = io.StringIO()
         self.para_lines = []
         self.linkResolver = None
         self.linkResolver = linkres
@@ -534,10 +530,10 @@ class DescriptionFormatter:
         self.state = self.stNONE
         whole_para = ' '.join(self.para_lines)
         self.addtext(whole_para)
-        wrapped = textwrap.fill(whole_para.decode('utf-8'), 80,
+        wrapped = textwrap.fill(whole_para, 80,
                                 break_long_words=False,
                                 break_on_hyphens=False)
-        self.text.write(wrapped.encode('utf-8'))
+        self.text.write(wrapped)
         self.html.write('</p>')
         del self.para_lines[:]
 
@@ -709,7 +705,7 @@ def parse_srclib(metadatapath):
     if not os.path.exists(metadatapath):
         return thisinfo
 
-    metafile = open(metadatapath, "r")
+    metafile = open(metadatapath, "r", encoding='utf-8')
 
     n = 0
     for line in metafile:
@@ -797,7 +793,7 @@ def read_metadata(xref=True):
                 return ("fdroid.app:" + appid, "Dummy name - don't know yet")
             raise MetaDataException("Cannot resolve app id " + appid)
 
-        for appid, app in apps.iteritems():
+        for appid, app in apps.items():
             try:
                 description_html(app.Description, linkres)
             except MetaDataException as e:
@@ -826,7 +822,7 @@ def get_default_app_info(metadatapath=None):
     if metadatapath is None:
         appid = None
     else:
-        appid, _ = common.get_extension(os.path.basename(metadatapath))
+        appid, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
 
     app = App()
     app.metadatapath = metadatapath
@@ -846,17 +842,14 @@ esc_newlines = re.compile(r'\\( |\n)')
 # This function uses __dict__ to be faster
 def post_metadata_parse(app):
 
-    for k, v in app.__dict__.iteritems():
+    for k in app._modified:
-        if k not in app._modified:
+        v = app.__dict__[k]
-            continue
         if type(v) in (float, int):
             app.__dict__[k] = str(v)
 
     for build in app.builds:
-        for k, v in build.__dict__.iteritems():
+        for k in build._modified:
-
+            v = build.__dict__[k]
-            if k not in build._modified:
-                continue
             if type(v) in (float, int):
                 build.__dict__[k] = str(v)
                 continue
@@ -866,7 +859,7 @@ def post_metadata_parse(app):
                 build.__dict__[k] = re.sub(esc_newlines, '', v).lstrip().rstrip()
             elif ftype == TYPE_BOOL:
                 # TODO handle this using <xsd:element type="xsd:boolean> in a schema
-                if isinstance(v, basestring):
+                if isinstance(v, str):
                     build.__dict__[k] = _decode_bool(v)
             elif ftype == TYPE_STRING:
                 if isinstance(v, bool) and v:
@@ -904,36 +897,6 @@ def post_metadata_parse(app):
 #
 
 
-def _decode_list(data):
-    '''convert items in a list from unicode to basestring'''
-    rv = []
-    for item in data:
-        if isinstance(item, unicode):
-            item = item.encode('utf-8')
-        elif isinstance(item, list):
-            item = _decode_list(item)
-        elif isinstance(item, dict):
-            item = _decode_dict(item)
-        rv.append(item)
-    return rv
-
-
-def _decode_dict(data):
-    '''convert items in a dict from unicode to basestring'''
-    rv = {}
-    for k, v in data.iteritems():
-        if isinstance(k, unicode):
-            k = k.encode('utf-8')
-        if isinstance(v, unicode):
-            v = v.encode('utf-8')
-        elif isinstance(v, list):
-            v = _decode_list(v)
-        elif isinstance(v, dict):
-            v = _decode_dict(v)
-        rv[k] = v
-    return rv
-
-
 bool_true = re.compile(r'([Yy]es|[Tt]rue)')
 bool_false = re.compile(r'([Nn]o|[Ff]alse)')
 
@@ -947,17 +910,17 @@ def _decode_bool(s):
 
 
 def parse_metadata(metadatapath):
-    _, ext = common.get_extension(metadatapath)
+    _, ext = fdroidserver.common.get_extension(metadatapath)
-    accepted = common.config['accepted_formats']
+    accepted = fdroidserver.common.config['accepted_formats']
     if ext not in accepted:
         raise MetaDataException('"%s" is not an accepted format, convert to: %s' % (
             metadatapath, ', '.join(accepted)))
 
     app = App()
     app.metadatapath = metadatapath
-    app.id, _ = common.get_extension(os.path.basename(metadatapath))
+    app.id, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
 
-    with open(metadatapath, 'r') as mf:
+    with open(metadatapath, 'r', encoding='utf-8') as mf:
         if ext == 'txt':
             parse_txt_metadata(mf, app)
         elif ext == 'json':
@@ -975,11 +938,9 @@ def parse_metadata(metadatapath):
 
 def parse_json_metadata(mf, app):
 
-    # fdroid metadata is only strings and booleans, no floats or ints. And
+    # fdroid metadata is only strings and booleans, no floats or ints.
-    # json returns unicode, and fdroidserver still uses plain python strings
     # TODO create schema using https://pypi.python.org/pypi/jsonschema
-    jsoninfo = json.load(mf, object_hook=_decode_dict,
+    jsoninfo = json.load(mf, parse_int=lambda s: s,
-                         parse_int=lambda s: s,
                          parse_float=lambda s: s)
     app.update_fields(jsoninfo)
     for f in ['Description', 'Maintainer Notes']:
@@ -1253,7 +1214,7 @@ def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
             w_field_always('Binaries')
         mf.write('\n')
 
-    for build in sorted_builds(app.builds):
+    for build in app.builds:
 
         if build.version == "Ignore":
             continue

fdroidserver/net.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 #
 # net.py - part of the FDroid server tools
 # Copyright (C) 2015 Hans-Christoph Steiner <hans@eds.org>

fdroidserver/publish.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # publish.py - part of the FDroid server tools
 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
@@ -21,14 +20,14 @@
 import sys
 import os
 import shutil
-import md5
 import glob
+import hashlib
 from argparse import ArgumentParser
 import logging
 
-import common
+from . import common
-import metadata
+from . import metadata
-from common import FDroidPopen, SdkToolsPopen, BuildException
+from .common import FDroidPopen, SdkToolsPopen, BuildException
 
 config = None
 options = None
@@ -91,8 +90,8 @@ def main():
     vercodes = common.read_pkg_args(options.appid, True)
     allaliases = []
     for appid in allapps:
-        m = md5.new()
+        m = hashlib.md5()
-        m.update(appid)
+        m.update(appid.encode('utf-8'))
         keyalias = m.hexdigest()[:8]
         if keyalias in allaliases:
             logging.error("There is a keyalias collision - publishing halted")
@@ -156,12 +155,12 @@ def main():
                 # For this particular app, the key alias is overridden...
                 keyalias = config['keyaliases'][appid]
                 if keyalias.startswith('@'):
-                    m = md5.new()
+                    m = hashlib.md5()
-                    m.update(keyalias[1:])
+                    m.update(keyalias[1:].encode('utf-8'))
                     keyalias = m.hexdigest()[:8]
             else:
-                m = md5.new()
+                m = hashlib.md5()
-                m.update(appid)
+                m.update(appid.encode('utf-8'))
                 keyalias = m.hexdigest()[:8]
             logging.info("Key alias: " + keyalias)
 

fdroidserver/readmeta.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # readmeta.py - part of the FDroid server tools
 # Copyright (C) 2014 Daniel Martí <mvdan@mvdan.cc>
@@ -18,8 +17,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from argparse import ArgumentParser
-import common
+from . import common
-import metadata
+from . import metadata
 
 
 def main():

fdroidserver/rewritemeta.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # rewritemeta.py - part of the FDroid server tools
 # This cleans up the original .txt metadata file format.
@@ -21,20 +20,17 @@
 from argparse import ArgumentParser
 import os
 import logging
-try:
+import io
-    from cStringIO import StringIO
-except:
-    from StringIO import StringIO
 
-import common
+from . import common
-import metadata
+from . import metadata
 
 config = None
 options = None
 
 
 def proper_format(app):
-    s = StringIO()
+    s = io.StringIO()
     # TODO: currently reading entire file again, should reuse first
     # read in metadata.py
     with open(app.metadatapath, 'r') as f:
@@ -73,7 +69,7 @@ def main():
     if options.to is not None and options.to not in supported:
         parser.error("Must give a valid format to --to")
 
-    for appid, app in apps.iteritems():
+    for appid, app in apps.items():
         base, ext = common.get_extension(app.metadatapath)
         if not options.to and ext not in supported:
             logging.info("Ignoring %s file at '%s'" % (ext, app.metadatapath))
@@ -85,7 +81,7 @@ def main():
 
         if options.list:
             if not proper_format(app):
-                print app.metadatapath
+                print(app.metadatapath)
             continue
 
         with open(base + '.' + to_ext, 'w') as f:

fdroidserver/scanner.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # scanner.py - part of the FDroid server tools
 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
@@ -23,9 +22,9 @@ import traceback
 from argparse import ArgumentParser
 import logging
 
-import common
+from . import common
-import metadata
+from . import metadata
-from common import BuildException, VCSException
+from .common import BuildException, VCSException
 
 config = None
 options = None
@@ -68,7 +67,7 @@ def scan_source(build_dir, root_dir, build):
     }
 
     def suspects_found(s):
-        for n, r in usual_suspects.iteritems():
+        for n, r in usual_suspects.items():
             if r.match(s):
                 yield n
 
@@ -95,7 +94,7 @@ def scan_source(build_dir, root_dir, build):
     scandelete_worked = set()
 
     def toignore(fd):
-        for k, paths in scanignore.iteritems():
+        for k, paths in scanignore.items():
             for p in paths:
                 if fd.startswith(p):
                     scanignore_worked.add(k)
@@ -103,7 +102,7 @@ def scan_source(build_dir, root_dir, build):
         return False
 
     def todelete(fd):
-        for k, paths in scandelete.iteritems():
+        for k, paths in scandelete.items():
             for p in paths:
                 if fd.startswith(p):
                     scandelete_worked.add(k)
@@ -200,10 +199,11 @@ def scan_source(build_dir, root_dir, build):
             elif ext == 'java':
                 if not os.path.isfile(fp):
                     continue
-                for line in file(fp):
+                with open(fp, 'r') as f:
-                    if 'DexClassLoader' in line:
+                    for line in f:
-                        count += handleproblem('DexClassLoader', fd, fp)
+                        if 'DexClassLoader' in line:
-                        break
+                            count += handleproblem('DexClassLoader', fd, fp)
+                            break
 
             elif ext == 'gradle':
                 if not os.path.isfile(fp):
@@ -267,7 +267,7 @@ def main():
     srclib_dir = os.path.join(build_dir, 'srclib')
     extlib_dir = os.path.join(build_dir, 'extlib')
 
-    for appid, app in apps.iteritems():
+    for appid, app in apps.items():
 
         if app.Disabled:
             logging.info("Skipping %s: disabled" % appid)

fdroidserver/server.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # server.py - part of the FDroid server tools
 # Copyright (C) 2010-15, Ciaran Gultnieks, ciaran@ciarang.com
@@ -26,7 +25,8 @@ import pwd
 import subprocess
 from argparse import ArgumentParser
 import logging
-import common
+
+from . import common
 
 config = None
 options = None
@@ -296,12 +296,12 @@ def main():
             sftp = ssh.open_sftp()
             if os.path.basename(remotepath) \
                     not in sftp.listdir(os.path.dirname(remotepath)):
-                sftp.mkdir(remotepath, mode=0755)
+                sftp.mkdir(remotepath, mode=0o755)
             for repo_section in repo_sections:
                 repo_path = os.path.join(remotepath, repo_section)
                 if os.path.basename(repo_path) \
                         not in sftp.listdir(remotepath):
-                    sftp.mkdir(repo_path, mode=0755)
+                    sftp.mkdir(repo_path, mode=0o755)
             sftp.close()
             ssh.close()
     elif options.command == 'update':

fdroidserver/signindex.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # gpgsign.py - part of the FDroid server tools
 # Copyright (C) 2015, Ciaran Gultnieks, ciaran@ciarang.com
@@ -22,8 +21,8 @@ import os
 from argparse import ArgumentParser
 import logging
 
-import common
+from . import common
-from common import FDroidPopen
+from .common import FDroidPopen
 
 config = None
 options = None

fdroidserver/stats.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # stats.py - part of the FDroid server tools
 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
@@ -28,11 +27,12 @@ from argparse import ArgumentParser
 import paramiko
 import socket
 import logging
-import common
-import metadata
 import subprocess
 from collections import Counter
 
+from . import common
+from . import metadata
+
 
 def carbon_send(key, value):
     s = socket.socket()
@@ -75,7 +75,7 @@ def main():
         sys.exit(1)
 
     # Get all metadata-defined apps...
-    allmetaapps = [app for app in metadata.read_metadata().itervalues()]
+    allmetaapps = [app for app in metadata.read_metadata().values()]
     metaapps = [app for app in allmetaapps if not app.Disabled]
 
     statsdir = 'stats'

fdroidserver/update.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # update.py - part of the FDroid server tools
 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
@@ -27,7 +26,7 @@ import socket
 import zipfile
 import hashlib
 import pickle
-import urlparse
+import urllib.parse
 from datetime import datetime, timedelta
 from xml.dom.minidom import Document
 from argparse import ArgumentParser
@@ -35,16 +34,15 @@ import time
 from pyasn1.error import PyAsn1Error
 from pyasn1.codec.der import decoder, encoder
 from pyasn1_modules import rfc2315
-from hashlib import md5
 from binascii import hexlify, unhexlify
 
 from PIL import Image
 import logging
 
-import common
+from . import common
-import metadata
+from . import metadata
-from common import FDroidPopen, SdkToolsPopen
+from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
-from metadata import MetaDataException
+from .metadata import MetaDataException
 
 screen_densities = ['640', '480', '320', '240', '160', '120']
 
@@ -292,7 +290,7 @@ def delete_disabled_builds(apps, apkcache, repodirs):
     :param apkcache: current apk cache information
     :param repodirs: the repo directories to process
     """
-    for appid, app in apps.iteritems():
+    for appid, app in apps.items():
         for build in app.builds:
             if not build.disable:
                 continue
@@ -402,7 +400,7 @@ def getsig(apkpath):
 
     cert_encoded = encoder.encode(certificates)[4:]
 
-    return md5(cert_encoded.encode('hex')).hexdigest()
+    return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 
 
 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
@@ -713,7 +711,7 @@ repo_pubkey_fingerprint = None
 def cert_fingerprint(data):
     digest = hashlib.sha256(data).digest()
     ret = []
-    ret.append(' '.join("%02X" % ord(b) for b in digest))
+    ret.append(' '.join("%02X" % b for b in bytearray(digest)))
     return " ".join(ret)
 
 
@@ -722,12 +720,12 @@ def extract_pubkey():
     if 'repo_pubkey' in config:
         pubkey = unhexlify(config['repo_pubkey'])
     else:
-        p = FDroidPopen([config['keytool'], '-exportcert',
+        p = FDroidPopenBytes([config['keytool'], '-exportcert',
-                         '-alias', config['repo_keyalias'],
+                              '-alias', config['repo_keyalias'],
-                         '-keystore', config['keystore'],
+                              '-keystore', config['keystore'],
-                         '-storepass:file', config['keystorepassfile']]
+                              '-storepass:file', config['keystorepassfile']]
-                        + config['smartcardoptions'],
+                             + config['smartcardoptions'],
-                        output=False, stderr_to_stdout=False)
+                             output=False, stderr_to_stdout=False)
         if p.returncode != 0 or len(p.output) < 20:
             msg = "Failed to get repo pubkey!"
             if config['keystore'] == 'NONE':
@@ -774,7 +772,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
 
     mirrorcheckfailed = False
     for mirror in config.get('mirrors', []):
-        base = os.path.basename(urlparse.urlparse(mirror).path.rstrip('/'))
+        base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
             mirrorcheckfailed = True
@@ -788,9 +786,9 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
         repoel.setAttribute("url", config['archive_url'])
         addElement('description', config['archive_description'], doc, repoel)
-        urlbasepath = os.path.basename(urlparse.urlparse(config['archive_url']).path)
+        urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
         for mirror in config.get('mirrors', []):
-            addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
+            addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 
     else:
         repoel.setAttribute("name", config['repo_name'])
@@ -799,9 +797,9 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
         repoel.setAttribute("url", config['repo_url'])
         addElement('description', config['repo_description'], doc, repoel)
-        urlbasepath = os.path.basename(urlparse.urlparse(config['repo_url']).path)
+        urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
         for mirror in config.get('mirrors', []):
-            addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
+            addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 
     repoel.setAttribute("version", "15")
     repoel.setAttribute("timestamp", str(int(time.time())))
@@ -828,7 +826,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
             logging.warning("\tfdroid update --create-key")
             sys.exit(1)
 
-    repoel.setAttribute("pubkey", extract_pubkey())
+    repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
     root.appendChild(repoel)
 
     for appid in sortedids:
@@ -968,9 +966,9 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
                     os.symlink(sigfile_path, siglinkname)
 
     if options.pretty:
-        output = doc.toprettyxml()
+        output = doc.toprettyxml(encoding='utf-8')
     else:
-        output = doc.toxml()
+        output = doc.toxml(encoding='utf-8')
 
     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
         f.write(output)
@@ -1025,7 +1023,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
 
 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 
-    for appid, app in apps.iteritems():
+    for appid, app in apps.items():
 
         if app.ArchivePolicy:
             keepversions = int(app.ArchivePolicy[:-9])
@@ -1199,7 +1197,7 @@ def main():
 
     # Generate a list of categories...
     categories = set()
-    for app in apps.itervalues():
+    for app in apps.values():
         categories.update(app.Categories)
 
     # Read known apks data (will be updated and written back when we've finished)
@@ -1269,7 +1267,7 @@ def main():
     # level. When doing this, we use the info from the most recent version's apk.
     # We deal with figuring out when the app was added and last updated at the
     # same time.
-    for appid, app in apps.iteritems():
+    for appid, app in apps.items():
         bestver = 0
         for apk in apks + archapks:
             if apk['id'] == appid:
@@ -1303,13 +1301,13 @@ def main():
     # Sort the app list by name, then the web site doesn't have to by default.
     # (we had to wait until we'd scanned the apks to do this, because mostly the
     # name comes from there!)
-    sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid].Name.upper())
+    sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
 
     # APKs are placed into multiple repos based on the app package, providing
     # per-app subscription feeds for nightly builds and things like it
     if config['per_app_repos']:
         add_apks_to_per_app_repos(repodirs[0], apks)
-        for appid, app in apps.iteritems():
+        for appid, app in apps.items():
             repodir = os.path.join(appid, 'fdroid', 'repo')
             appdict = dict()
             appdict[appid] = app
@@ -1338,14 +1336,15 @@ def main():
         # Generate latest apps data for widget
         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
             data = ''
-            for line in file(os.path.join('stats', 'latestapps.txt')):
+            with open(os.path.join('stats', 'latestapps.txt'), 'r') as f:
-                appid = line.rstrip()
+                for line in f:
-                data += appid + "\t"
+                    appid = line.rstrip()
-                app = apps[appid]
+                    data += appid + "\t"
-                data += app.Name + "\t"
+                    app = apps[appid]
-                if app.icon is not None:
+                    data += app.Name + "\t"
-                    data += app.icon + "\t"
+                    if app.icon is not None:
-                data += app.License + "\n"
+                        data += app.icon + "\t"
+                    data += app.License + "\n"
             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
                 f.write(data)
 

fdroidserver/verify.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # verify.py - part of the FDroid server tools
 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
@@ -23,9 +22,9 @@ import glob
 from argparse import ArgumentParser
 import logging
 
-import common
+from . import common
-import net
+from . import net
-from common import FDroidException
+from .common import FDroidException
 
 options = None
 config = None

hooks/pre-commit

@@ -46,17 +46,16 @@ else
     done
 fi
 
-# In the default configuration, the checks E123, E133, E226, E241 and E242 are
+# We ignore the following PEP8 warnings
-# ignored because they are not rules unanimously accepted
+# * E123: closing bracket does not match indentation of opening bracket's line
-# On top of those, we ignore:
+#   - Broken if multiple indentation levels start on a single line
 # * E501: line too long (82 > 79 characters)
 #   - Recommended for readability but not enforced
 #   - Some lines are awkward to wrap around a char limit
 # * W503: line break before binary operator
-#   - It's quite new
 #   - Quite pedantic
 
-PEP8_IGNORE="E123,E133,E226,E241,E242,E501,W503"
+PEP8_IGNORE="E123,E501,W503"
 
 err() {
 	echo ERROR: "$@"
@@ -71,22 +70,21 @@ cmd_exists() {
 	command -v $1 1>/dev/null
 }
 
-if cmd_exists pyflakes-python2; then
+find_command() {
-	PYFLAKES=pyflakes-python2
+	local name=$1
-elif cmd_exists pyflakes; then
+	for suff in "3" "-python3" ""; do
-	PYFLAKES=pyflakes
+		cmd=${1}${suff}
-else
+		if cmd_exists $cmd; then
-	PYFLAKES=echo
+			echo -n $cmd
-	warn "pyflakes is not installed, using dummy placeholder!"
+			return 0
-fi
+		fi
+	done
+	warn "$1 is not installed, using dummy placeholder!"
+	echo -n echo
+}
 
-if cmd_exists pep8-python2; then
+PYFLAKES=$(find_command pyflakes)
-	PEP8=pep8-python2
+PEP8=$(find_command pep8)
-elif cmd_exists pep8; then
-	PEP8=pep8
-else
-	err "pep8 is not installed!"
-fi
 
 if [ "$PY_FILES $PY_TEST_FILES" != " " ]; then
     if ! $PYFLAKES $PY_FILES $PY_TEST_FILES; then

makebuildserver

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 import os
 import sys
@@ -26,7 +26,7 @@ def vagrant(params, cwd=None, printout=False):
             line = p.stdout.readline()
             if len(line) == 0:
                 break
-            print line,
+            print(line)
             out += line
         p.wait()
     else:
@@ -63,13 +63,13 @@ config = {
 
 # load config file, if present
 if os.path.exists('makebuildserver.config.py'):
-    execfile('makebuildserver.config.py', config)
+    exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
 elif os.path.exists('makebs.config.py'):
     # this is the old name for the config file
-    execfile('makebs.config.py', config)
+    exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
 
 if not os.path.exists('makebuildserver') or not os.path.exists(serverdir):
-    print 'This must be run from the correct directory!'
+    print('This must be run from the correct directory!')
     sys.exit(1)
 
 if os.path.exists(boxfile):
@@ -81,7 +81,7 @@ if options.clean:
 # Update cached files.
 cachedir = config['cachedir']
 if not os.path.exists(cachedir):
-    os.makedirs(cachedir, 0755)
+    os.makedirs(cachedir, 0o755)
 
 cachefiles = [
     ('android-sdk_r24.4.1-linux.tgz',
@@ -318,17 +318,17 @@ for f, src, shasum in cachefiles:
     if os.path.exists(relpath) and os.stat(relpath).st_size == 0:
         os.remove(relpath)
     if not os.path.exists(relpath):
-        print "Downloading " + f + " to cache"
+        print("Downloading " + f + " to cache")
         if subprocess.call(['wget', src, '-O', f], cwd=cachedir) != 0:
-            print "...download of " + f + " failed."
+            print("...download of " + f + " failed.")
             sys.exit(1)
     if shasum:
         v = sha256_for_file(relpath)
         if v != shasum:
-            print "Invalid shasum of '" + v + "' detected for " + f
+            print("Invalid shasum of '" + v + "' detected for " + f)
             sys.exit(1)
         else:
-            print "...shasum verified for " + f
+            print("...shasum verified for " + f)
 
     wanted.append(f)
 
@@ -418,57 +418,57 @@ if os.path.exists(vf):
     with open(vf, 'r') as f:
         oldvf = f.read()
     if oldvf != vagrantfile:
-        print "Server configuration has changed, rebuild from scratch is required"
+        print("Server configuration has changed, rebuild from scratch is required")
         vagrant(['destroy', '-f'], serverdir)
     else:
-        print "Re-provisioning existing server"
+        print("Re-provisioning existing server")
         writevf = False
 else:
-    print "No existing server - building from scratch"
+    print("No existing server - building from scratch")
 if writevf:
     with open(vf, 'w') as f:
         f.write(vagrantfile)
 
 
-print "Configuring build server VM"
+print("Configuring build server VM")
 returncode, out = vagrant(['up', '--provision'], serverdir, printout=True)
 with open(os.path.join(serverdir, 'up.log'), 'w') as log:
     log.write(out)
 if returncode != 0:
-    print "Failed to configure server"
+    print("Failed to configure server")
     sys.exit(1)
 
-print "Writing buildserver ID"
+print("Writing buildserver ID")
 p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE)
 buildserverid = p.communicate()[0].strip()
-print "...ID is " + buildserverid
+print("...ID is " + buildserverid)
 subprocess.call(
     ['vagrant', 'ssh', '-c', 'sh -c "echo {0} >/home/vagrant/buildserverid"'
         .format(buildserverid)],
     cwd=serverdir)
 
-print "Stopping build server VM"
+print("Stopping build server VM")
 vagrant(['halt'], serverdir)
 
-print "Waiting for build server VM to be finished"
+print("Waiting for build server VM to be finished")
 ready = False
 while not ready:
     time.sleep(2)
     returncode, out = vagrant(['status'], serverdir)
     if returncode != 0:
-        print "Error while checking status"
+        print("Error while checking status")
         sys.exit(1)
     for line in out.splitlines():
         if line.startswith("default"):
             if line.find("poweroff") != -1:
                 ready = True
             else:
-                print "Status: " + line
+                print("Status: " + line)
 
-print "Packaging"
+print("Packaging")
 vagrant(['package', '--output', os.path.join('..', boxfile)], serverdir,
         printout=options.verbose)
-print "Adding box"
+print("Adding box")
 vagrant(['box', 'add', 'buildserver', boxfile, '-f'],
         printout=options.verbose)
 

setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 from setuptools import setup
 import sys

tests/build.TestCase

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
 
@@ -37,7 +36,7 @@ class BuildTest(unittest.TestCase):
                     break
             return True
         else:
-            print 'no build-tools found: ' + build_tools
+            print('no build-tools found: ' + build_tools)
             return False
 
     def _find_all(self):

tests/common.TestCase

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
 
@@ -37,7 +36,7 @@ class CommonTest(unittest.TestCase):
                     break
             return True
         else:
-            print 'no build-tools found: ' + build_tools
+            print('no build-tools found: ' + build_tools)
             return False
 
     def _find_all(self):
@@ -61,7 +60,7 @@ class CommonTest(unittest.TestCase):
             if self._set_build_tools():
                 self._find_all()
             else:
-                print 'no build-tools found: ' + build_tools
+                print('no build-tools found: ' + build_tools)
 
     def testIsApkDebuggable(self):
         config = dict()

tests/complete-ci-tests

@@ -51,10 +51,10 @@ cd $WORKSPACE/tests
 #------------------------------------------------------------------------------#
 # test building the source tarball, then installing it
 cd $WORKSPACE
-python2 setup.py sdist
+python3 setup.py sdist
 
 rm -rf $WORKSPACE/env
-virtualenv --python=python2 $WORKSPACE/env
+virtualenv --python=python3 $WORKSPACE/env
 . $WORKSPACE/env/bin/activate
 pip install dist/fdroidserver-*.tar.gz
 
@@ -66,10 +66,10 @@ fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource
 # test install using install direct from git repo
 cd $WORKSPACE
 rm -rf $WORKSPACE/env
-virtualenv --python=python2 --system-site-packages $WORKSPACE/env
+virtualenv --python=python3 --system-site-packages $WORKSPACE/env
 . $WORKSPACE/env/bin/activate
 pip install -e $WORKSPACE
-python2 setup.py install
+python3 setup.py install
 
 # run tests in new pip+virtualenv install
 fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource
@@ -86,7 +86,7 @@ sh hooks/pre-commit
 cd $WORKSPACE
 set +e
 # use the virtualenv python so pylint checks against its installed libs
-    PYTHONPATH=$WORKSPACE/.pylint-plugins python2 /usr/bin/pylint \
+    PYTHONPATH=$WORKSPACE/.pylint-plugins python3 /usr/bin/pylint \
         --output-format=parseable --reports=n \
         --load-plugins astng_hashlib \
         fdroidserver/*.py fdroid makebuildserver setup.py > $WORKSPACE/pylint.parseable

tests/description-parsing.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import os
 import sys

tests/import.TestCase

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
 

tests/install.TestCase

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
 
@@ -32,7 +31,7 @@ class InstallTest(unittest.TestCase):
         devices = fdroidserver.install.devices()
         self.assertIsInstance(devices, list, 'install.devices() did not return a list!')
         for device in devices:
-            self.assertIsInstance(device, basestring)
+            self.assertIsInstance(device, str)
 
 
 if __name__ == "__main__":

tests/metadata.TestCase

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
 
@@ -43,7 +42,7 @@ class MetadataTest(unittest.TestCase):
             savepath = os.path.join('metadata', appid + '.pickle')
             frommeta = app.field_dict()
             self.assertTrue(appid in apps)
-            with open(savepath, 'r') as f:
+            with open(savepath, 'rb') as f:
                 frompickle = pickle.load(f)
             self.assertEquals(frommeta, frompickle)
             # Uncomment to overwrite

tests/run-tests

@@ -83,7 +83,7 @@ fi
 
 # allow the location of python to be overridden
 if [ -z $python ]; then
-    python=python2
+    python=python3
 fi
 
 set -x # show each command as it is executed

tests/update.TestCase

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
 
@@ -9,6 +8,7 @@ import optparse
 import os
 import sys
 import unittest
+from binascii import unhexlify
 
 localmodule = os.path.realpath(
     os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
@@ -56,10 +56,10 @@ class UpdateTest(unittest.TestCase):
         self.assertEquals(len(sig), len(pysig),
                           "the length of the two sigs are different!")
         try:
-            self.assertEquals(sig.decode('hex'), pysig.decode('hex'),
+            self.assertEquals(unhexlify(sig), unhexlify(pysig),
                               "the length of the two sigs are different!")
         except TypeError as e:
-            print e
+            print(e)
             self.assertTrue(False, 'TypeError!')
 
     def testBadGetsig(self):