From 444e8ad982a7431e7529c856a7431f9d97e3afc4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 7 Jun 2016 13:26:40 +0200 Subject: [PATCH 1/4] read/write Java .properties files in proper encoding They are officially defined as always in ISO 8859-1: https://docs.oracle.com/javase/6/docs/api/java/util/Properties.html --- fdroidserver/common.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 86937640..b1d955d9 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1017,7 +1017,7 @@ def get_library_references(root_dir): proppath = os.path.join(root_dir, 'project.properties') if not os.path.isfile(proppath): return libraries - with open(proppath, 'r') as f: + with open(proppath, 'r', encoding='iso-8859-1') as f: for line in f: if not line.startswith('android.library.reference.'): continue @@ -1356,7 +1356,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= props = "" if os.path.isfile(path): logging.info("Updating local.properties file at %s" % path) - with open(path, 'r') as f: + with open(path, 'r', encoding='iso-8859-1') as f: props += f.read() props += '\n' else: @@ -1378,7 +1378,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= # Add java.encoding if necessary if build.encoding: props += "java.encoding=%s\n" % build.encoding - with open(path, 'w') as f: + with open(path, 'w', encoding='iso-8859-1') as f: f.write(props) flavours = [] @@ -1771,12 +1771,12 @@ def remove_signing_keys(build_dir): if propfile in files: path = os.path.join(root, propfile) - with open(path, "r") as o: + with open(path, "r", encoding='iso-8859-1') as o: lines = o.readlines() changed = False - with open(path, "w") as o: + with open(path, "w", encoding='iso-8859-1') as o: for line in lines: if any(line.startswith(s) for s in ('key.store', 'key.alias')): changed = True @@ -1838,10 +1838,10 @@ def place_srclib(root_dir, number, libpath): lines = [] if os.path.isfile(proppath): - with open(proppath, "r") as o: + with open(proppath, "r", encoding='iso-8859-1') as o: lines = o.readlines() - with open(proppath, "w") as o: + with open(proppath, "w", encoding='iso-8859-1') as o: placed = False for line in lines: if line.startswith('android.library.reference.%d=' % number): From afd528731a48a38c969a6aef443a045949431073 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 7 Jun 2016 13:35:13 +0200 Subject: [PATCH 2/4] read/write F-Droid files always as UTF-8 This makes UTF-8 the sole supported encoding for F-Droid's files. This is mostly codifying the already existing practice for config.py and index.xml. The other files where always just ASCII before. * config.py * metadata/*.txt * known_apks.txt * categories.txt * latestapps.txt * latestapps.dat * index.xml Note: this does not change the read/write encoding of stats files. That is still ASCII. --- fdroidserver/common.py | 8 ++++---- fdroidserver/init.py | 4 ++-- fdroidserver/metadata.py | 2 +- fdroidserver/rewritemeta.py | 2 +- fdroidserver/update.py | 8 ++++---- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index b1d955d9..0a3c8e75 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1539,7 +1539,7 @@ class KnownApks: self.path = os.path.join('stats', 'known_apks.txt') self.apks = {} if os.path.isfile(self.path): - with open(self.path, 'r') as f: + with open(self.path, 'r', encoding='utf8') as f: for line in f: t = line.rstrip().split(' ') if len(t) == 2: @@ -1563,7 +1563,7 @@ class KnownApks: line += ' ' + time.strftime('%Y-%m-%d', added) lst.append(line) - with open(self.path, 'w') as f: + with open(self.path, 'w', encoding='utf8') as f: for line in sorted(lst, key=natural_key): f.write(line + '\n') @@ -2009,7 +2009,7 @@ def write_to_config(thisconfig, key, value=None): if value is None: origkey = key + '_orig' value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key] - with open('config.py', 'r') as f: + with open('config.py', 'r', encoding='utf8') as f: data = f.read() pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"' repl = '\n' + key + ' = "' + value + '"' @@ -2020,7 +2020,7 @@ def write_to_config(thisconfig, key, value=None): # make sure the file ends with a carraige return if not re.match('\n$', data): data += '\n' - with open('config.py', 'w') as f: + with open('config.py', 'w', encoding='utf8') as f: f.writelines(data) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 4b301de9..caa77734 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -35,12 +35,12 @@ options = None def disable_in_config(key, value): '''write a key/value to the local config.py, then comment it out''' - with open('config.py', 'r') as f: + with open('config.py', 'r', encoding='utf8') as f: data = f.read() pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"' repl = '\n#' + key + ' = "' + value + '"' data = re.sub(pattern, repl, data) - with open('config.py', 'w') as f: + with open('config.py', 'w', encoding='utf8') as f: f.writelines(data) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 71da8453..a9be8ef0 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -1384,7 +1384,7 @@ def write_metadata(metadatapath, app): raise MetaDataException('Cannot write "%s", not an accepted format, use: %s' % ( metadatapath, ', '.join(accepted))) - with open(metadatapath, 'w') as mf: + with open(metadatapath, 'w', encoding='utf8') as mf: if ext == 'txt': return write_txt(mf, app) elif ext == 'yml': diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py index 052deb4e..65a50841 100644 --- a/fdroidserver/rewritemeta.py +++ b/fdroidserver/rewritemeta.py @@ -33,7 +33,7 @@ def proper_format(app): s = io.StringIO() # TODO: currently reading entire file again, should reuse first # read in metadata.py - with open(app.metadatapath, 'r') as f: + with open(app.metadatapath, 'r', encoding='utf8') as f: cur_content = f.read() metadata.write_txt(s, app) content = s.getvalue() diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 80d8d8ab..ba7a3998 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1017,7 +1017,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): catdata = '' for cat in categories: catdata += cat + '\n' - with open(os.path.join(repodir, 'categories.txt'), 'w') as f: + with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f: f.write(catdata) @@ -1229,7 +1229,7 @@ def main(): if 'name' not in apk: logging.error(apk['id'] + ' does not have a name! Skipping...') continue - f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w') + f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8') f.write("License:Unknown\n") f.write("Web Site:\n") f.write("Source Code:\n") @@ -1339,7 +1339,7 @@ def main(): # Generate latest apps data for widget if os.path.exists(os.path.join('stats', 'latestapps.txt')): data = '' - with open(os.path.join('stats', 'latestapps.txt'), 'r') as f: + with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f: for line in f: appid = line.rstrip() data += appid + "\t" @@ -1348,7 +1348,7 @@ def main(): if app.icon is not None: data += app.icon + "\t" data += app.License + "\n" - with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f: + with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f: f.write(data) if cachechanged: From 2b6d692f063b34338931e5a79d62fa3c30edc77e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 7 Jun 2016 20:13:54 +0200 Subject: [PATCH 3/4] use UTF8 as default instead of ASCII for .java .gradle pom.xml .java .gradle and XML files all can use any encoding. Most code is ASCII, but authors' names, etc. can easily be non-ASCII. UTF-8 is by far the most common file encoding. While UTF-8 is the default encoding inside the code in Python 3, it still has to deal with the real world, so the encoding needs to be explicitly set when reading and writing files. So this switches fdroidserver to expect UTF-8 instead of ASCII when parsing these files. For now, this commit means that we only support UTF-8 encoded *.java, pom.xml or *.gradle files. Ideally, the code would detect the encoding and use the actual one, but that's a lot more work, and its something that will not happen often. We can cross that bridge when we come to it. One approach, which is taken in the commit when possible, is to keep the data as `bytes`, in which case the encoding doesn't matter. This also fixes this crash when parsing gradle and maven files with non-ASCII chars: ERROR: test_adapt_gradle (__main__.BuildTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/var/lib/jenkins/workspace/fdroidserver-eighthave/tests/build.TestCase", line 59, in test_adapt_gradle fdroidserver.build.adapt_gradle(testsdir) File "/var/lib/jenkins/workspace/fdroidserver-eighthave/fdroidserver/build.py", line 445, in adapt_gradle path) File "/var/lib/jenkins/workspace/fdroidserver-eighthave/fdroidserver/common.py", line 188, in regsub_file text = f.read() File "/usr/lib/python3.4/encodings/ascii.py", line 26, in decode return codecs.ascii_decode(input, self.errors)[0] UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 9460: ordinal not in range(128) --- fdroidserver/common.py | 10 +++++----- fdroidserver/metadata.py | 2 +- fdroidserver/scanner.py | 4 ++-- tests/build.TestCase | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 0a3c8e75..03a8743d 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -184,10 +184,10 @@ def fill_config_defaults(thisconfig): def regsub_file(pattern, repl, path): - with open(path, 'r') as f: + with open(path, 'rb') as f: text = f.read() - text = re.sub(pattern, repl, text) - with open(path, 'w') as f: + text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text) + with open(path, 'wb') as f: f.write(text) @@ -1724,14 +1724,14 @@ def remove_signing_keys(build_dir): if 'build.gradle' in files: path = os.path.join(root, 'build.gradle') - with open(path, "r") as o: + with open(path, "r", encoding='utf8') as o: lines = o.readlines() changed = False opened = 0 i = 0 - with open(path, "w") as o: + with open(path, "w", encoding='utf8') as o: while i < len(lines): line = lines[i] i += 1 diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index a9be8ef0..ded6cd1a 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -836,7 +836,7 @@ def get_default_app_info(metadatapath=None): for root, dirs, files in os.walk(os.getcwd()): if 'build.gradle' in files: p = os.path.join(root, 'build.gradle') - with open(p) as f: + with open(p, 'rb') as f: data = f.read() m = pattern.search(data) if m: diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 53c938fb..41ecbb50 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -199,7 +199,7 @@ def scan_source(build_dir, root_dir, build): elif ext == 'java': if not os.path.isfile(fp): continue - with open(fp, 'r') as f: + with open(fp, 'r', encoding='utf8') as f: for line in f: if 'DexClassLoader' in line: count += handleproblem('DexClassLoader', fd, fp) @@ -208,7 +208,7 @@ def scan_source(build_dir, root_dir, build): elif ext == 'gradle': if not os.path.isfile(fp): continue - with open(fp, 'r') as f: + with open(fp, 'r', encoding='utf8') as f: lines = f.readlines() for i, line in enumerate(lines): if is_used_by_gradle(line): diff --git a/tests/build.TestCase b/tests/build.TestCase index b3b90fc3..369c3836 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -57,13 +57,13 @@ class BuildTest(unittest.TestCase): fdroidserver.build.config = {} fdroidserver.build.config['build_tools'] = teststring fdroidserver.build.adapt_gradle(testsdir) - pattern = re.compile("buildToolsVersion[\s=]+'%s'\s+" % teststring) + pattern = re.compile(bytes("buildToolsVersion[\s=]+'%s'\s+" % teststring, 'utf8')) for p in ('source-files/fdroid/fdroidclient/build.gradle', 'source-files/Zillode/syncthing-silk/build.gradle', 'source-files/open-keychain/open-keychain/build.gradle', 'source-files/osmandapp/osmand/build.gradle', 'source-files/open-keychain/open-keychain/OpenKeychain/build.gradle'): - with open(os.path.join(testsdir, p), 'r') as f: + with open(os.path.join(testsdir, p), 'rb') as f: filedata = f.read() self.assertIsNotNone(pattern.search(filedata)) From 733ef52424cdf565c09f5d91f397baa8063958f2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 9 Jun 2016 12:15:11 +0200 Subject: [PATCH 4/4] gitlab-ci: make sure pip3 install dirs exist It doesn't want to create them itself. --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 467f1cb1..1128b640 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,8 @@ image: mvdan/fdroid-ci:server-20160429 test: script: + - ls -l /usr/lib/python3* + - mkdir -p /usr/lib/python3.4/site-packages/ - pip3 install -e . - cd tests - ./complete-ci-tests