read/write F-Droid files always as UTF-8

This makes UTF-8 the sole supported encoding for F-Droid's files. This is
mostly codifying the already existing practice for config.py and index.xml.
The other files where always just ASCII before.

* config.py
* metadata/*.txt
* known_apks.txt
* categories.txt
* latestapps.txt
* latestapps.dat
* index.xml

Note: this does not change the read/write encoding of stats files.  That is
still ASCII.

fdroidserver/common.py

@@ -1539,7 +1539,7 @@ class KnownApks:
         self.path = os.path.join('stats', 'known_apks.txt')
         self.apks = {}
         if os.path.isfile(self.path):
-            with open(self.path, 'r') as f:
+            with open(self.path, 'r', encoding='utf8') as f:
                 for line in f:
                     t = line.rstrip().split(' ')
                     if len(t) == 2:
@@ -1563,7 +1563,7 @@ class KnownApks:
                 line += ' ' + time.strftime('%Y-%m-%d', added)
             lst.append(line)
 
-        with open(self.path, 'w') as f:
+        with open(self.path, 'w', encoding='utf8') as f:
             for line in sorted(lst, key=natural_key):
                 f.write(line + '\n')
 
@@ -2009,7 +2009,7 @@ def write_to_config(thisconfig, key, value=None):
     if value is None:
         origkey = key + '_orig'
         value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
-    with open('config.py', 'r') as f:
+    with open('config.py', 'r', encoding='utf8') as f:
         data = f.read()
     pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
     repl = '\n' + key + ' = "' + value + '"'
@@ -2020,7 +2020,7 @@ def write_to_config(thisconfig, key, value=None):
     # make sure the file ends with a carraige return
     if not re.match('\n$', data):
         data += '\n'
-    with open('config.py', 'w') as f:
+    with open('config.py', 'w', encoding='utf8') as f:
         f.writelines(data)
 
 

fdroidserver/init.py

@@ -35,12 +35,12 @@ options = None
 
 def disable_in_config(key, value):
     '''write a key/value to the local config.py, then comment it out'''
-    with open('config.py', 'r') as f:
+    with open('config.py', 'r', encoding='utf8') as f:
         data = f.read()
     pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
     repl = '\n#' + key + ' = "' + value + '"'
     data = re.sub(pattern, repl, data)
-    with open('config.py', 'w') as f:
+    with open('config.py', 'w', encoding='utf8') as f:
         f.writelines(data)
 
 

fdroidserver/metadata.py

@@ -1384,7 +1384,7 @@ def write_metadata(metadatapath, app):
         raise MetaDataException('Cannot write "%s", not an accepted format, use: %s' % (
             metadatapath, ', '.join(accepted)))
 
-    with open(metadatapath, 'w') as mf:
+    with open(metadatapath, 'w', encoding='utf8') as mf:
         if ext == 'txt':
             return write_txt(mf, app)
         elif ext == 'yml':

fdroidserver/rewritemeta.py

@@ -33,7 +33,7 @@ def proper_format(app):
     s = io.StringIO()
     # TODO: currently reading entire file again, should reuse first
     # read in metadata.py
-    with open(app.metadatapath, 'r') as f:
+    with open(app.metadatapath, 'r', encoding='utf8') as f:
         cur_content = f.read()
     metadata.write_txt(s, app)
     content = s.getvalue()

fdroidserver/update.py

@@ -1017,7 +1017,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
     catdata = ''
     for cat in categories:
         catdata += cat + '\n'
-    with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
+    with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
         f.write(catdata)
 
 
@@ -1229,7 +1229,7 @@ def main():
                 if 'name' not in apk:
                     logging.error(apk['id'] + ' does not have a name! Skipping...')
                     continue
-                f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
+                f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
                 f.write("License:Unknown\n")
                 f.write("Web Site:\n")
                 f.write("Source Code:\n")
@@ -1339,7 +1339,7 @@ def main():
         # Generate latest apps data for widget
         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
             data = ''
-            with open(os.path.join('stats', 'latestapps.txt'), 'r') as f:
+            with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
                 for line in f:
                     appid = line.rstrip()
                     data += appid + "\t"
@@ -1348,7 +1348,7 @@ def main():
                     if app.icon is not None:
                         data += app.icon + "\t"
                     data += app.License + "\n"
-            with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
+            with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
                 f.write(data)
 
     if cachechanged: