Merge branch '334-make-index-generation-fully-reproducible' into 'master'

Resolve "make index generation fully reproducible"

Closes #334

See merge request fdroid/fdroidserver!343

fdroidserver/index.py

@@ -30,6 +30,7 @@ import shutil
 import tempfile
 import urllib.parse
 import zipfile
+import calendar
 from binascii import hexlify, unhexlify
 from datetime import datetime
 from xml.dom.minidom import Document
@@ -163,9 +164,16 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
 
     def _index_encoder_default(obj):
         if isinstance(obj, set):
-            return list(obj)
+            return sorted(list(obj))
         if isinstance(obj, datetime):
-            return int(obj.timestamp() * 1000)  # Java expects milliseconds
+            # Java prefers milliseconds
+            # we also need to accound for time zone/daylight saving time
+            return int(calendar.timegm(obj.timetuple()) * 1000)
+        if isinstance(obj, dict):
+            d = collections.OrderedDict()
+            for key in sorted(obj.keys()):
+                d[key] = obj[key]
+            return d
         raise TypeError(repr(obj) + " is not JSON serializable")
 
     output = collections.OrderedDict()
@@ -205,6 +213,17 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
                 k = k[:1].lower() + k[1:]
             d[k] = v
 
+    # establish sort order in localized dicts
+    for app in output['apps']:
+        localized = app.get('localized')
+        if localized:
+            lordered = collections.OrderedDict()
+            for lkey, lvalue in sorted(localized.items()):
+                lordered[lkey] = collections.OrderedDict()
+                for ikey, iname in sorted(lvalue.items()):
+                    lordered[lkey][ikey] = iname
+            app['localized'] = lordered
+
     output_packages = collections.OrderedDict()
     output['packages'] = output_packages
     for package in packages:

fdroidserver/update.py

@@ -28,6 +28,7 @@ import socket
 import zipfile
 import hashlib
 import pickle
+import time
 from datetime import datetime, timedelta
 from argparse import ArgumentParser
 
@@ -930,7 +931,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 
         if use_date_from_file:
             timestamp = stat.st_ctime
-            default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
+            default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
         else:
             default_date_param = None