update: add --rename-apks to force APK filenames to fdroid standard

uses the standard package.name_123.apk.  If that exists, it appends the
shasum.  If that exists, then its a duplicate, so its deleted. This should
help @SergeWinters with his 12,000 APKs.

completion/bash-completion

@@ -125,7 +125,7 @@ __complete_update() {
 	opts="-c -v -q -b -i -I -e -w"
 	lopts="--create-metadata --verbose --quiet --buildreport
  --interactive --icons --editor --wiki --pretty --clean --delete-unknown
- --nosign --use-date-from-apk"
+ --nosign --rename-apks --use-date-from-apk"
 	case "${prev}" in
 		-e|--editor)
 			_filedir

fdroidserver/update.py

@@ -1089,8 +1089,14 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
     """
 
     if ' ' in apkfilename:
-        logging.critical("Spaces in filenames are not allowed.")
-        return True, None, False
+        if options.rename_apks:
+            newfilename = apkfilename.replace(' ', '_')
+            os.rename(os.path.join(repodir, apkfilename),
+                      os.path.join(repodir, newfilename))
+            apkfilename = newfilename
+        else:
+            logging.critical("Spaces in filenames are not allowed.")
+            return True, None, False
 
     apkfile = os.path.join(repodir, apkfilename)
     shasum = sha256sum(apkfile)
@@ -1108,13 +1114,8 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
     if not usecache:
         logging.debug("Processing " + apkfilename)
         apk = {}
-        apk['apkName'] = apkfilename
         apk['hash'] = shasum
         apk['hashType'] = 'sha256'
-        srcfilename = apkfilename[:-4] + "_src.tar.gz"
-        if os.path.exists(os.path.join(repodir, srcfilename)):
-            apk['srcname'] = srcfilename
-        apk['size'] = os.path.getsize(apkfile)
         apk['uses-permission'] = []
         apk['uses-permission-sdk-23'] = []
         apk['features'] = []
@@ -1147,6 +1148,35 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
             logging.critical("Failed to get apk signature")
             return True, None, False
 
+        if options.rename_apks:
+            n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
+            std_short_name = os.path.join(repodir, n)
+            if apkfile != std_short_name:
+                if os.path.exists(std_short_name):
+                    std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
+                    if apkfile != std_long_name:
+                        if os.path.exists(std_long_name):
+                            dupdir = os.path.join('duplicates', repodir)
+                            if not os.path.isdir(dupdir):
+                                os.makedirs(dupdir, exist_ok=True)
+                            dupfile = os.path.join('duplicates', std_long_name)
+                            logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
+                            os.rename(apkfile, dupfile)
+                            return True, None, False
+                        else:
+                            os.rename(apkfile, std_long_name)
+                    apkfile = std_long_name
+                else:
+                    os.rename(apkfile, std_short_name)
+                    apkfile = std_short_name
+                apkfilename = apkfile[len(repodir) + 1:]
+
+        apk['apkName'] = apkfilename
+        srcfilename = apkfilename[:-4] + "_src.tar.gz"
+        if os.path.exists(os.path.join(repodir, srcfilename)):
+            apk['srcname'] = srcfilename
+        apk['size'] = os.path.getsize(apkfile)
+
         apkzip = zipfile.ZipFile(apkfile, 'r')
 
         # if an APK has files newer than the system time, suggest updating
@@ -1498,6 +1528,8 @@ def main():
                         help="When configured for signed indexes, create only unsigned indexes at this stage")
     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
                         help="Use date from apk instead of current time for newly added apks")
+    parser.add_argument("--rename-apks", action="store_true", default=False,
+                        help="Rename APK files that do not match package.name_123.apk")
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
     metadata.warnings_action = options.W
@@ -1517,6 +1549,9 @@ def main():
         resize_all_icons(repodirs)
         sys.exit(0)
 
+    if options.rename_apks:
+        options.clean = True
+
     # check that icons exist now, rather than fail at the end of `fdroid update`
     for k in ['repo_icon', 'archive_icon']:
         if k in config:

tests/run-tests

@@ -239,6 +239,35 @@ test -e repo/obb.main.twoversions_1101617_src.tar.gz.asc
 sed -i --expression='s,timestamp="[0-9]*",timestamp="1480431575",' repo/index.xml
 diff $WORKSPACE/tests/repo/index.xml repo/index.xml
 
+
+#------------------------------------------------------------------------------#
+echo_header 'rename apks with `fdroid update --rename-apks`'
+
+REPOROOT=`create_test_dir`
+cd $REPOROOT
+cp $WORKSPACE/tests/keystore.jks $REPOROOT/
+$fdroid init --keystore keystore.jks --repo-keyalias=sova
+echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo "accepted_formats = ['txt', 'yml']" >> config.py
+echo 'keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.py
+test -d metadata || mkdir metadata
+cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/
+test -d repo || mkdir repo
+cp $WORKSPACE/tests/urzip.apk "repo/asdfiuhk urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234 ö.apk"
+$fdroid update --rename-apks
+test -e repo/info.guardianproject.urzip_100.apk
+cp $WORKSPACE/tests/urzip-release.apk repo/
+$fdroid update --rename-apks
+test -e repo/info.guardianproject.urzip_100.apk
+test -e repo/info.guardianproject.urzip_100_b4964fd.apk
+cp $WORKSPACE/tests/urzip-release.apk repo/
+$fdroid update --rename-apks
+test -e repo/info.guardianproject.urzip_100.apk
+test -e repo/info.guardianproject.urzip_100_b4964fd.apk
+test -e duplicates/repo/info.guardianproject.urzip_100_b4964fd.apk
+
+
 #------------------------------------------------------------------------------#
 echo_header "test metadata checks"
 

tests/update.TestCase

@@ -200,6 +200,7 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options = type('', (), {})()
         fdroidserver.update.options.clean = True
         fdroidserver.update.options.delete_unknown = True
+        fdroidserver.update.options.rename_apks = False
 
         apps = fdroidserver.metadata.read_metadata(xref=True)
         knownapks = fdroidserver.common.KnownApks()