Merge branch 'update-import-fixups' into 'master'

update/import fixups

Closes #548 and #472

See merge request fdroid/fdroidserver!584

completion/bash-completion

@@ -186,10 +186,14 @@ __complete_checkupdates() {
 }
 
 __complete_import() {
-	opts="-u -s -q"
+	opts="-c -h -l -q -s -u -v -W"
-	lopts="--url --subdir --rev --quiet"
+	lopts="--categories --license --quiet --rev --subdir --url"
 	case "${prev}" in
-		-u|--url|-s|--subdir|--rev) return 0;;
+		-c|-l|-s|-u|--categories|--license|--quiet|--rev|--subdir|--url)
+		return 0;;
+		-W)
+		COMPREPLY=( $( compgen -W "error warn ignore" -- $cur ) )
+		return 0;;
 	esac
 	__complete_options
 }

fdroidserver/import.py

@@ -227,11 +227,11 @@ def main():
     if local_metadata_files != []:
         raise FDroidException(_("This repo already has local metadata: %s") % local_metadata_files[0])
 
+    build = metadata.Build()
     if options.url is None and os.path.isdir('.git'):
         app.AutoName = os.path.basename(os.getcwd())
         app.RepoType = 'git'
 
-        build = metadata.Build()
         root_dir = get_subdir(os.getcwd())
         if os.path.exists('build.gradle'):
             build.gradle = ['yes']
@@ -250,7 +250,6 @@ def main():
         write_local_file = True
     elif options.url:
         root_dir, build_dir = get_metadata_from_url(app, options.url)
-        build = metadata.Build()
         build.commit = '?'
         build.disable = 'Generated by import.py - check/set version fields and commit id'
         write_local_file = False
@@ -286,14 +285,14 @@ def main():
         raise FDroidException("Package " + package + " already exists")
 
     # Create a build line...
-    build.versionName = versionName or '?'
+    build.versionName = versionName or 'Unknown'
-    build.versionCode = versionCode or '?'
+    build.versionCode = versionCode or '0'  # TODO heinous but this is still a str
     if options.subdir:
         build.subdir = options.subdir
     if options.license:
         app.License = options.license
     if options.categories:
-        app.Categories = options.categories
+        app.Categories = options.categories.split(',')
     if os.path.exists(os.path.join(root_dir, 'jni')):
         build.buildjni = ['yes']
     if os.path.exists(os.path.join(root_dir, 'build.gradle')):

fdroidserver/metadata.py

@@ -1096,6 +1096,11 @@ def parse_yaml_metadata(mf, app):
 
 
 def write_yaml(mf, app):
+    """Write metadata in yaml format.
+
+    :param mf: active file discriptor for writing
+    :param app: app metadata to written to the yaml file
+    """
 
     # import rumael.yaml and check version
     try:
@@ -1176,12 +1181,10 @@ def write_yaml(mf, app):
         return cm
 
     def _builds_to_yaml(app):
-        fields = ['versionName', 'versionCode']
-        fields.extend(build_flags_order)
         builds = ruamel.yaml.comments.CommentedSeq()
         for build in app.builds:
             b = ruamel.yaml.comments.CommentedMap()
-            for field in fields:
+            for field in build_flags:
                 if hasattr(build, field) and getattr(build, field):
                     value = getattr(build, field)
                     if field == 'gradle' and value == ['off']:

fdroidserver/update.py

@@ -1090,8 +1090,6 @@ def scan_apk(apk_file):
     if 'minSdkVersion' not in apk:
         logging.warning("No SDK version information found in {0}".format(apk_file))
         apk['minSdkVersion'] = 3  # aapt defaults to 3 as the min
-    if 'targetSdkVersion' not in apk:
-        apk['targetSdkVersion'] = apk['minSdkVersion']
 
     # Check for known vulnerabilities
     if has_known_vulnerability(apk_file):

tests/config.py

@@ -17,6 +17,8 @@ archive_description = """
 The repository of older versions of applications from the main demo repository.
 """
 
+make_current_version_link = False
+
 repo_keyalias = "sova"
 keystore = "keystore.jks"
 keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="

tests/lint.TestCase

@@ -103,6 +103,16 @@ class LintTest(unittest.TestCase):
                     'Multimedia,Graphics',
                 ],
             },
+            'WebSite': {
+                'good': [
+                    'https://homepage.com',
+                ],
+                'bad': [
+                    [],
+                    ['nope', ],
+                    29,
+                ],
+            },
         }
 
         for field, values in fields.items():

tests/metadata.TestCase

@@ -165,6 +165,50 @@ class MetadataTest(unittest.TestCase):
             with self.assertRaises(MetaDataException):
                 fdroidserver.metadata.parse_yaml_metadata(mf, {})
 
+    def test_write_yaml_with_placeholder_values(self):
+        mf = io.StringIO()
+
+        app = fdroidserver.metadata.App()
+        app.Categories = ['None']
+        app.SourceCode = "https://gitlab.com/fdroid/fdroidclient.git"
+        app.IssueTracker = "https://gitlab.com/fdroid/fdroidclient/issues"
+        app.RepoType = 'git'
+        app.Repo = 'https://gitlab.com/fdroid/fdroidclient.git'
+        app.AutoUpdateMode = 'None'
+        app.UpdateCheckMode = 'Tags'
+        build = fdroidserver.metadata.Build()
+        build.versionName = 'Unknown'  # taken from fdroidserver/import.py
+        build.versionCode = '0'  # taken from fdroidserver/import.py
+        build.disable = 'Generated by import.py ...'
+        build.commit = 'Unknown'
+        build.gradle = [True]
+        app.builds = [build]
+
+        fdroidserver.metadata.write_yaml(mf, app)
+
+        mf.seek(0)
+        self.assertEqual(mf.read(), textwrap.dedent("""\
+            Categories:
+              - None
+            License: Unknown
+            SourceCode: https://gitlab.com/fdroid/fdroidclient.git
+            IssueTracker: https://gitlab.com/fdroid/fdroidclient/issues
+
+            RepoType: git
+            Repo: https://gitlab.com/fdroid/fdroidclient.git
+
+            Builds:
+              - versionName: Unknown
+                versionCode: 0
+                disable: Generated by import.py ...
+                commit: Unknown
+                gradle:
+                  - true
+
+            AutoUpdateMode: None
+            UpdateCheckMode: Tags
+            """))
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))
@@ -176,4 +220,4 @@ if __name__ == "__main__":
 
     newSuite = unittest.TestSuite()
     newSuite.addTest(unittest.makeSuite(MetadataTest))
-    unittest.main(failfast=False)
+    unittest.main(failfast=True)

tests/metadata/info.zwanenburg.caffeinetile.yml

@@ -0,0 +1,4 @@
+Categories:
+- Development
+Name: Caffeine Tile
+Summary: Test app for extracting icons when an XML one is default

tests/metadata/no.min.target.sdk.yml

@@ -0,0 +1,3 @@
+Categories: [Development]
+Name: No minSdkVersion or targetSdkVersion
+Summary: An APK without any <uses-sdk> block in AndroidManifest.xml

tests/repo/categories.txt

@@ -3,8 +3,8 @@
 Development
 GuardianProject
 Multimedia
-None
 Phone & SMS
 Security
 System
 Time
+tests

tests/repo/index-v1.json

@@ -41,6 +41,19 @@
       "packageName": "souch.smsbypass",
       "lastUpdated": 1524700800000
     },
+    {
+      "categories": [
+        "Development"
+      ],
+      "suggestedVersionCode": "4",
+      "license": "Unknown",
+      "name": "Caffeine Tile",
+      "summary": "Test app for extracting icons when an XML one is default",
+      "added": 1539129600000,
+      "icon": "info.zwanenburg.caffeinetile.4.xml",
+      "packageName": "info.zwanenburg.caffeinetile",
+      "lastUpdated": 1539129600000
+    },
     {
       "categories": [
         "tests"
@@ -72,6 +85,19 @@
       "packageName": "fake.ota.update",
       "lastUpdated": 1457568000000
     },
+    {
+      "categories": [
+        "Development"
+      ],
+      "suggestedVersionCode": "987",
+      "license": "Unknown",
+      "name": "No minSdkVersion or targetSdkVersion",
+      "summary": "An APK without any <uses-sdk> block in AndroidManifest.xml",
+      "added": 1539129600000,
+      "icon": "no.min.target.sdk.987.png",
+      "packageName": "no.min.target.sdk",
+      "lastUpdated": 1539129600000
+    },
     {
       "bitcoin": "1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk",
       "categories": [
@@ -248,7 +274,6 @@
         "sig": "b4964fd759edaa54e65bb476d0276880",
         "signer": "32a23624c201b949f085996ba5ed53d40f703aca4989476949cae891022e0ed6",
         "size": 18489,
-        "targetSdkVersion": "3",
         "uses-permission": [
           [
             "android.permission.READ_CALENDAR",
@@ -290,7 +315,6 @@
         "sig": "b4964fd759edaa54e65bb476d0276880",
         "signer": "32a23624c201b949f085996ba5ed53d40f703aca4989476949cae891022e0ed6",
         "size": 17552,
-        "targetSdkVersion": "3",
         "uses-permission": [
           [
             "android.permission.READ_CALENDAR",
@@ -400,6 +424,57 @@
         "versionName": "0.1"
       }
     ],
+    "info.zwanenburg.caffeinetile": [
+      {
+        "added": 1539129600000,
+        "apkName": "info.zwanenburg.caffeinetile_4.apk",
+        "hash": "dbbdd7deadb038862f426b71efe4a64df8c3edf25d669e935f349510e16f65db",
+        "hashType": "sha256",
+        "minSdkVersion": "24",
+        "packageName": "info.zwanenburg.caffeinetile",
+        "sig": "03f9b2f848d22fd1d8d1331e8b1b486d",
+        "signer": "51cfa5c8a743833ad89acf81cb755936876a5c8b8eca54d1ffdcec0cdca25d0e",
+        "size": 11740,
+        "targetSdkVersion": "25",
+        "uses-permission": [
+          [
+            "android.permission.WAKE_LOCK",
+            null
+          ]
+        ],
+        "versionCode": 4,
+        "versionName": "1.3"
+      }
+    ],
+    "no.min.target.sdk": [
+      {
+        "added": 1539129600000,
+        "apkName": "no.min.target.sdk_987.apk",
+        "hash": "e2e1dc1d550df2b5bc383860139207258645b5540abeccd305ed8b2cb6459d2c",
+        "hashType": "sha256",
+        "minSdkVersion": 3,
+        "packageName": "no.min.target.sdk",
+        "sig": "b4964fd759edaa54e65bb476d0276880",
+        "signer": "32a23624c201b949f085996ba5ed53d40f703aca4989476949cae891022e0ed6",
+        "size": 14102,
+        "uses-permission": [
+          [
+            "android.permission.WRITE_EXTERNAL_STORAGE",
+            null
+          ],
+          [
+            "android.permission.READ_PHONE_STATE",
+            null
+          ],
+          [
+            "android.permission.READ_EXTERNAL_STORAGE",
+            null
+          ]
+        ],
+        "versionCode": 987,
+        "versionName": "1.2-fake"
+      }
+    ],
     "obb.main.oldversion": [
       {
         "added": 1388448000000,

tests/repo/index.xml

@@ -39,6 +39,35 @@
 			<permissions>READ_CONTACTS,READ_EXTERNAL_STORAGE,RECEIVE_SMS,SEND_SMS,VIBRATE,WRITE_EXTERNAL_STORAGE</permissions>
 		</package>
 	</application>
+	<application id="info.zwanenburg.caffeinetile">
+		<id>info.zwanenburg.caffeinetile</id>
+		<added>2018-10-10</added>
+		<lastupdated>2018-10-10</lastupdated>
+		<name>Caffeine Tile</name>
+		<summary>Test app for extracting icons when an XML one is default</summary>
+		<icon>info.zwanenburg.caffeinetile.4.xml</icon>
+		<desc>&lt;p&gt;No description available&lt;/p&gt;</desc>
+		<license>Unknown</license>
+		<categories>Development</categories>
+		<category>Development</category>
+		<web></web>
+		<source></source>
+		<tracker></tracker>
+		<marketversion></marketversion>
+		<marketvercode>4</marketvercode>
+		<package>
+			<version>1.3</version>
+			<versioncode>4</versioncode>
+			<apkname>info.zwanenburg.caffeinetile_4.apk</apkname>
+			<hash type="sha256">dbbdd7deadb038862f426b71efe4a64df8c3edf25d669e935f349510e16f65db</hash>
+			<size>11740</size>
+			<sdkver>24</sdkver>
+			<targetSdkVersion>25</targetSdkVersion>
+			<added>2018-10-10</added>
+			<sig>03f9b2f848d22fd1d8d1331e8b1b486d</sig>
+			<permissions>WAKE_LOCK</permissions>
+		</package>
+	</application>
 	<application id="duplicate.permisssions">
 		<id>duplicate.permisssions</id>
 		<added>2017-12-22</added>
@@ -95,6 +124,34 @@
 			<added>2016-03-10</added>
 		</package>
 	</application>
+	<application id="no.min.target.sdk">
+		<id>no.min.target.sdk</id>
+		<added>2018-10-10</added>
+		<lastupdated>2018-10-10</lastupdated>
+		<name>No minSdkVersion or targetSdkVersion</name>
+		<summary>An APK without any &lt;uses-sdk&gt; block in AndroidManifest.xml</summary>
+		<icon>no.min.target.sdk.987.png</icon>
+		<desc>&lt;p&gt;No description available&lt;/p&gt;</desc>
+		<license>Unknown</license>
+		<categories>Development</categories>
+		<category>Development</category>
+		<web></web>
+		<source></source>
+		<tracker></tracker>
+		<marketversion></marketversion>
+		<marketvercode>987</marketvercode>
+		<package>
+			<version>1.2-fake</version>
+			<versioncode>987</versioncode>
+			<apkname>no.min.target.sdk_987.apk</apkname>
+			<hash type="sha256">e2e1dc1d550df2b5bc383860139207258645b5540abeccd305ed8b2cb6459d2c</hash>
+			<size>14102</size>
+			<sdkver>3</sdkver>
+			<added>2018-10-10</added>
+			<sig>b4964fd759edaa54e65bb476d0276880</sig>
+			<permissions>READ_EXTERNAL_STORAGE,READ_PHONE_STATE,WRITE_EXTERNAL_STORAGE</permissions>
+		</package>
+	</application>
 	<application id="obb.main.oldversion">
 		<id>obb.main.oldversion</id>
 		<added>2013-12-31</added>
@@ -269,7 +326,6 @@
 			<hash type="sha256">c809bdff83715fbf919f3840ee09869b038e209378b906e135ee40d3f0e1f075</hash>
 			<size>18489</size>
 			<sdkver>3</sdkver>
-			<targetSdkVersion>3</targetSdkVersion>
 			<added>2017-06-23</added>
 			<sig>b4964fd759edaa54e65bb476d0276880</sig>
 			<permissions>READ_CALENDAR,READ_EXTERNAL_STORAGE,READ_PHONE_STATE,RECEIVE_BOOT_COMPLETED,WRITE_EXTERNAL_STORAGE</permissions>
@@ -281,7 +337,6 @@
 			<hash type="sha256">665d03d61ebc642289fda697f71a59305b0202b16cafc5ffdae91cbe91f0b25d</hash>
 			<size>17552</size>
 			<sdkver>3</sdkver>
-			<targetSdkVersion>3</targetSdkVersion>
 			<added>2017-06-23</added>
 			<sig>b4964fd759edaa54e65bb476d0276880</sig>
 			<permissions>READ_CALENDAR,READ_EXTERNAL_STORAGE,READ_PHONE_STATE,RECEIVE_BOOT_COMPLETED,WRITE_EXTERNAL_STORAGE</permissions>

tests/stats/known_apks.txt

@@ -4,6 +4,8 @@ com.politedroid_5.apk com.politedroid 2017-06-23
 com.politedroid_6.apk com.politedroid 2017-06-23
 duplicate.permisssions_9999999.apk duplicate.permisssions 2017-12-22
 fake.ota.update_1234.zip fake.ota.update 2016-03-10
+info.zwanenburg.caffeinetile_4.apk info.zwanenburg.caffeinetile 2018-10-10
+no.min.target.sdk_987.apk no.min.target.sdk 2018-10-10
 obb.main.oldversion_1444412523.apk obb.main.oldversion 2013-12-31
 obb.main.twoversions_1101613.apk obb.main.twoversions 2015-10-12
 obb.main.twoversions_1101615.apk obb.main.twoversions 2016-01-01

tests/update.TestCase

@@ -254,14 +254,14 @@ class UpdateTest(unittest.TestCase):
         apps = fdroidserver.metadata.read_metadata(xref=True)
         knownapks = fdroidserver.common.KnownApks()
         apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
-        self.assertEqual(len(apks), 14)
+        self.assertEqual(len(apks), 15)
         apk = apks[0]
         self.assertEqual(apk['packageName'], 'com.politedroid')
         self.assertEqual(apk['versionCode'], 3)
         self.assertEqual(apk['minSdkVersion'], '3')
-        self.assertEqual(apk['targetSdkVersion'], '3')
+        self.assertIsNone(apk.get('targetSdkVersion'))
         self.assertFalse('maxSdkVersion' in apk)
-        apk = apks[6]
+        apk = apks[7]
         self.assertEqual(apk['packageName'], 'obb.main.oldversion')
         self.assertEqual(apk['versionCode'], 1444412523)
         self.assertEqual(apk['minSdkVersion'], '4')
@@ -321,7 +321,7 @@ class UpdateTest(unittest.TestCase):
 
         fdroidserver.update.options.clean = False
         read_from_json = fdroidserver.update.get_cache()
-        self.assertEqual(16, len(read_from_json))
+        self.assertEqual(17, len(read_from_json))
         for f in glob.glob('repo/*.apk'):
             self.assertTrue(os.path.basename(f) in read_from_json)
 
@@ -623,7 +623,7 @@ class UpdateTest(unittest.TestCase):
         knownapks = fdroidserver.common.KnownApks()
         apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
         fdroidserver.update.translate_per_build_anti_features(apps, apks)
-        self.assertEqual(len(apks), 14)
+        self.assertEqual(len(apks), 15)
         foundtest = False
         for apk in apks:
             if apk['packageName'] == 'com.politedroid' and apk['versionCode'] == 3: