Merge branch 'collection-fixes-and-KnownVuln' into 'master'

add KnownVuln anti-feature, and other fixes

This is a bit of a random collection of things that I have added in the process of working on the drozer/scanner support, YAML support, etc.  The only new things are:

* adding new AntiFeature for security issues: `KnownVuln`
* removing broken, incomplete XML metadata support

Everything else included are just code improvements.  This also includes the first check for `KnownVuln`, which is a scanner to check whether custom OpenSSL binaries in apps are not old with known vulnerabilities.  `KnownVuln` will then ultimately be used for things like the drozer scanner !187

See merge request !189

examples/config.py

@@ -240,8 +240,7 @@ The repository of older versions of applications from the main demo repository.
 # build_server_always = True
 
 # By default, fdroid will use YAML .yml and the custom .txt metadata formats. It
-# is also possible to have metadata in JSON and XML by adding 'json' and
+# is also possible to have metadata in JSON by adding 'json'.
-# 'xml'.
 # accepted_formats = ['txt', 'yml']
 
 # Limit in number of characters that fields can take up

fdroidserver/checkupdates.py

@@ -109,9 +109,7 @@ def check_tags(app, pattern):
 
         vcs.gotorevision(None)
 
-        last_build = metadata.Build()
+        last_build = app.get_last_build()
-        if len(app.builds) > 0:
-            last_build = app.builds[-1]
 
         if last_build.submodules:
             vcs.initsubmodules()
@@ -322,9 +320,7 @@ def possible_subdirs(app):
     else:
         build_dir = os.path.join('build', app.id)
 
-    last_build = metadata.Build()
+    last_build = app.get_last_build()
-    if len(app.builds) > 0:
-        last_build = app.builds[-1]
 
     for d in dirs_with_manifest(build_dir):
         m_paths = common.manifest_paths(d, last_build.gradle)
@@ -351,9 +347,7 @@ def fetch_autoname(app, tag):
     except VCSException:
         return None
 
-    last_build = metadata.Build()
+    last_build = app.get_last_build()
-    if len(app.builds) > 0:
-        last_build = app.builds[-1]
 
     logging.debug("...fetch auto name from " + build_dir)
     new_name = None

fdroidserver/common.py

@@ -36,8 +36,8 @@ import socket
 import base64
 import xml.etree.ElementTree as XMLElementTree
 
+from distutils.version import LooseVersion
 from queue import Queue
-
 from zipfile import ZipFile
 
 import fdroidserver.metadata
@@ -291,15 +291,37 @@ def find_sdk_tools_cmd(cmd):
             tooldirs.append(sdk_platform_tools)
     tooldirs.append('/usr/bin')
     for d in tooldirs:
-        if os.path.isfile(os.path.join(d, cmd)):
+        path = os.path.join(d, cmd)
-            return os.path.join(d, cmd)
+        if os.path.isfile(path):
+            if cmd == 'aapt':
+                test_aapt_version(path)
+            return path
     # did not find the command, exit with error message
     ensure_build_tools_exists(config)
 
 
+def test_aapt_version(aapt):
+    '''Check whether the version of aapt is new enough'''
+    output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
+    if output is None or output == '':
+        logging.error(aapt + ' failed to execute!')
+    else:
+        m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
+        if m:
+            major = m.group(1)
+            minor = m.group(2)
+            bugfix = m.group(3)
+            # the Debian package has the version string like "v0.2-23.0.2"
+            if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
+                logging.warning(aapt + ' is too old, fdroid requires build-tools-23.0.0 or newer!')
+        else:
+            logging.warning('Unknown version of aapt, might cause problems: ' + output)
+
+
 def test_sdk_exists(thisconfig):
     if 'sdk_path' not in thisconfig:
         if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
+            test_aapt_version(thisconfig['aapt'])
             return True
         else:
             logging.error("'sdk_path' not set in config.py!")
@@ -1684,6 +1706,8 @@ def SdkToolsPopen(commands, cwd=None, output=True):
     if abscmd is None:
         logging.critical("Could not find '%s' on your system" % cmd)
         sys.exit(1)
+    if cmd == 'aapt':
+        test_aapt_version(config['aapt'])
     return FDroidPopen([abscmd] + commands[1:],
                        cwd=cwd, output=output)
 

fdroidserver/metadata.py

@@ -35,9 +35,6 @@ except ImportError:
     from yaml import Loader
     YamlLoader = Loader
 
-# use the C implementation when available
-import xml.etree.cElementTree as ElementTree
-
 import fdroidserver.common
 
 srclibs = None
@@ -239,6 +236,12 @@ class App():
                 self.__dict__[k] = v
                 self._modified.add(k)
 
+    def get_last_build(self):
+        if len(self.builds) > 0:
+            return self.builds[-1]
+        else:
+            return Build()
+
 
 TYPE_UNKNOWN = 0
 TYPE_OBSOLETE = 1
@@ -423,22 +426,21 @@ def flagtype(name):
     return TYPE_STRING
 
 
-# Designates a metadata field type and checks that it matches
-#
-# 'name'     - The long name of the field type
-# 'matching' - List of possible values or regex expression
-# 'sep'      - Separator to use if value may be a list
-# 'fields'   - Metadata fields (Field:Value) of this type
-# 'flags'    - Build flags (flag=value) of this type
-#
 class FieldValidator():
+    """
+    Designates App metadata field types and checks that it matches
 
-    def __init__(self, name, matching, fields, flags):
+    'name'     - The long name of the field type
+    'matching' - List of possible values or regex expression
+    'sep'      - Separator to use if value may be a list
+    'fields'   - Metadata fields (Field:Value) of this type
+    """
+
+    def __init__(self, name, matching, fields):
         self.name = name
         self.matching = matching
         self.compiled = re.compile(matching)
         self.fields = fields
-        self.flags = flags
 
     def check(self, v, appid):
         if not v:
@@ -455,63 +457,49 @@ class FieldValidator():
 
 # Generic value types
 valuetypes = {
-    FieldValidator("Integer",
-                   r'^[1-9][0-9]*$',
-                   [],
-                   ['vercode']),
-
     FieldValidator("Hexadecimal",
                    r'^[0-9a-f]+$',
-                   ['FlattrID'],
+                   ['FlattrID']),
-                   []),
 
     FieldValidator("HTTP link",
                    r'^http[s]?://',
-                   ["WebSite", "SourceCode", "IssueTracker", "Changelog", "Donate"], []),
+                   ["WebSite", "SourceCode", "IssueTracker", "Changelog", "Donate"]),
 
     FieldValidator("Email",
                    r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$',
-                   ["AuthorEmail"], []),
+                   ["AuthorEmail"]),
 
     FieldValidator("Bitcoin address",
                    r'^[a-zA-Z0-9]{27,34}$',
-                   ["Bitcoin"],
+                   ["Bitcoin"]),
-                   []),
 
     FieldValidator("Litecoin address",
                    r'^L[a-zA-Z0-9]{33}$',
-                   ["Litecoin"],
+                   ["Litecoin"]),
-                   []),
 
     FieldValidator("Repo Type",
                    r'^(git|git-svn|svn|hg|bzr|srclib)$',
-                   ["RepoType"],
+                   ["RepoType"]),
-                   []),
 
     FieldValidator("Binaries",
                    r'^http[s]?://',
-                   ["Binaries"],
+                   ["Binaries"]),
-                   []),
 
     FieldValidator("Archive Policy",
                    r'^[0-9]+ versions$',
-                   ["ArchivePolicy"],
+                   ["ArchivePolicy"]),
-                   []),
 
     FieldValidator("Anti-Feature",
-                   r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets)$',
+                   r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets|KnownVuln)$',
-                   ["AntiFeatures"],
+                   ["AntiFeatures"]),
-                   []),
 
     FieldValidator("Auto Update Mode",
                    r"^(Version .+|None)$",
-                   ["AutoUpdateMode"],
+                   ["AutoUpdateMode"]),
-                   []),
 
     FieldValidator("Update Check Mode",
                    r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$",
-                   ["UpdateCheckMode"],
+                   ["UpdateCheckMode"])
-                   [])
 }
 
 
@@ -522,11 +510,6 @@ def check_metadata(app):
             if k not in app._modified:
                 continue
             v.check(app.__dict__[k], app.id)
-        for build in app.builds:
-            for k in v.flags:
-                if k not in build._modified:
-                    continue
-                v.check(build.__dict__[k], app.id)
 
 
 # Formatter for descriptions. Create an instance, and call parseline() with
@@ -818,10 +801,8 @@ def read_metadata(xref=True, check_vcs=[]):
 
     for metadatapath in sorted(glob.glob(os.path.join('metadata', '*.txt'))
                                + glob.glob(os.path.join('metadata', '*.json'))
-                               + glob.glob(os.path.join('metadata', '*.xml'))
                                + glob.glob(os.path.join('metadata', '*.yml'))
                                + glob.glob('.fdroid.json')
-                               + glob.glob('.fdroid.xml')
                                + glob.glob('.fdroid.yml')):
         packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
         if packageName in apps:
@@ -1001,8 +982,6 @@ def parse_metadata(metadatapath, check_vcs=False):
             parse_txt_metadata(mf, app)
         elif ext == 'json':
             parse_json_metadata(mf, app)
-        elif ext == 'xml':
-            parse_xml_metadata(mf, app)
         elif ext == 'yml':
             parse_yaml_metadata(mf, app)
         else:
@@ -1013,7 +992,8 @@ def parse_metadata(metadatapath, check_vcs=False):
         metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
         if not os.path.isfile(metadata_in_repo):
             vcs, build_dir = fdroidserver.common.setup_vcs(app)
-            vcs.gotorevision('HEAD')  # HEAD since we can't know where else to go
+            if isinstance(vcs, fdroidserver.common.vcs_git):
+                vcs.gotorevision('HEAD')  # HEAD since we can't know where else to go
         if os.path.isfile(metadata_in_repo):
             logging.debug('Including metadata from ' + metadata_in_repo)
             app.update(parse_metadata(metadata_in_repo))
@@ -1046,38 +1026,6 @@ def parse_json_metadata(mf, app):
     return app
 
 
-def parse_xml_metadata(mf, app):
-
-    tree = ElementTree.ElementTree(file=mf)
-    root = tree.getroot()
-
-    if root.tag != 'resources':
-        warn_or_exception('resources file does not have root element <resources/>')
-
-    for child in root:
-        if child.tag != 'builds':
-            # builds does not have name="" attrib
-            name = child.attrib['name']
-
-        if child.tag == 'string':
-            app.set_field(name, child.text)
-        elif child.tag == 'string-array':
-            for item in child:
-                app.append_field(name, item.text)
-        elif child.tag == 'builds':
-            for b in child:
-                build = Build()
-                for key in b:
-                    build.set_flag(key.tag, key.text)
-                app.builds.append(build)
-
-    # TODO handle this using <xsd:element type="xsd:boolean> in a schema
-    if not isinstance(app.RequiresRoot, bool):
-        app.RequiresRoot = app.RequiresRoot == 'true'
-
-    return app
-
-
 def parse_yaml_metadata(mf, app):
 
     yamlinfo = yaml.load(mf, Loader=YamlLoader)
@@ -1121,6 +1069,8 @@ def parse_txt_metadata(mf, app):
         build = Build()
         build.version = parts[0]
         build.vercode = parts[1]
+        check_versionCode(build.vercode)
+
         if parts[2].startswith('!'):
             # For backwards compatibility, handle old-style disabling,
             # including attempting to extract the commit from the message
@@ -1139,6 +1089,12 @@ def parse_txt_metadata(mf, app):
 
         return build
 
+    def check_versionCode(versionCode):
+        try:
+            int(versionCode)
+        except ValueError:
+            warn_or_exception('Invalid versionCode: "' + versionCode + '" is not an integer!')
+
     def add_comments(key):
         if not curcomments:
             return
@@ -1222,6 +1178,7 @@ def parse_txt_metadata(mf, app):
                 build = Build()
                 build.version = vv[0]
                 build.vercode = vv[1]
+                check_versionCode(build.vercode)
                 if build.vercode in vc_seen:
                     warn_or_exception('Duplicate build recipe found for vercode %s in %s'
                                       % (build.vercode, linedesc))

fdroidserver/update.py

@@ -440,6 +440,34 @@ def sha256sum(filename):
     return sha.hexdigest()
 
 
+def has_old_openssl(filename):
+    '''checks for known vulnerable openssl versions in the APK'''
+
+    # statically load this pattern
+    if not hasattr(has_old_openssl, "pattern"):
+        has_old_openssl.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
+
+    with zipfile.ZipFile(filename) as zf:
+        for name in zf.namelist():
+            if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
+                lib = zf.open(name)
+                while True:
+                    chunk = lib.read(4096)
+                    if chunk == b'':
+                        break
+                    m = has_old_openssl.pattern.search(chunk)
+                    if m:
+                        version = m.group(1).decode('ascii')
+                        if version.startswith('1.0.1') and version[5] >= 'r' \
+                           or version.startswith('1.0.2') and version[5] >= 'f':
+                            logging.debug('"%s" contains recent %s (%s)', filename, name, version)
+                        else:
+                            logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
+                            return True
+                        break
+    return False
+
+
 def insert_obbs(repodir, apps, apks):
     """Scans the .obb files in a given repo directory and adds them to the
     relevant APK instances.  OBB files have versionCodes like APK
@@ -639,6 +667,9 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
             apk['features'] = set()
             apk['icons_src'] = {}
             apk['icons'] = {}
+            apk['antiFeatures'] = set()
+            if has_old_openssl(apkfile):
+                apk['antiFeatures'].add('KnownVuln')
             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
             if p.returncode != 0:
                 if options.delete_unknown:
@@ -1109,10 +1140,6 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
         addElement('marketversion', app.CurrentVersion, doc, apel)
         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 
-        if app.AntiFeatures:
-            af = app.AntiFeatures
-            if af:
-                addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
         if app.Provides:
             pv = app.Provides.split(',')
             addElementNonEmpty('provides', ','.join(pv), doc, apel)
@@ -1123,6 +1150,11 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
         # doesn't have to do any work by default...
         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 
+        if 'antiFeatures' in apklist[0]:
+            app.AntiFeatures.extend(apklist[0]['antiFeatures'])
+        if app.AntiFeatures:
+            addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
+
         # Check for duplicates - they will make the client unhappy...
         for i in range(len(apklist) - 1):
             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:

tests/metadata.TestCase

@@ -5,9 +5,9 @@
 import inspect
 import optparse
 import os
-import pickle
 import sys
 import unittest
+import yaml
 
 localmodule = os.path.realpath(
     os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
@@ -33,21 +33,21 @@ class MetadataTest(unittest.TestCase):
         config = dict()
         config['sdk_path'] = '/opt/android-sdk'
         config['ndk_paths'] = dict()
-        config['accepted_formats'] = ['json', 'txt', 'xml', 'yml']
+        config['accepted_formats'] = ['json', 'txt', 'yml']
         fdroidserver.common.config = config
 
         apps = fdroidserver.metadata.read_metadata(xref=True)
-        for appid in ('org.smssecure.smssecure', 'org.adaway', 'net.osmand.plus', 'org.videolan.vlc'):
+        for appid in ('org.smssecure.smssecure', 'org.adaway', 'org.videolan.vlc'):
             app = apps[appid]
-            savepath = os.path.join('metadata', appid + '.pickle')
+            savepath = os.path.join('metadata', 'dump', appid + '.yaml')
             frommeta = app.field_dict()
             self.assertTrue(appid in apps)
-            with open(savepath, 'rb') as f:
+            with open(savepath, 'r') as f:
-                frompickle = pickle.load(f)
+                frompickle = yaml.load(f)
             self.assertEqual(frommeta, frompickle)
             # Uncomment to overwrite
-            # with open(savepath, 'wb') as f:
+            # with open(savepath, 'w') as f:
-            #     pickle.dump(frommeta, f)
+            #     yaml.dump(frommeta, f, default_flow_style=False)
 
 
 if __name__ == "__main__":

tests/metadata/dump/org.smssecure.smssecure.yaml

@@ -0,0 +1,339 @@
+AntiFeatures: []
+Archive Policy: null
+Author Email: null
+Author Name: null
+Auto Name: SMSSecure
+Auto Update Mode: Version v%v
+Binaries: null
+Bitcoin: null
+Categories:
+- Phone & SMS
+Changelog: ''
+Current Version: 0.6.0
+Current Version Code: '102'
+Description: 'SMSSecure is an SMS/MMS application that allows you to protect your
+  privacy while communicating with friends.
+
+  Using SMSSecure, you can send SMS messages and share media or attachments with complete
+  privacy.
+
+
+  * Easy. SMSSecure works like any other SMS application. There''s nothing to sign
+  up for and no new service your friends need to join.
+
+  * Reliable. SMSSecure communicates using encrypted SMS messages. No servers or internet
+  connection required.
+
+  * Private. SMSSecure uses the TextSecure encryption protocol to provide privacy
+  for every message, every time.
+
+  * Safe. All messages are encrypted locally, so if your phone is lost or stolen,
+  your messages are protected.
+
+  * Open Source. SMSSecure is Free and Open Source, enabling anyone to verify its
+  security by auditing the code.'
+Disabled: null
+Donate: null
+FlattrID: null
+Issue Tracker: https://github.com/SMSSecure/SMSSecure/issues
+License: GPLv3
+Litecoin: null
+Maintainer Notes: ''
+Name: null
+No Source Since: ''
+Provides: null
+Repo: https://github.com/SMSSecure/SMSSecure
+Repo Type: git
+Requires Root: false
+Source Code: https://github.com/SMSSecure/SMSSecure
+Summary: Send encrypted text messages (SMS)
+Update Check Data: null
+Update Check Ignore: null
+Update Check Mode: Tags
+Update Check Name: null
+Vercode Operation: null
+Web Site: http://www.smssecure.org
+added: null
+builds:
+- antcommands: []
+  build: ''
+  buildjni: []
+  commit: 66367479a4f57f347b5cbe8f6f8f632adaae7727
+  disable: builds, merge changes into upstream
+  encoding: null
+  extlibs: []
+  forcevercode: true
+  forceversion: false
+  gradle:
+  - 'yes'
+  gradleprops: []
+  init: ''
+  kivy: false
+  maven: false
+  ndk: null
+  novcheck: false
+  oldsdkloc: false
+  output: null
+  patch: []
+  preassemble: []
+  prebuild: touch signing.properties && pushd $$GradleWitness$$ && gradle jar && popd
+    && cp $$GradleWitness$$/build/libs/GradleWitness.jar libs/gradle-witness.jar &&
+    sed -i -e '20,22d' build.gradle && pushd $$PreferenceFragment$$ && gradle uploadArchives
+    && popd && sed -i -e '/5470f5872514a6226fa1fc6f4e000991f38805691c534cf0bd2778911fc773ad/d'
+    build.gradle && mkdir smil && pushd smil && wget -c http://www.w3.org/TR/smil-boston-dom/java-binding.zip
+    && unzip java-binding.zip && popd && cp -fR smil/java/org src/ && rm -fR smil
+    && sed -i -e '/org.w3c.smil/d' build.gradle && cp -fR $$AospMms$$/src/org src/
+  rm:
+  - libs/*
+  scandelete: []
+  scanignore: []
+  srclibs:
+  - GradleWitness@10f1269c0aafdc1d478efc005ed48f3a47d44278
+  - PreferenceFragment@717a45433b927d2f0dfc5328f79e77c9682c37bc
+  - ShortcutBadger@3815ce2ec0c66acd7d7c0b4f2479df8fa70fed87
+  - AospMms@android-5.1.0_r3
+  subdir: null
+  submodules: false
+  target: null
+  update: []
+  vercode: '5'
+  version: 0.3.3
+- antcommands: []
+  build: ''
+  buildjni: []
+  commit: 9675ce5eecb929dcaddb43b3d9486fdb88b9ae1a
+  disable: builds, wait for upstream
+  encoding: null
+  extlibs: []
+  forcevercode: false
+  forceversion: false
+  gradle:
+  - 'yes'
+  gradleprops: []
+  init: ''
+  kivy: false
+  maven: false
+  ndk: null
+  novcheck: false
+  oldsdkloc: false
+  output: null
+  patch: []
+  preassemble: []
+  prebuild: touch signing.properties && pushd $$GradleWitness$$ && gradle jar && popd
+    && cp $$GradleWitness$$/build/libs/GradleWitness.jar libs/gradle-witness.jar
+  rm:
+  - libs/*.jar
+  scandelete: []
+  scanignore: []
+  srclibs:
+  - GradleWitness@10f1269c0aafdc1d478efc005ed48f3a47d44278
+  subdir: null
+  submodules: true
+  target: null
+  update: []
+  vercode: '6'
+  version: 0.3.3
+- antcommands: []
+  build: ''
+  buildjni: []
+  commit: v0.4.2
+  disable: builds locally, but not on BS
+  encoding: null
+  extlibs: []
+  forcevercode: false
+  forceversion: false
+  gradle:
+  - 'yes'
+  gradleprops: []
+  init: ''
+  kivy: false
+  maven: false
+  ndk: null
+  novcheck: false
+  oldsdkloc: false
+  output: null
+  patch: []
+  preassemble: []
+  prebuild: touch signing.properties && ./build-witness.sh && rm -rf libs/gradle-witness/build
+    && echo "org.gradle.jvmargs=-Xms512m -Xmx512m -XX:MaxPermSize=512m" >> gradle.properties
+  rm:
+  - libs/*.jar
+  scandelete: []
+  scanignore: []
+  srclibs: []
+  subdir: null
+  submodules: true
+  target: null
+  update: []
+  vercode: '9'
+  version: 0.4.2
+- antcommands: []
+  build: ''
+  buildjni: []
+  commit: v0.5.1
+  disable: false
+  encoding: null
+  extlibs: []
+  forcevercode: false
+  forceversion: false
+  gradle:
+  - 'yes'
+  gradleprops: []
+  init: ''
+  kivy: false
+  maven: false
+  ndk: null
+  novcheck: false
+  oldsdkloc: false
+  output: null
+  patch: []
+  preassemble: []
+  prebuild: touch signing.properties && ./build-witness.sh && rm -rf libs/gradle-witness/build
+    && echo "org.gradle.jvmargs=-Xms512m -Xmx512m -XX:MaxPermSize=512m" >> gradle.properties
+  rm:
+  - libs/*.jar
+  scandelete: []
+  scanignore: []
+  srclibs: []
+  subdir: null
+  submodules: true
+  target: null
+  update: []
+  vercode: '11'
+  version: 0.5.1
+- antcommands: []
+  build: ''
+  buildjni: []
+  commit: v0.5.2
+  disable: broken in upstream
+  encoding: null
+  extlibs: []
+  forcevercode: false
+  forceversion: false
+  gradle:
+  - 'yes'
+  gradleprops: []
+  init: ''
+  kivy: false
+  maven: false
+  ndk: null
+  novcheck: false
+  oldsdkloc: false
+  output: null
+  patch: []
+  preassemble: []
+  prebuild: touch signing.properties && ./scripts/build-witness.sh && rm -rf libs/gradle-witness/build
+  rm:
+  - libs/*.jar
+  scandelete: []
+  scanignore: []
+  srclibs: []
+  subdir: null
+  submodules: true
+  target: null
+  update: []
+  vercode: '12'
+  version: 0.5.2
+- antcommands: []
+  build: ''
+  buildjni: []
+  commit: v0.5.3
+  disable: false
+  encoding: null
+  extlibs: []
+  forcevercode: false
+  forceversion: false
+  gradle:
+  - 'yes'
+  gradleprops: []
+  init: ''
+  kivy: false
+  maven: false
+  ndk: null
+  novcheck: false
+  oldsdkloc: false
+  output: null
+  patch: []
+  preassemble: []
+  prebuild: touch signing.properties && ./scripts/build-witness.sh && rm -rf libs/gradle-witness/build
+  rm:
+  - libs/*.jar
+  scandelete: []
+  scanignore: []
+  srclibs: []
+  subdir: null
+  submodules: true
+  target: null
+  update: []
+  vercode: '100'
+  version: 0.5.3
+- antcommands: []
+  build: ''
+  buildjni: []
+  commit: v0.5.4
+  disable: false
+  encoding: null
+  extlibs: []
+  forcevercode: false
+  forceversion: false
+  gradle:
+  - 'yes'
+  gradleprops: []
+  init: ''
+  kivy: false
+  maven: false
+  ndk: null
+  novcheck: false
+  oldsdkloc: false
+  output: null
+  patch: []
+  preassemble: []
+  prebuild: touch signing.properties && ./scripts/build-witness.sh && rm -rf libs/gradle-witness/build
+  rm:
+  - libs/*.jar
+  scandelete: []
+  scanignore: []
+  srclibs: []
+  subdir: null
+  submodules: true
+  target: null
+  update: []
+  vercode: '101'
+  version: 0.5.4
+- antcommands: []
+  build: ''
+  buildjni: []
+  commit: v0.6.0
+  disable: false
+  encoding: null
+  extlibs: []
+  forcevercode: false
+  forceversion: false
+  gradle:
+  - 'yes'
+  gradleprops: []
+  init: ''
+  kivy: false
+  maven: false
+  ndk: null
+  novcheck: false
+  oldsdkloc: false
+  output: null
+  patch: []
+  preassemble: []
+  prebuild: touch signing.properties && ./scripts/build-witness.sh && rm -rf libs/gradle-witness/build
+  rm:
+  - libs/*.jar
+  scandelete: []
+  scanignore: []
+  srclibs: []
+  subdir: null
+  submodules: true
+  target: null
+  update: []
+  vercode: '102'
+  version: 0.6.0
+comments: {}
+id: org.smssecure.smssecure
+lastupdated: null
+metadatapath: metadata/org.smssecure.smssecure.txt

tests/metadata/net.osmand.plus.xml

@@ -1,180 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-
-<resources>
-  <string-array name="AntiFeatures">
-    <item>Tracking</item>
-    <item>NonFreeNet</item>
-  </string-array>
-
-  <string-array name="Categories">
-    <item>Navigation</item>
-  </string-array>
-
-  <string name="License">GPLv3</string>
-  <string name="Web Site">http://osmand.net</string>
-  <string name="Source Code">https://github.com/osmandapp/Osmand</string>
-  <string name="Issue Tracker">https://github.com/osmandapp/Osmand/issues</string>
-  <string name="Donate">https://code.google.com/p/osmand/#Please_support_the_project</string>
-
-  <string name="Name">OsmAnd~</string>
-  <string name="Summary">Offline/online maps and navigation</string>
-  <string name="Description">Osmand~'s features can be extended by enabling the plugins via the settings,
-which include online maps from many sources, tracking, OpenStreetMap (OSM) editing and
-accessibility enhancements.
-
-Map data of both vector and raster types can be stored on the phone memory
-card for offline usage, and navigation by default uses offline methods. Map
-data packages for many territories can be downloaded from within the app and
-there is a desktop program available on the website as well for creating your
-own.
-
-Anti-Features: Tracking - It will send your device and application specs to an
-Analytics server upon downloading the list of maps you can download.
-
-[https://osmandapp.github.io/changes.html Changelog]
-</string>
-
-  <string name="Repo Type">git</string>
-  <string name="Repo">https://github.com/mvdan/OsmAnd-submodules</string>
-<!--  <string name="Repo">https://github.com/osmandapp/Osmand</string/ -->
-
-<!--
-# Old builds with the old repo
-#Build:0.6.5,34
-#    commit=v0.6.5
-#    subdir=OsmAnd
-#    encoding=utf-8
-#    prebuild=mkdir assets && \
-#        mkdir raw
-#
-#Build:0.6.6,36
-#    commit=v0.6.6_2
-#    subdir=OsmAnd
-#    encoding=utf-8
-#    prebuild=mkdir raw
-#
-#Build:0.6.7,37
-#    commit=v0.6.7
-#    subdir=OsmAnd
-#    encoding=utf-8
-#    patch=code37.patch
-#    prebuild=mkdir raw
-#
-#Build:0.6.8,39
-#    commit=v0.6.8
-#    subdir=OsmAnd
-#    encoding=utf-8
-#    prebuild=mkdir raw
-#
-#Build:0.6.8',41
-#    disable=No corresponding source for whatever this is
-#    commit=unknown - see disabled
-#
-#Build:0.6.9,42
-#    commit=v0.6.9
-#    subdir=OsmAnd
-#    encoding=utf-8
-#    prebuild=mkdir raw
-#
-#Build:0.6.9',43
-#    disable=No corresponding source for whatever this is
-#    commit=unknown - see disabled
-#
-#Build:0.8.1,65
-#    commit=d62472532d8
-#    subdir=OsmAnd
-#    target=android-8
-#    init=rm -f build.xml
-#    encoding=utf-8
-#    forceversion=yes
-#    prebuild=cd ../DataExtractionOSM && \
-#        ant compile build && \
-#        cd ../OsmAnd/ && \
-#        cp ../DataExtractionOSM/build/OsmAndMapCreator.jar libs/ && \
-#        zip -d libs/OsmAndMapCreator.jar net/osmand/LogUtil.class && \
-#        cp -r ../DataExtractionOSM/build/lib/ libs/
-#    buildjni=no
-#
-#Build:0.8.2,71
-#    commit=50a4733475cd
-#    subdir=OsmAnd
-#    submodules=yes
-#    target=android-8
-#    init=rm -f build.xml
-#    encoding=utf-8
-#    forceversion=yes
-#    forcevercode=yes
-#    prebuild=cd ../DataExtractionOSM && \
-#        ant compile build && \
-#        cd ../OsmAnd/ && \
-#        sed -i 's/app_version">[^<]*/app_version">0.8.2-fdroid/' res/values/no_translate.xml && \
-#        cp ../DataExtractionOSM/build/OsmAndMapCreator.jar libs/ && \
-#        zip -d libs/OsmAndMapCreator.jar net/osmand/LogUtil.class && \
-#        cp -r ../DataExtractionOSM/build/lib/ libs/
-#    buildjni=yes
--->
-
-<builds>
-
-  <build>
-    <versionCode>182</versionCode>
-    <versionName>1.8.2</versionName>
-    <commit>76ada6c8a08afe69acb755503373ac36328ef665</commit>
-    <subdir>android/OsmAnd</subdir>
-    <submodules>true</submodules>
-    <output>bin/OsmAnd-release-unsigned.apk</output>
-    <prebuild>sed -i 's/"OsmAnd+"/"OsmAnd~"/g' build.xml</prebuild>
-    <build>./old-ndk-build.sh &amp;&amp; ant -Dsdk.dir="$ANDROID_SDK" -Dndk.dir="$ANDROID_NDK" -DBLACKBERRY_BUILD=false -DBUILD_SUFFIX= -DAPK_NUMBER_VERSION=182 "-DFEATURES=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" -DCLEAN_CPP=false -DPACKAGE_TO_BUILT=net.osmand.plus -DAPK_VERSION=1.8.2 -Dnet.osmand.plus= -Dbuild.version=1.8.2 -Dbuild.version.code=182 -Dnativeoff=false "-DversionFeatures=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" clean release</build>
-    <buildjni>no</buildjni>
-  </build>
-
-  <build>
-    <versionName>1.8.3</versionName>
-    <versionCode>183</versionCode>
-    <commit>1.8.3</commit>
-    <subdir>android/OsmAnd</subdir>
-    <submodules>true</submodules>
-    <output>bin/OsmAnd-release-unsigned.apk</output>
-    <build>../../build</build>
-    <buildjni>no</buildjni>
-  </build>
-
-  <build>
-    <versionName>1.9.4</versionName>
-    <versionCode>196</versionCode>
-    <commit>1.9.4</commit>
-    <subdir>android/OsmAnd</subdir>
-    <submodules>true</submodules>
-    <output>bin/OsmAnd-release-unsigned.apk</output>
-    <build>../../build</build>
-    <buildjni>no</buildjni>
-    <ndk>r10d</ndk>
-  </build>
-
-  <build>
-    <versionName>1.9.5</versionName>
-    <versionCode>197</versionCode>
-    <commit>1.9.5</commit>
-    <subdir>android/OsmAnd</subdir>
-    <submodules>true</submodules>
-    <output>bin/OsmAnd-release-unsigned.apk</output>
-    <build>../../build</build>
-    <buildjni>no</buildjni>
-    <ndk>r10d</ndk>
-  </build>
-
-</builds>
-
-  <string name="Maintainer Notes">
-No UCMs apply because git never contains actual releases, only pre-releses.
-
-The build instructions have been moved to a script in the root of the repo,
-'build'. This way it can be updated along with the submodules.
-  </string>
-
-  <string name="Auto Update Mode">None</string>
-  <string name="Update Check Mode">None</string>
-  <string name="Current Version">1.9.5</string>
-  <string name="Current Version Code">197</string>
-
-</resources>

tests/metadata/org.smssecure.smssecure.pickle

@@ -1,696 +0,0 @@
-(dp0
-S'Update Check Data'
-p1
-NsS'Bitcoin'
-p2
-NsS'AntiFeatures'
-p3
-(lp4
-sS'Litecoin'
-p5
-NsS'comments'
-p6
-(dp7
-sS'Provides'
-p8
-NsS'Issue Tracker'
-p9
-S'https://github.com/SMSSecure/SMSSecure/issues'
-p10
-sS'Donate'
-p11
-NsS'Archive Policy'
-p12
-NsS'Description'
-p13
-S"SMSSecure is an SMS/MMS application that allows you to protect your privacy while communicating with friends.\nUsing SMSSecure, you can send SMS messages and share media or attachments with complete privacy.\n\n* Easy. SMSSecure works like any other SMS application. There's nothing to sign up for and no new service your friends need to join.\n* Reliable. SMSSecure communicates using encrypted SMS messages. No servers or internet connection required.\n* Private. SMSSecure uses the TextSecure encryption protocol to provide privacy for every message, every time.\n* Safe. All messages are encrypted locally, so if your phone is lost or stolen, your messages are protected.\n* Open Source. SMSSecure is Free and Open Source, enabling anyone to verify its security by auditing the code."
-p14
-sS'Requires Root'
-p15
-I00
-sS'lastupdated'
-p16
-NsS'id'
-p17
-S'org.smssecure.smssecure'
-p18
-sS'Repo'
-p19
-S'https://github.com/SMSSecure/SMSSecure'
-p20
-sS'No Source Since'
-p21
-S''
-p22
-sS'Author Name'
-p23
-NsS'Repo Type'
-p24
-S'git'
-p25
-sS'Auto Name'
-p26
-S'SMSSecure'
-p27
-sS'Categories'
-p28
-(lp29
-S'Phone & SMS'
-p30
-asS'Source Code'
-p31
-S'https://github.com/SMSSecure/SMSSecure'
-p32
-sS'added'
-p33
-NsS'Update Check Ignore'
-p34
-NsS'Name'
-p35
-NsS'License'
-p36
-S'GPLv3'
-p37
-sS'Changelog'
-p38
-g22
-sS'Update Check Mode'
-p39
-S'Tags'
-p40
-sS'Summary'
-p41
-S'Send encrypted text messages (SMS)'
-p42
-sS'Current Version'
-p43
-S'0.6.0'
-p44
-sS'Author Email'
-p45
-NsS'Maintainer Notes'
-p46
-g22
-sS'Current Version Code'
-p47
-S'102'
-p48
-sS'Binaries'
-p49
-NsS'builds'
-p50
-(lp51
-(dp52
-S'submodules'
-p53
-I00
-sS'vercode'
-p54
-S'5'
-p55
-sS'forceversion'
-p56
-I00
-sS'oldsdkloc'
-p57
-I00
-sS'gradleprops'
-p58
-(lp59
-sS'kivy'
-p60
-I00
-sS'patch'
-p61
-(lp62
-sS'scanignore'
-p63
-(lp64
-sS'srclibs'
-p65
-(lp66
-S'GradleWitness@10f1269c0aafdc1d478efc005ed48f3a47d44278'
-p67
-aS'PreferenceFragment@717a45433b927d2f0dfc5328f79e77c9682c37bc'
-p68
-aS'ShortcutBadger@3815ce2ec0c66acd7d7c0b4f2479df8fa70fed87'
-p69
-aS'AospMms@android-5.1.0_r3'
-p70
-asS'encoding'
-p71
-NsS'extlibs'
-p72
-(lp73
-sS'init'
-p74
-g22
-sS'version'
-p75
-S'0.3.3'
-p76
-sS'build'
-p77
-g22
-sS'rm'
-p78
-(lp79
-S'libs/*'
-p80
-asS'subdir'
-p81
-NsS'forcevercode'
-p82
-I01
-sS'preassemble'
-p83
-(lp84
-sS'update'
-p85
-(lp86
-sS'maven'
-p87
-I00
-sS'disable'
-p88
-S'builds, merge changes into upstream'
-p89
-sS'output'
-p90
-NsS'scandelete'
-p91
-(lp92
-sS'buildjni'
-p93
-(lp94
-sS'ndk'
-p95
-NsS'target'
-p96
-NsS'antcommands'
-p97
-(lp98
-sS'gradle'
-p99
-(lp100
-S'yes'
-p101
-asS'prebuild'
-p102
-S"touch signing.properties && pushd $$GradleWitness$$ && gradle jar && popd && cp $$GradleWitness$$/build/libs/GradleWitness.jar libs/gradle-witness.jar && sed -i -e '20,22d' build.gradle && pushd $$PreferenceFragment$$ && gradle uploadArchives && popd && sed -i -e '/5470f5872514a6226fa1fc6f4e000991f38805691c534cf0bd2778911fc773ad/d' build.gradle && mkdir smil && pushd smil && wget -c http://www.w3.org/TR/smil-boston-dom/java-binding.zip && unzip java-binding.zip && popd && cp -fR smil/java/org src/ && rm -fR smil && sed -i -e '/org.w3c.smil/d' build.gradle && cp -fR $$AospMms$$/src/org src/"
-p103
-sS'novcheck'
-p104
-I00
-sS'commit'
-p105
-S'66367479a4f57f347b5cbe8f6f8f632adaae7727'
-p106
-sa(dp107
-g53
-I01
-sg54
-S'6'
-p108
-sg56
-I00
-sg57
-I00
-sg58
-(lp109
-sg60
-I00
-sg61
-(lp110
-sg63
-(lp111
-sg65
-(lp112
-S'GradleWitness@10f1269c0aafdc1d478efc005ed48f3a47d44278'
-p113
-asg71
-Nsg72
-(lp114
-sg74
-g22
-sg75
-S'0.3.3'
-p115
-sg77
-g22
-sg78
-(lp116
-S'libs/*.jar'
-p117
-asg81
-Nsg82
-I00
-sg83
-(lp118
-sg85
-(lp119
-sg87
-I00
-sg88
-S'builds, wait for upstream'
-p120
-sg90
-Nsg91
-(lp121
-sg93
-(lp122
-sg95
-Nsg96
-Nsg97
-(lp123
-sg99
-(lp124
-S'yes'
-p125
-asg102
-S'touch signing.properties && pushd $$GradleWitness$$ && gradle jar && popd && cp $$GradleWitness$$/build/libs/GradleWitness.jar libs/gradle-witness.jar'
-p126
-sg104
-I00
-sg105
-S'9675ce5eecb929dcaddb43b3d9486fdb88b9ae1a'
-p127
-sa(dp128
-g53
-I01
-sg54
-S'9'
-p129
-sg56
-I00
-sg57
-I00
-sg58
-(lp130
-sg60
-I00
-sg61
-(lp131
-sg63
-(lp132
-sg65
-(lp133
-sg71
-Nsg72
-(lp134
-sg74
-g22
-sg75
-S'0.4.2'
-p135
-sg77
-g22
-sg78
-(lp136
-S'libs/*.jar'
-p137
-asg81
-Nsg82
-I00
-sg83
-(lp138
-sg85
-(lp139
-sg87
-I00
-sg88
-S'builds locally, but not on BS'
-p140
-sg90
-Nsg91
-(lp141
-sg93
-(lp142
-sg95
-Nsg96
-Nsg97
-(lp143
-sg99
-(lp144
-S'yes'
-p145
-asg102
-S'touch signing.properties && ./build-witness.sh && rm -rf libs/gradle-witness/build && echo "org.gradle.jvmargs=-Xms512m -Xmx512m -XX:MaxPermSize=512m" >> gradle.properties'
-p146
-sg104
-I00
-sg105
-S'v0.4.2'
-p147
-sa(dp148
-g53
-I01
-sg54
-S'11'
-p149
-sg56
-I00
-sg57
-I00
-sg58
-(lp150
-sg60
-I00
-sg61
-(lp151
-sg63
-(lp152
-sg65
-(lp153
-sg71
-Nsg72
-(lp154
-sg74
-g22
-sg75
-S'0.5.1'
-p155
-sg77
-g22
-sg78
-(lp156
-S'libs/*.jar'
-p157
-asg81
-Nsg82
-I00
-sg83
-(lp158
-sg85
-(lp159
-sg87
-I00
-sg88
-I00
-sg90
-Nsg91
-(lp160
-sg93
-(lp161
-sg95
-Nsg96
-Nsg97
-(lp162
-sg99
-(lp163
-S'yes'
-p164
-asg102
-S'touch signing.properties && ./build-witness.sh && rm -rf libs/gradle-witness/build && echo "org.gradle.jvmargs=-Xms512m -Xmx512m -XX:MaxPermSize=512m" >> gradle.properties'
-p165
-sg104
-I00
-sg105
-S'v0.5.1'
-p166
-sa(dp167
-g53
-I01
-sg54
-S'12'
-p168
-sg56
-I00
-sg57
-I00
-sg58
-(lp169
-sg60
-I00
-sg61
-(lp170
-sg63
-(lp171
-sg65
-(lp172
-sg71
-Nsg72
-(lp173
-sg74
-g22
-sg75
-S'0.5.2'
-p174
-sg77
-g22
-sg78
-(lp175
-S'libs/*.jar'
-p176
-asg81
-Nsg82
-I00
-sg83
-(lp177
-sg85
-(lp178
-sg87
-I00
-sg88
-S'broken in upstream'
-p179
-sg90
-Nsg91
-(lp180
-sg93
-(lp181
-sg95
-Nsg96
-Nsg97
-(lp182
-sg99
-(lp183
-S'yes'
-p184
-asg102
-S'touch signing.properties && ./scripts/build-witness.sh && rm -rf libs/gradle-witness/build'
-p185
-sg104
-I00
-sg105
-S'v0.5.2'
-p186
-sa(dp187
-g53
-I01
-sg54
-S'100'
-p188
-sg56
-I00
-sg57
-I00
-sg58
-(lp189
-sg60
-I00
-sg61
-(lp190
-sg63
-(lp191
-sg65
-(lp192
-sg71
-Nsg72
-(lp193
-sg74
-g22
-sg75
-S'0.5.3'
-p194
-sg77
-g22
-sg78
-(lp195
-S'libs/*.jar'
-p196
-asg81
-Nsg82
-I00
-sg83
-(lp197
-sg85
-(lp198
-sg87
-I00
-sg88
-I00
-sg90
-Nsg91
-(lp199
-sg93
-(lp200
-sg95
-Nsg96
-Nsg97
-(lp201
-sg99
-(lp202
-S'yes'
-p203
-asg102
-S'touch signing.properties && ./scripts/build-witness.sh && rm -rf libs/gradle-witness/build'
-p204
-sg104
-I00
-sg105
-S'v0.5.3'
-p205
-sa(dp206
-g53
-I01
-sg54
-S'101'
-p207
-sg56
-I00
-sg57
-I00
-sg58
-(lp208
-sg60
-I00
-sg61
-(lp209
-sg63
-(lp210
-sg65
-(lp211
-sg71
-Nsg72
-(lp212
-sg74
-g22
-sg75
-S'0.5.4'
-p213
-sg77
-g22
-sg78
-(lp214
-S'libs/*.jar'
-p215
-asg81
-Nsg82
-I00
-sg83
-(lp216
-sg85
-(lp217
-sg87
-I00
-sg88
-I00
-sg90
-Nsg91
-(lp218
-sg93
-(lp219
-sg95
-Nsg96
-Nsg97
-(lp220
-sg99
-(lp221
-S'yes'
-p222
-asg102
-S'touch signing.properties && ./scripts/build-witness.sh && rm -rf libs/gradle-witness/build'
-p223
-sg104
-I00
-sg105
-S'v0.5.4'
-p224
-sa(dp225
-g53
-I01
-sg54
-S'102'
-p226
-sg56
-I00
-sg57
-I00
-sg58
-(lp227
-sg60
-I00
-sg61
-(lp228
-sg63
-(lp229
-sg65
-(lp230
-sg71
-Nsg72
-(lp231
-sg74
-g22
-sg75
-S'0.6.0'
-p232
-sg77
-g22
-sg78
-(lp233
-S'libs/*.jar'
-p234
-asg81
-Nsg82
-I00
-sg83
-(lp235
-sg85
-(lp236
-sg87
-I00
-sg88
-I00
-sg90
-Nsg91
-(lp237
-sg93
-(lp238
-sg95
-Nsg96
-Nsg97
-(lp239
-sg99
-(lp240
-S'yes'
-p241
-asg102
-S'touch signing.properties && ./scripts/build-witness.sh && rm -rf libs/gradle-witness/build'
-p242
-sg104
-I00
-sg105
-S'v0.6.0'
-p243
-sasS'FlattrID'
-p244
-NsS'metadatapath'
-p245
-S'metadata/org.smssecure.smssecure.txt'
-p246
-sS'Disabled'
-p247
-NsS'Web Site'
-p248
-S'http://www.smssecure.org'
-p249
-sS'Update Check Name'
-p250
-NsS'Vercode Operation'
-p251
-NsS'Auto Update Mode'
-p252
-S'Version v%v'
-p253
-s.

tests/run-tests

@@ -163,7 +163,7 @@ $fdroid init
 cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $REPOROOT/
 cp -a $WORKSPACE/tests/gnupghome $GNUPGHOME
 chmod 0700 $GNUPGHOME
-echo "accepted_formats = ['json', 'txt', 'xml', 'yml']" >> config.py
+echo "accepted_formats = ['json', 'txt', 'yml']" >> config.py
 echo "install_list = 'org.adaway'" >> config.py
 echo "uninstall_list = {'com.android.vending', 'com.facebook.orca',}" >> config.py
 echo "gpghome = '$GNUPGHOME'" >> config.py

tests/update.TestCase

@@ -91,7 +91,7 @@ class UpdateTest(unittest.TestCase):
         config = dict()
         fdroidserver.common.fill_config_defaults(config)
         config['ndk_paths'] = dict()
-        config['accepted_formats'] = ['json', 'txt', 'xml', 'yml']
+        config['accepted_formats'] = ['json', 'txt', 'yml']
         fdroidserver.common.config = config
         fdroidserver.update.config = config
 

wp-fdroid/wp-fdroid.php

@@ -695,6 +695,8 @@ class FDroid
 		$antifeatureDescription['UpstreamNonFree']['description'] = 'The upstream source code is non-free.';
 		$antifeatureDescription['NonFreeAssets']['name'] = 'Non-Free Assets';
 		$antifeatureDescription['NonFreeAssets']['description'] = 'This application contains non-free assets.';
+		$antifeatureDescription['KnownVuln']['name'] = 'Known Vulnerability';
+		$antifeatureDescription['KnownVuln']['description'] = 'This application known security vulnerabilities.';
 
 		if(isset($antifeatureDescription[$antifeature])) {
 			return $antifeatureDescription[$antifeature];