Merge branch 'extlib_scanignore' into 'master'

minor bugfixes

Closes #795, #796, and #759

See merge request fdroid/fdroidserver!771

fdroidserver/common.py

@@ -994,12 +994,14 @@ class vcs_git(vcs):
                 if p.returncode != 0:
                     lines = p.output.splitlines()
                     if 'Multiple remote HEAD branches' not in lines[0]:
-                        raise VCSException(_("Git remote set-head failed"), p.output)
+                        logging.warning(_("Git remote set-head failed: \"%s\"") % p.output.strip())
-                    branch = lines[1].split(' ')[-1]
+                    else:
-                    p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
+                        branch = lines[1].split(' ')[-1]
-                                     cwd=self.local, output=False)
+                        p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
-                    if p2.returncode != 0:
+                                         cwd=self.local, output=False)
-                        raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
+                        if p2.returncode != 0:
+                            logging.warning(_("Git remote set-head failed: \"%s\"")
+                                            % p.output.strip() + '\n' + p2.output.strip())
                 self.refreshed = True
         # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
         # a github repo. Most of the time this is the same as origin/master.
@@ -2111,6 +2113,13 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
             if not os.path.exists(libsrc):
                 raise BuildException("Missing extlib file {0}".format(libsrc))
             shutil.copyfile(libsrc, os.path.join(libsdir, libf))
+            # Add extlibs to scanignore (this is relative to the build dir root, *sigh*)
+            if build.subdir:
+                scanignorepath = os.path.join(build.subdir, 'libs', libf)
+            else:
+                scanignorepath = os.path.join('libs', libf)
+            if scanignorepath not in build.scanignore:
+                build.scanignore.append(scanignorepath)
 
     # Run a pre-build command if one is required
     if build.prebuild:

fdroidserver/index.py

@@ -134,7 +134,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
             return sorted(list(obj))
         if isinstance(obj, datetime):
             # Java prefers milliseconds
-            # we also need to accound for time zone/daylight saving time
+            # we also need to account for time zone/daylight saving time
             return int(calendar.timegm(obj.timetuple()) * 1000)
         if isinstance(obj, dict):
             d = collections.OrderedDict()

fdroidserver/scanner.py

@@ -62,7 +62,7 @@ def scan_binary(apkfile):
     usual_suspects = {
         # The `apkanalyzer dex packages` output looks like this:
         # M d 1   1       93      <packagename> <other stuff>
-        # The first column has P/C/M/F for package, class, methos or field
+        # The first column has P/C/M/F for package, class, method or field
         # The second column has x/k/r/d for removed, kept, referenced and defined.
         # We already filter for defined only in the apkanalyzer call. 'r' will be
         # for things referenced but not distributed in the apk.
@@ -177,6 +177,11 @@ def scan_source(build_dir, build=metadata.Build()):
         return False
 
     def ignoreproblem(what, path_in_build_dir):
+        """
+        :param what: string describing the problem, will be printed in log messages
+        :param path_in_build_dir: path to the file relative to `build`-dir
+        "returns: 0 as we explicitly ignore the file, so don't count an error
+        """
         msg = ('Ignoring %s at %s' % (what, path_in_build_dir))
         logging.info(msg)
         if json_per_build is not None:
@@ -184,14 +189,31 @@ def scan_source(build_dir, build=metadata.Build()):
         return 0
 
     def removeproblem(what, path_in_build_dir, filepath):
+        """
+        :param what: string describing the problem, will be printed in log messages
+        :param path_in_build_dir: path to the file relative to `build`-dir
+        :param filepath: Path (relative to our current path) to the file
+        "returns: 0 as we deleted the offending file
+        """
         msg = ('Removing %s at %s' % (what, path_in_build_dir))
         logging.info(msg)
         if json_per_build is not None:
             json_per_build['infos'].append([msg, path_in_build_dir])
-        os.remove(filepath)
+        try:
+            os.remove(filepath)
+        except FileNotFoundError:
+            # File is already gone, nothing to do.
+            # This can happen if we find multiple problems in one file that is setup for scandelete
+            # I.e. build.gradle files containig multiple unknown maven repos.
+            pass
         return 0
 
     def warnproblem(what, path_in_build_dir):
+        """
+        :param what: string describing the problem, will be printed in log messages
+        :param path_in_build_dir: path to the file relative to `build`-dir
+        :returns: 0, as warnings don't count as errors
+        """
         if toignore(path_in_build_dir):
             return 0
         logging.warning('Found %s at %s' % (what, path_in_build_dir))
@@ -200,6 +222,14 @@ def scan_source(build_dir, build=metadata.Build()):
         return 0
 
     def handleproblem(what, path_in_build_dir, filepath):
+        """Dispatches to problem handlers (ignore, delete, warn) or returns 1
+         for increasing the error count
+
+        :param what: string describing the problem, will be printed in log messages
+        :param path_in_build_dir: path to the file relative to `build`-dir
+        :param filepath: Path (relative to our current path) to the file
+        :returns: 0 if the problem was ignored/deleted/is only a warning, 1 otherwise
+        """
         if toignore(path_in_build_dir):
             return ignoreproblem(what, path_in_build_dir)
         if todelete(path_in_build_dir):

fdroidserver/update.py

@@ -1974,12 +1974,6 @@ def apply_info_from_latest_apk(apps, apks):
                 if app.NoSourceSince:
                     apk['antiFeatures'].add('NoSourceSince')
 
-                if 'added' in apk:
-                    if not app.added or apk['added'] < app.added:
-                        app.added = apk['added']
-                    if not app.lastUpdated or apk['added'] > app.lastUpdated:
-                        app.lastUpdated = apk['added']
-
         if not app.added:
             logging.debug("Don't know when " + appid + " was added")
         if not app.lastUpdated:
@@ -2166,6 +2160,27 @@ def create_metadata_from_template(apk):
     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
 
 
+def read_added_date_from_all_apks(apps, apks):
+    """
+    Added dates come from the stats/known_apks.txt file but are
+    read when scanning apks and thus need to be applied form apk
+    level to app level for _all_ apps and not only from non-archived
+    ones
+
+    TODO: read the added dates directly from known_apks.txt instead of
+          going through apks that way it also works for for repos that
+          don't keep an archive of apks.
+    """
+    for appid, app in apps.items():
+        for apk in apks:
+            if apk['packageName'] == appid:
+                if 'added' in apk:
+                    if not app.added or apk['added'] < app.added:
+                        app.added = apk['added']
+                    if not app.lastUpdated or apk['added'] > app.lastUpdated:
+                        app.lastUpdated = apk['added']
+
+
 def read_names_from_apks(apps, apks):
     """This is a stripped down copy of apply_info_from_latest_apk that only parses app names"""
     for appid, app in apps.items():
@@ -2384,6 +2399,10 @@ def main():
     # This will be done again (as part of apply_info_from_latest_apk) for repo and archive
     # separately later on, but it's fairly cheap anyway.
     read_names_from_apks(apps, apks + archapks)
+    # The added date currently comes from the oldest apk which might be in the archive.
+    # So we need this populated at app level before continuing with only processing /repo
+    # or /archive
+    read_added_date_from_all_apks(apps, apks + archapks)
 
     if len(repodirs) > 1:
         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])

tests/build.TestCase

@@ -23,6 +23,7 @@ if localmodule not in sys.path:
 import fdroidserver.build
 import fdroidserver.common
 import fdroidserver.metadata
+import fdroidserver.scanner
 
 
 class BuildTest(unittest.TestCase):
@@ -198,6 +199,50 @@ class BuildTest(unittest.TestCase):
         self.assertFalse(os.path.exists('gen'))
         self.assertFalse(os.path.exists('gradle-wrapper.jar'))
 
+    def test_scan_with_extlib(self):
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        os.chdir(testdir)
+        os.mkdir("build")
+
+        config = dict()
+        config['sdk_path'] = os.getenv('ANDROID_HOME')
+        config['ndk_paths'] = {'r10d': os.getenv('ANDROID_NDK_HOME')}
+        config['build_tools'] = 'FAKE_BUILD_TOOLS_VERSION'
+        fdroidserver.common.config = config
+        app = fdroidserver.metadata.App()
+        app.id = 'com.gpl.rpg.AndorsTrail'
+        build = fdroidserver.metadata.Build()
+        build.commit = 'master'
+        build.androidupdate = ['no']
+        os.makedirs("extlib/android")
+        # write a fake binary jar file the scanner should definitely error on
+        with open('extlib/android/android-support-v4r11.jar', 'wb') as file:
+            file.write(b'PK\x03\x04\x14\x00\x08\x00\x08\x00-\x0eiA\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\x00\x04\x00META-INF/\xfe\xca\x00\x00\x03\x00PK\x07\x08\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00')
+
+        class FakeVcs():
+            # no need to change to the correct commit here
+            def gotorevision(self, rev, refresh=True):
+                pass
+
+            def getsrclib(self):
+                return None
+
+        # Test we trigger a scanner error without extlibs
+        build.extlibs = []
+        os.makedirs('build/libs')
+        shutil.copy('extlib/android/android-support-v4r11.jar', 'build/libs')
+        fdroidserver.common.prepare_source(FakeVcs(), app, build,
+                                           "build", "ignore", "extlib")
+        count = fdroidserver.scanner.scan_source("build", build)
+        self.assertEqual(1, count, "Should produce a scanner error without extlib")
+
+        # Now try again as an extlib
+        build.extlibs = ['android/android-support-v4r11.jar']
+        fdroidserver.common.prepare_source(FakeVcs(), app, build,
+                                           "build", "ignore", "extlib")
+        count = fdroidserver.scanner.scan_source("build", build)
+        self.assertEqual(0, count, "Shouldn't error on jar from extlib")
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/run-tests

@@ -607,6 +607,29 @@ grep -F 'info.guardianproject.urzip_100.apk' repo/index-v1.json repo/index.xml
 grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index-v1.json
 ! grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index.xml
 
+#------------------------------------------------------------------------------#
+echo_header "test for added date being set correctly for repo and archive"
+REPOROOT=`create_test_dir`
+cd $REPOROOT
+fdroid_init_with_prebuilt_keystore
+mkdir -p {repo,archive,metadata,stats}
+cp $WORKSPACE/tests/repo/com.politedroid_5.apk archive
+cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo
+cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata
+#TODO: the timestamp of the oldest apk in the file should be used, even if that
+# doesn't exist anymore
+echo "com.politedroid_4.apk com.politedroid 2016-01-01" > stats/known_apks.txt
+echo "com.politedroid_5.apk com.politedroid 2017-01-01" >> stats/known_apks.txt
+echo "com.politedroid_6.apk com.politedroid 2018-01-01" >> stats/known_apks.txt
+sed -i -e 's/ArchivePolicy:.*/ArchivePolicy: 1 versions/' metadata/com.politedroid.yml
+# Get the java ms timestamp from UTC time
+timestamp=$(date -u --date=2017-01-01 +%s)000
+
+$fdroid update --pretty --nosign
+grep -F "\"added\": $timestamp" repo/index-v1.json
+# the archive will have the added timestamp for the app and for the apk, both need to be there
+if [ $(grep -F "\"added\": $timestamp" archive/index-v1.json | wc -l) == 2 ]; then true; else false;fi
+
 #------------------------------------------------------------------------------#
 echo_header "test whatsnew from fastlane without CVC set"
 REPOROOT=`create_test_dir`

tests/scanner.TestCase

@@ -270,6 +270,27 @@ class ScannerTest(unittest.TestCase):
             self.assertTrue(found, 'this block should produce a URL:\n' + entry)
         self.assertEqual(len(data), len(urls), 'each data example should produce a URL')
 
+    def test_scan_gradle_file_with_multiple_problems(self):
+        """Check that the scanner can handle scandelete with gradle files with multiple problems"""
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        os.chdir(testdir)
+        fdroidserver.scanner.config = None
+        fdroidserver.scanner.options = mock.Mock()
+        build = fdroidserver.metadata.Build()
+        build.scandelete = ['build.gradle']
+        with open('build.gradle', 'w') as fp:
+            fp.write(textwrap.dedent("""
+                 maven {
+                    url 'https://maven.fabric.io/public'
+                 }
+                 maven {
+                    url 'https://evilcorp.com/maven'
+                 }
+            """))
+        count = fdroidserver.scanner.scan_source(testdir, build)
+        self.assertFalse(os.path.exists("build.gradle"))
+        self.assertEqual(0, count, 'there should be this many errors')
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/vcs.TestCase

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
+
+import inspect
+import logging
+import optparse
+import os
+import sys
+import tempfile
+import unittest
+
+from git import Repo
+
+localmodule = os.path.realpath(
+    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
+print('localmodule: ' + localmodule)
+if localmodule not in sys.path:
+    sys.path.insert(0, localmodule)
+
+import fdroidserver.build
+import fdroidserver.common
+import fdroidserver.metadata
+import fdroidserver.scanner
+
+
+class VCSTest(unittest.TestCase):
+    """For some reason the VCS classes are in fdroidserver/common.py"""
+
+    def setUp(self):
+        logging.basicConfig(level=logging.DEBUG)
+        self.basedir = os.path.join(localmodule, 'tests')
+        self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
+        if not os.path.exists(self.tmpdir):
+            os.makedirs(self.tmpdir)
+        os.chdir(self.basedir)
+
+    def test_remote_set_head_can_fail(self):
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        os.chdir(testdir)
+        # First create an upstream repo with one commit
+        upstream_repo = Repo.init("upstream_repo")
+        with open(upstream_repo.working_dir + "/file", 'w') as f:
+            f.write("Hello World!")
+        upstream_repo.index.add([upstream_repo.working_dir + "/file"])
+        upstream_repo.index.commit("initial commit")
+        commitid = upstream_repo.head.commit.hexsha
+
+        # Now clone it once manually, like gitlab runner gitlab-runner sets up a repo during CI
+        clone1 = Repo.init("clone1")
+        clone1.create_remote("upstream", "file://" + upstream_repo.working_dir)
+        clone1.remote("upstream").fetch()
+        clone1.head.reference = clone1.commit(commitid)
+        clone1.head.reset(index=True, working_tree=True)
+        self.assertTrue(clone1.head.is_detached)
+
+        # and now we want to use this clone as a source repo for fdroid build
+        config = {}
+        os.mkdir("build")
+        config['sdk_path'] = os.getenv('ANDROID_HOME')
+        config['ndk_paths'] = {'r10d': os.getenv('ANDROID_NDK_HOME')}
+        config['build_tools'] = 'FAKE_BUILD_TOOLS_VERSION'
+        config['java_paths'] = {'fake': 'fake'}
+        fdroidserver.common.config = config
+        app = fdroidserver.metadata.App()
+        app.RepoType = 'git'
+        app.Repo = clone1.working_dir
+        app.id = 'com.gpl.rpg.AndorsTrail'
+        build = fdroidserver.metadata.Build()
+        build.commit = commitid
+        build.androidupdate = ['no']
+        vcs, build_dir = fdroidserver.common.setup_vcs(app)
+        # force an init of the repo, the remote head error only occurs on the second gotorevision call
+        vcs.gotorevision(build.commit)
+        fdroidserver.common.prepare_source(vcs, app, build,
+                                           build_dir=build_dir, srclib_dir="ignore", extlib_dir="ignore")
+        self.assertTrue(os.path.isfile("build/com.gpl.rpg.AndorsTrail/file"))
+
+
+if __name__ == "__main__":
+    os.chdir(os.path.dirname(__file__))
+
+    parser = optparse.OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    (fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
+
+    newSuite = unittest.TestSuite()
+    newSuite.addTest(unittest.makeSuite(VCSTest))
+    unittest.main(failfast=False)