diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 5f057e40..dd259963 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -994,12 +994,14 @@ class vcs_git(vcs): if p.returncode != 0: lines = p.output.splitlines() if 'Multiple remote HEAD branches' not in lines[0]: - raise VCSException(_("Git remote set-head failed"), p.output) - branch = lines[1].split(' ')[-1] - p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch], - cwd=self.local, output=False) - if p2.returncode != 0: - raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output) + logging.warning(_("Git remote set-head failed: \"%s\"") % p.output.strip()) + else: + branch = lines[1].split(' ')[-1] + p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch], + cwd=self.local, output=False) + if p2.returncode != 0: + logging.warning(_("Git remote set-head failed: \"%s\"") + % p.output.strip() + '\n' + p2.output.strip()) self.refreshed = True # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on # a github repo. Most of the time this is the same as origin/master. @@ -2111,6 +2113,13 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= if not os.path.exists(libsrc): raise BuildException("Missing extlib file {0}".format(libsrc)) shutil.copyfile(libsrc, os.path.join(libsdir, libf)) + # Add extlibs to scanignore (this is relative to the build dir root, *sigh*) + if build.subdir: + scanignorepath = os.path.join(build.subdir, 'libs', libf) + else: + scanignorepath = os.path.join('libs', libf) + if scanignorepath not in build.scanignore: + build.scanignore.append(scanignorepath) # Run a pre-build command if one is required if build.prebuild: diff --git a/fdroidserver/index.py b/fdroidserver/index.py index b115643b..7e76f299 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -134,7 +134,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_ return sorted(list(obj)) if isinstance(obj, datetime): # Java prefers milliseconds - # we also need to accound for time zone/daylight saving time + # we also need to account for time zone/daylight saving time return int(calendar.timegm(obj.timetuple()) * 1000) if isinstance(obj, dict): d = collections.OrderedDict() diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 7e589e45..4b0c4be0 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -62,7 +62,7 @@ def scan_binary(apkfile): usual_suspects = { # The `apkanalyzer dex packages` output looks like this: # M d 1 1 93 - # The first column has P/C/M/F for package, class, methos or field + # The first column has P/C/M/F for package, class, method or field # The second column has x/k/r/d for removed, kept, referenced and defined. # We already filter for defined only in the apkanalyzer call. 'r' will be # for things referenced but not distributed in the apk. @@ -177,6 +177,11 @@ def scan_source(build_dir, build=metadata.Build()): return False def ignoreproblem(what, path_in_build_dir): + """ + :param what: string describing the problem, will be printed in log messages + :param path_in_build_dir: path to the file relative to `build`-dir + "returns: 0 as we explicitly ignore the file, so don't count an error + """ msg = ('Ignoring %s at %s' % (what, path_in_build_dir)) logging.info(msg) if json_per_build is not None: @@ -184,14 +189,31 @@ def scan_source(build_dir, build=metadata.Build()): return 0 def removeproblem(what, path_in_build_dir, filepath): + """ + :param what: string describing the problem, will be printed in log messages + :param path_in_build_dir: path to the file relative to `build`-dir + :param filepath: Path (relative to our current path) to the file + "returns: 0 as we deleted the offending file + """ msg = ('Removing %s at %s' % (what, path_in_build_dir)) logging.info(msg) if json_per_build is not None: json_per_build['infos'].append([msg, path_in_build_dir]) - os.remove(filepath) + try: + os.remove(filepath) + except FileNotFoundError: + # File is already gone, nothing to do. + # This can happen if we find multiple problems in one file that is setup for scandelete + # I.e. build.gradle files containig multiple unknown maven repos. + pass return 0 def warnproblem(what, path_in_build_dir): + """ + :param what: string describing the problem, will be printed in log messages + :param path_in_build_dir: path to the file relative to `build`-dir + :returns: 0, as warnings don't count as errors + """ if toignore(path_in_build_dir): return 0 logging.warning('Found %s at %s' % (what, path_in_build_dir)) @@ -200,6 +222,14 @@ def scan_source(build_dir, build=metadata.Build()): return 0 def handleproblem(what, path_in_build_dir, filepath): + """Dispatches to problem handlers (ignore, delete, warn) or returns 1 + for increasing the error count + + :param what: string describing the problem, will be printed in log messages + :param path_in_build_dir: path to the file relative to `build`-dir + :param filepath: Path (relative to our current path) to the file + :returns: 0 if the problem was ignored/deleted/is only a warning, 1 otherwise + """ if toignore(path_in_build_dir): return ignoreproblem(what, path_in_build_dir) if todelete(path_in_build_dir): diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 3ddf602c..1ddbd8e8 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1974,12 +1974,6 @@ def apply_info_from_latest_apk(apps, apks): if app.NoSourceSince: apk['antiFeatures'].add('NoSourceSince') - if 'added' in apk: - if not app.added or apk['added'] < app.added: - app.added = apk['added'] - if not app.lastUpdated or apk['added'] > app.lastUpdated: - app.lastUpdated = apk['added'] - if not app.added: logging.debug("Don't know when " + appid + " was added") if not app.lastUpdated: @@ -2166,6 +2160,27 @@ def create_metadata_from_template(apk): logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName'])) +def read_added_date_from_all_apks(apps, apks): + """ + Added dates come from the stats/known_apks.txt file but are + read when scanning apks and thus need to be applied form apk + level to app level for _all_ apps and not only from non-archived + ones + + TODO: read the added dates directly from known_apks.txt instead of + going through apks that way it also works for for repos that + don't keep an archive of apks. + """ + for appid, app in apps.items(): + for apk in apks: + if apk['packageName'] == appid: + if 'added' in apk: + if not app.added or apk['added'] < app.added: + app.added = apk['added'] + if not app.lastUpdated or apk['added'] > app.lastUpdated: + app.lastUpdated = apk['added'] + + def read_names_from_apks(apps, apks): """This is a stripped down copy of apply_info_from_latest_apk that only parses app names""" for appid, app in apps.items(): @@ -2384,6 +2399,10 @@ def main(): # This will be done again (as part of apply_info_from_latest_apk) for repo and archive # separately later on, but it's fairly cheap anyway. read_names_from_apks(apps, apks + archapks) + # The added date currently comes from the oldest apk which might be in the archive. + # So we need this populated at app level before continuing with only processing /repo + # or /archive + read_added_date_from_all_apks(apps, apks + archapks) if len(repodirs) > 1: archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older']) diff --git a/tests/build.TestCase b/tests/build.TestCase index 183f6661..25655229 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -23,6 +23,7 @@ if localmodule not in sys.path: import fdroidserver.build import fdroidserver.common import fdroidserver.metadata +import fdroidserver.scanner class BuildTest(unittest.TestCase): @@ -198,6 +199,50 @@ class BuildTest(unittest.TestCase): self.assertFalse(os.path.exists('gen')) self.assertFalse(os.path.exists('gradle-wrapper.jar')) + def test_scan_with_extlib(self): + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + os.mkdir("build") + + config = dict() + config['sdk_path'] = os.getenv('ANDROID_HOME') + config['ndk_paths'] = {'r10d': os.getenv('ANDROID_NDK_HOME')} + config['build_tools'] = 'FAKE_BUILD_TOOLS_VERSION' + fdroidserver.common.config = config + app = fdroidserver.metadata.App() + app.id = 'com.gpl.rpg.AndorsTrail' + build = fdroidserver.metadata.Build() + build.commit = 'master' + build.androidupdate = ['no'] + os.makedirs("extlib/android") + # write a fake binary jar file the scanner should definitely error on + with open('extlib/android/android-support-v4r11.jar', 'wb') as file: + file.write(b'PK\x03\x04\x14\x00\x08\x00\x08\x00-\x0eiA\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\x00\x04\x00META-INF/\xfe\xca\x00\x00\x03\x00PK\x07\x08\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00') + + class FakeVcs(): + # no need to change to the correct commit here + def gotorevision(self, rev, refresh=True): + pass + + def getsrclib(self): + return None + + # Test we trigger a scanner error without extlibs + build.extlibs = [] + os.makedirs('build/libs') + shutil.copy('extlib/android/android-support-v4r11.jar', 'build/libs') + fdroidserver.common.prepare_source(FakeVcs(), app, build, + "build", "ignore", "extlib") + count = fdroidserver.scanner.scan_source("build", build) + self.assertEqual(1, count, "Should produce a scanner error without extlib") + + # Now try again as an extlib + build.extlibs = ['android/android-support-v4r11.jar'] + fdroidserver.common.prepare_source(FakeVcs(), app, build, + "build", "ignore", "extlib") + count = fdroidserver.scanner.scan_source("build", build) + self.assertEqual(0, count, "Shouldn't error on jar from extlib") + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) diff --git a/tests/run-tests b/tests/run-tests index a5435351..551c5f55 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -607,6 +607,29 @@ grep -F 'info.guardianproject.urzip_100.apk' repo/index-v1.json repo/index.xml grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index-v1.json ! grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index.xml +#------------------------------------------------------------------------------# +echo_header "test for added date being set correctly for repo and archive" +REPOROOT=`create_test_dir` +cd $REPOROOT +fdroid_init_with_prebuilt_keystore +mkdir -p {repo,archive,metadata,stats} +cp $WORKSPACE/tests/repo/com.politedroid_5.apk archive +cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo +cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata +#TODO: the timestamp of the oldest apk in the file should be used, even if that +# doesn't exist anymore +echo "com.politedroid_4.apk com.politedroid 2016-01-01" > stats/known_apks.txt +echo "com.politedroid_5.apk com.politedroid 2017-01-01" >> stats/known_apks.txt +echo "com.politedroid_6.apk com.politedroid 2018-01-01" >> stats/known_apks.txt +sed -i -e 's/ArchivePolicy:.*/ArchivePolicy: 1 versions/' metadata/com.politedroid.yml +# Get the java ms timestamp from UTC time +timestamp=$(date -u --date=2017-01-01 +%s)000 + +$fdroid update --pretty --nosign +grep -F "\"added\": $timestamp" repo/index-v1.json +# the archive will have the added timestamp for the app and for the apk, both need to be there +if [ $(grep -F "\"added\": $timestamp" archive/index-v1.json | wc -l) == 2 ]; then true; else false;fi + #------------------------------------------------------------------------------# echo_header "test whatsnew from fastlane without CVC set" REPOROOT=`create_test_dir` diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index 2dbfbbba..3c18becc 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -270,6 +270,27 @@ class ScannerTest(unittest.TestCase): self.assertTrue(found, 'this block should produce a URL:\n' + entry) self.assertEqual(len(data), len(urls), 'each data example should produce a URL') + def test_scan_gradle_file_with_multiple_problems(self): + """Check that the scanner can handle scandelete with gradle files with multiple problems""" + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + fdroidserver.scanner.config = None + fdroidserver.scanner.options = mock.Mock() + build = fdroidserver.metadata.Build() + build.scandelete = ['build.gradle'] + with open('build.gradle', 'w') as fp: + fp.write(textwrap.dedent(""" + maven { + url 'https://maven.fabric.io/public' + } + maven { + url 'https://evilcorp.com/maven' + } + """)) + count = fdroidserver.scanner.scan_source(testdir, build) + self.assertFalse(os.path.exists("build.gradle")) + self.assertEqual(0, count, 'there should be this many errors') + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) diff --git a/tests/vcs.TestCase b/tests/vcs.TestCase new file mode 100755 index 00000000..ffed7b89 --- /dev/null +++ b/tests/vcs.TestCase @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + +# http://www.drdobbs.com/testing/unit-testing-with-python/240165163 + +import inspect +import logging +import optparse +import os +import sys +import tempfile +import unittest + +from git import Repo + +localmodule = os.path.realpath( + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) +print('localmodule: ' + localmodule) +if localmodule not in sys.path: + sys.path.insert(0, localmodule) + +import fdroidserver.build +import fdroidserver.common +import fdroidserver.metadata +import fdroidserver.scanner + + +class VCSTest(unittest.TestCase): + """For some reason the VCS classes are in fdroidserver/common.py""" + + def setUp(self): + logging.basicConfig(level=logging.DEBUG) + self.basedir = os.path.join(localmodule, 'tests') + self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles')) + if not os.path.exists(self.tmpdir): + os.makedirs(self.tmpdir) + os.chdir(self.basedir) + + def test_remote_set_head_can_fail(self): + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + # First create an upstream repo with one commit + upstream_repo = Repo.init("upstream_repo") + with open(upstream_repo.working_dir + "/file", 'w') as f: + f.write("Hello World!") + upstream_repo.index.add([upstream_repo.working_dir + "/file"]) + upstream_repo.index.commit("initial commit") + commitid = upstream_repo.head.commit.hexsha + + # Now clone it once manually, like gitlab runner gitlab-runner sets up a repo during CI + clone1 = Repo.init("clone1") + clone1.create_remote("upstream", "file://" + upstream_repo.working_dir) + clone1.remote("upstream").fetch() + clone1.head.reference = clone1.commit(commitid) + clone1.head.reset(index=True, working_tree=True) + self.assertTrue(clone1.head.is_detached) + + # and now we want to use this clone as a source repo for fdroid build + config = {} + os.mkdir("build") + config['sdk_path'] = os.getenv('ANDROID_HOME') + config['ndk_paths'] = {'r10d': os.getenv('ANDROID_NDK_HOME')} + config['build_tools'] = 'FAKE_BUILD_TOOLS_VERSION' + config['java_paths'] = {'fake': 'fake'} + fdroidserver.common.config = config + app = fdroidserver.metadata.App() + app.RepoType = 'git' + app.Repo = clone1.working_dir + app.id = 'com.gpl.rpg.AndorsTrail' + build = fdroidserver.metadata.Build() + build.commit = commitid + build.androidupdate = ['no'] + vcs, build_dir = fdroidserver.common.setup_vcs(app) + # force an init of the repo, the remote head error only occurs on the second gotorevision call + vcs.gotorevision(build.commit) + fdroidserver.common.prepare_source(vcs, app, build, + build_dir=build_dir, srclib_dir="ignore", extlib_dir="ignore") + self.assertTrue(os.path.isfile("build/com.gpl.rpg.AndorsTrail/file")) + + +if __name__ == "__main__": + os.chdir(os.path.dirname(__file__)) + + parser = optparse.OptionParser() + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + (fdroidserver.common.options, args) = parser.parse_args(['--verbose']) + + newSuite = unittest.TestSuite() + newSuite.addTest(unittest.makeSuite(VCSTest)) + unittest.main(failfast=False)