diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 5aa2d7c6..281ad523 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1100,9 +1100,10 @@ def main(): extlib_dir = os.path.join(build_dir, 'extlib') # Read all app and srclib metadata - allapps = metadata.read_metadata(xref=not options.onserver) - + pkgs = common.read_pkg_args(options.appid, True) + allapps = metadata.read_metadata(not options.onserver, pkgs) apps = common.read_app_args(options.appid, allapps, True) + for appid, app in list(apps.items()): if (app.Disabled and not options.force) or not app.RepoType or not app.builds: del apps[appid] @@ -1141,16 +1142,7 @@ def main(): # the source repo. We can reuse it on subsequent builds, if # there are any. if first: - if app.RepoType == 'srclib': - build_dir = os.path.join('build', 'srclib', app.Repo) - else: - build_dir = os.path.join('build', appid) - - # Set up vcs interface and make sure we have the latest code... - logging.debug("Getting {0} vcs interface for {1}" - .format(app.RepoType, app.Repo)) - vcs = common.getvcs(app.RepoType, app.Repo, build_dir) - + vcs, build_dir = common.setup_vcs(app) first = False logging.debug("Checking " + build.version) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index a023c57d..121f6edd 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -486,6 +486,27 @@ def getcvname(app): return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode) +def get_build_dir(app): + '''get the dir that this app will be built in''' + + if app.RepoType == 'srclib': + return os.path.join('build', 'srclib', app.Repo) + + return os.path.join('build', app.id) + + +def setup_vcs(app): + '''checkout code from VCS and return instance of vcs and the build dir''' + build_dir = get_build_dir(app) + + # Set up vcs interface and make sure we have the latest code... + logging.debug("Getting {0} vcs interface for {1}" + .format(app.RepoType, app.Repo)) + vcs = getvcs(app.RepoType, app.Repo, build_dir) + + return vcs, build_dir + + def getvcs(vcstype, remote, local): if vcstype == 'git': return vcs_git(remote, local) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index f8dc2b5c..986d240a 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -224,6 +224,21 @@ class App(): else: self.set_field(f, v) + def update(self, d): + '''Like dict.update()''' + for k, v in d.__dict__.items(): + if k == '_modified': + continue + elif k == 'builds': + for b in v: + build = Build() + del(b.__dict__['_modified']) + build.update_flags(b.__dict__) + self.builds.append(build) + elif v: + self.__dict__[k] = v + self._modified.add(k) + TYPE_UNKNOWN = 0 TYPE_OBSOLETE = 1 @@ -778,10 +793,12 @@ def read_srclibs(): srclibs[srclibname] = parse_srclib(metadatapath) -def read_metadata(xref=True): +def read_metadata(xref=True, check_vcs=[]): """ Read all metadata. Returns a list of 'app' objects (which are dictionaries as returned by the parse_txt_metadata function. + + check_vcs is the list of packageNames to check for .fdroid.yml in source """ # Always read the srclibs before the apps, since they can use a srlib as @@ -809,7 +826,7 @@ def read_metadata(xref=True): packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath)) if packageName in apps: warn_or_exception("Found multiple metadata files for " + packageName) - app = parse_metadata(metadatapath) + app = parse_metadata(metadatapath, packageName in check_vcs) check_metadata(app) apps[app.id] = app @@ -962,7 +979,9 @@ def _decode_bool(s): warn_or_exception("Invalid bool '%s'" % s) -def parse_metadata(metadatapath): +def parse_metadata(metadatapath, check_vcs=False): + '''parse metadata file, optionally checking the git repo for metadata first''' + _, ext = fdroidserver.common.get_extension(metadatapath) accepted = fdroidserver.common.config['accepted_formats'] if ext not in accepted: @@ -971,7 +990,11 @@ def parse_metadata(metadatapath): app = App() app.metadatapath = metadatapath - app.id, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath)) + name, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath)) + if name == '.fdroid': + check_vcs = False + else: + app.id = name with open(metadatapath, 'r', encoding='utf-8') as mf: if ext == 'txt': @@ -985,7 +1008,28 @@ def parse_metadata(metadatapath): else: warn_or_exception('Unknown metadata format: %s' % metadatapath) + if check_vcs and app.Repo: + build_dir = fdroidserver.common.get_build_dir(app) + metadata_in_repo = os.path.join(build_dir, '.fdroid.yml') + if not os.path.isfile(metadata_in_repo): + vcs, build_dir = fdroidserver.common.setup_vcs(app) + vcs.gotorevision('HEAD') # HEAD since we can't know where else to go + if os.path.isfile(metadata_in_repo): + logging.debug('Including metadata from ' + metadata_in_repo) + app.update(parse_metadata(metadata_in_repo)) + post_metadata_parse(app) + + if not app.id: + if app.builds: + build = app.builds[-1] + if build.subdir: + root_dir = build.subdir + else: + root_dir = '.' + paths = fdroidserver.common.manifest_paths(root_dir, build.gradle) + _, _, app.id = fdroidserver.common.parse_androidmanifests(paths, app) + return app diff --git a/tests/import.TestCase b/tests/import.TestCase index 8eef5acb..3cfdb92e 100755 --- a/tests/import.TestCase +++ b/tests/import.TestCase @@ -25,6 +25,7 @@ class ImportTest(unittest.TestCase): '''fdroid import''' def test_import_gitlab(self): + os.chdir(os.path.dirname(__file__)) # FDroidPopen needs some config to work config = dict() fdroidserver.common.fill_config_defaults(config) diff --git a/tests/metadata/info.guardianproject.checkey.txt b/tests/metadata/info.guardianproject.checkey.txt new file mode 100644 index 00000000..43faaade --- /dev/null +++ b/tests/metadata/info.guardianproject.checkey.txt @@ -0,0 +1,21 @@ +Categories:Development,GuardianProject +License:GPLv3 +Web Site:https://dev.guardianproject.info/projects/checkey +Source Code:https://github.com/guardianproject/checkey +Issue Tracker:https://dev.guardianproject.info/projects/checkey/issues +Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk + +Auto Name:Checkey +Summary:Info on local apps +Description: +Checkey is a utility for getting information about the APKs that are installed +on your device. Starting with a list of all of the apps that you have +installed on your device, it will show you the APK signature with a single +touch, and provides links to virustotal.com and androidobservatory.org to +easily access the profiles of that APK. It will also let you export the +signing certificate and generate ApkSignaturePin pin files for use with the +TrustedIntents library. +. + + +Current Version Code:9999999 diff --git a/tests/metadata/org.fdroid.ci.test.app.txt b/tests/metadata/org.fdroid.ci.test.app.txt new file mode 100644 index 00000000..466b7af6 --- /dev/null +++ b/tests/metadata/org.fdroid.ci.test.app.txt @@ -0,0 +1,2 @@ +Repo Type:git +Repo:https://gitlab.com/eighthave/ci-test-app diff --git a/tests/metadata/raw.template.txt b/tests/metadata/raw.template.txt new file mode 100644 index 00000000..ec2fe286 --- /dev/null +++ b/tests/metadata/raw.template.txt @@ -0,0 +1,9 @@ +License:Unknown +Web Site: +Source Code: +Issue Tracker: +Changelog: +Summary:Template +Description: +Template +. diff --git a/tests/run-tests b/tests/run-tests index a4474daa..354e6782 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -100,6 +100,8 @@ echo_header "test python getsig replacement" cd $WORKSPACE/tests/getsig ./make.sh + +cd $WORKSPACE/tests for testcase in $WORKSPACE/tests/*.TestCase; do $testcase done @@ -138,6 +140,19 @@ $fdroid readmeta $fdroid update +#------------------------------------------------------------------------------# +echo_header 'run `fdroid build` in fresh git checkout from import.TestCase' + +cd $WORKSPACE/tests/tmp/importer +if [ -d $ANDROID_HOME/platforms/android-23 ]; then + echo "build_tools = '`ls -1 $ANDROID_HOME/build-tools/ | sort -n | tail -1`'" > config.py + echo "force_build_tools = True" >> config.py + $fdroid build --verbose org.fdroid.ci.test.app:300 +else + echo 'WARNING: Skipping `fdroid build` test since android-23 is missing!' +fi + + #------------------------------------------------------------------------------# echo_header "copy tests/repo, generate java/gpg keys, update, and gpgsign"