From 9abb80b3b77622c7ad0a4d26ef33417804406c2d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 5 Aug 2015 10:55:54 +0200 Subject: [PATCH 1/8] support .fdroid.* metadata file in source root of app being built This allows app makers to include a .fdroid.(json|xml|yaml|txt) metadata file in the root of the git repo of their app, then they can build it using `fdroid build`. This is useful for developers who want to maintain the fdroid build recipe themselves, and run the fdroid build process for their own builds. --- fdroidserver/metadata.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index d69e9074..bbaccd91 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -22,6 +22,7 @@ import os import re import glob import cgi +import logging import textwrap import io @@ -778,7 +779,10 @@ def read_metadata(xref=True): for metadatapath in sorted(glob.glob(os.path.join('metadata', '*.txt')) + glob.glob(os.path.join('metadata', '*.json')) + glob.glob(os.path.join('metadata', '*.xml')) - + glob.glob(os.path.join('metadata', '*.yaml'))): + + glob.glob(os.path.join('metadata', '*.yaml')) + + glob.glob('.fdroid.json') + + glob.glob('.fdroid.xml') + + glob.glob('.fdroid.yaml')): app = parse_metadata(metadatapath) if app.id in apps: raise MetaDataException("Found multiple metadata files for " + app.id) @@ -824,6 +828,25 @@ def get_default_app_info(metadatapath=None): else: appid, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath)) + if appid == '.fdroid': # we have local metadata in the app's source + if os.path.exists('AndroidManifest.xml'): + manifestroot = fdroidserver.common.parse_xml('AndroidManifest.xml') + else: + pattern = re.compile(""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""") + for root, dirs, files in os.walk(os.getcwd()): + if 'build.gradle' in files: + p = os.path.join(root, 'build.gradle') + with open(p) as f: + data = f.read() + m = pattern.search(data) + if m: + logging.debug('Using: ' + os.path.join(root, 'AndroidManifest.xml')) + manifestroot = fdroidserver.common.parse_xml(os.path.join(root, 'AndroidManifest.xml')) + break + if manifestroot is None: + raise MetaDataException("Cannot find a packageName for {0}!".format(metadatapath)) + appid = manifestroot.attrib['package'] + app = App() app.metadatapath = metadatapath if appid is not None: From 3768d7a4d68f09005ccfe5dcd610752fee995a30 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 5 Aug 2015 14:39:58 +0200 Subject: [PATCH 2/8] refactor env handling for FDroidPopen to support .fdroid.* metadata The start up sequence of processes that are based on the .fdroid.* metadata is a bit different, so this ensures that the environment variables get properly initialized in all cases. This also creates a single function where the environment is set. Before it was being set in multiple places across multiple files. --- fdroidserver/build.py | 8 +------ fdroidserver/common.py | 50 +++++++++++++++++++++++++++++++----------- tests/common.TestCase | 2 +- tests/import.TestCase | 5 +++-- tests/install.TestCase | 2 +- tests/update.TestCase | 4 ++++ 6 files changed, 47 insertions(+), 24 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index b059124d..b64cc920 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -472,13 +472,7 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, logging.critical("Android NDK '%s' is not a directory!" % ndk_path) sys.exit(3) - # Set up environment vars that depend on each build - for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']: - common.env[n] = ndk_path - - common.reset_env_path() - # Set up the current NDK to the PATH - common.add_to_env_path(ndk_path) + common.set_FDroidPopen_env(build) # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, build, diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 725c7961..8b676cc5 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -197,7 +197,7 @@ def read_config(opts, config_file='config.py'): The config is read from config_file, which is in the current directory when any of the repo management commands are used. """ - global config, options, env, orig_path + global config, options, orig_path if config is not None: return config @@ -268,6 +268,21 @@ def read_config(opts, config_file='config.py'): return config +def get_ndk_path(version): + if config is None or 'ndk_paths' not in config: + ndk_path = os.getenv('ANDROID_NDK_HOME') + if ndk_path is None: + logging.error('No NDK found! Either set ANDROID_NDK_HOME or add ndk_path to your config.py') + else: + return ndk_path + if version is None: + version = 'r10e' # falls back to latest + paths = config['ndk_paths'] + if version not in paths: + return '' + return paths[version] or '' + + def find_sdk_tools_cmd(cmd): '''find a working path to a tool from the Android SDK''' @@ -1639,6 +1654,8 @@ def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True): """ global env + if env is None: + set_FDroidPopen_env() if cwd: cwd = os.path.normpath(cwd) @@ -1780,25 +1797,32 @@ def remove_signing_keys(build_dir): logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path)) -def reset_env_path(): +def set_FDroidPopen_env(build=None): + # There is only a weak standard, the variables used by gradle, so also set + # up the most commonly used environment variables for SDK and NDK global env, orig_path - env['PATH'] = orig_path + if env is None: + env = os.environ + orig_path = env['PATH'] + for n in ['ANDROID_HOME', 'ANDROID_SDK']: + env[n] = config['sdk_path'] + # Set up environment vars that depend on each build + if build is not None: + path = build.ndk_path() + paths = orig_path.split(os.pathsep) + if path in paths: + return + paths.append(path) + env['PATH'] = os.pathsep.join(paths) -def add_to_env_path(path): - global env - paths = env['PATH'].split(os.pathsep) - if path in paths: - return - paths.append(path) - env['PATH'] = os.pathsep.join(paths) + for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']: + env[n] = build.ndk_path() def replace_config_vars(cmd, build): - global env cmd = cmd.replace('$$SDK$$', config['sdk_path']) - # env['ANDROID_NDK'] is set in build_local right before prepare_source - cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK']) + cmd = cmd.replace('$$NDK$$', get_ndk_path(build['ndk'])) cmd = cmd.replace('$$MVN3$$', config['mvn3']) if build is not None: cmd = cmd.replace('$$COMMIT$$', build.commit) diff --git a/tests/common.TestCase b/tests/common.TestCase index 48e1d29d..d7dca14e 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -64,7 +64,7 @@ class CommonTest(unittest.TestCase): def testIsApkDebuggable(self): config = dict() - config['sdk_path'] = os.getenv('ANDROID_HOME') + fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config self._set_build_tools() config['aapt'] = fdroidserver.common.find_sdk_tools_cmd('aapt') diff --git a/tests/import.TestCase b/tests/import.TestCase index cce85d68..c53b53d4 100755 --- a/tests/import.TestCase +++ b/tests/import.TestCase @@ -25,8 +25,9 @@ class ImportTest(unittest.TestCase): def test_import_gitlab(self): # FDroidPopen needs some config to work - fdroidserver.common.config = dict() - fdroidserver.common.config['sdk_path'] = '/fake/path/to/android-sdk' + config = dict() + fdroidserver.common.fill_config_defaults(config) + fdroidserver.common.config = config url = 'https://gitlab.com/fdroid/fdroidclient' app = fdroidserver.metadata.get_default_app_info() diff --git a/tests/install.TestCase b/tests/install.TestCase index d1ed93ff..ce516117 100755 --- a/tests/install.TestCase +++ b/tests/install.TestCase @@ -23,7 +23,7 @@ class InstallTest(unittest.TestCase): def test_devices(self): config = dict() - config['sdk_path'] = os.getenv('ANDROID_HOME') + fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config config['adb'] = fdroidserver.common.find_sdk_tools_cmd('adb') self.assertTrue(os.path.exists(config['adb'])) diff --git a/tests/update.TestCase b/tests/update.TestCase index 83349f5e..0cc93e5c 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -29,6 +29,10 @@ class UpdateTest(unittest.TestCase): if not os.path.exists(getsig_dir + "/getsig.class"): logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir) sys.exit(1) + # FDroidPopen needs some config to work + config = dict() + fdroidserver.common.fill_config_defaults(config) + fdroidserver.common.config = config p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'), 'getsig', os.path.join(os.getcwd(), apkfile)]) sig = None From 3e6b263c8eda8992295e531be7a04eba88ca82f1 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 5 Aug 2015 14:42:41 +0200 Subject: [PATCH 3/8] do not require if config.py if using local metadata Since you can have a .fdroid.* metadata file included directly in an app's git repo, it seems annoying to force developers to also include a blank config.py in order to run `fdroid build` from their git repo. The next step after this is some kind of global config file for setting paths and things that need to be customized that would apply to all git repos, for example, to enable using a buildserver. This also required moving all of the env setting for FDroidPopen into a single place so that it is properly setup when using the local metadata. --- fdroidserver/common.py | 48 ++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 8b676cc5..df7977af 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -194,25 +194,29 @@ def regsub_file(pattern, repl, path): def read_config(opts, config_file='config.py'): """Read the repository config - The config is read from config_file, which is in the current directory when - any of the repo management commands are used. + The config is read from config_file, which is in the current + directory when any of the repo management commands are used. If + there is a local metadata file in the git repo, then config.py is + not required, just use defaults. + """ - global config, options, orig_path + global config, options if config is not None: return config - if not os.path.isfile(config_file): - logging.critical("Missing config file - is this a repo directory?") - sys.exit(2) options = opts config = {} - logging.debug("Reading %s" % config_file) - with io.open(config_file, "rb") as f: - code = compile(f.read(), config_file, 'exec') - exec(code, None, config) + if os.path.isfile(config_file): + logging.debug("Reading %s" % config_file) + with io.open(config_file, "rb") as f: + code = compile(f.read(), config_file, 'exec') + exec(code, None, config) + elif len(glob.glob('.fdroid.[a-z]*')) == 0: + logging.critical("Missing config file - is this a repo directory?") + sys.exit(2) # smartcardoptions must be a list since its command line args for Popen if 'smartcardoptions' in config: @@ -231,16 +235,6 @@ def read_config(opts, config_file='config.py'): fill_config_defaults(config) - # There is no standard, so just set up the most common environment - # variables - env = os.environ - orig_path = env['PATH'] - for n in ['ANDROID_HOME', 'ANDROID_SDK']: - env[n] = config['sdk_path'] - - for k, v in config['java_paths'].items(): - env['JAVA%s_HOME' % k] = v - for k in ["keystorepass", "keypass"]: if k in config: write_password_file(k) @@ -1798,16 +1792,24 @@ def remove_signing_keys(build_dir): def set_FDroidPopen_env(build=None): - # There is only a weak standard, the variables used by gradle, so also set - # up the most commonly used environment variables for SDK and NDK + ''' + set up the environment variables for the build environment + + There is only a weak standard, the variables used by gradle, so also set + up the most commonly used environment variables for SDK and NDK + ''' global env, orig_path + if env is None: env = os.environ orig_path = env['PATH'] for n in ['ANDROID_HOME', 'ANDROID_SDK']: env[n] = config['sdk_path'] + for k, v in config['java_paths'].items(): + env['JAVA%s_HOME' % k] = v - # Set up environment vars that depend on each build + # Set up environment vars that depend on each build, only set the + # NDK env vars if the NDK is not already in the PATH if build is not None: path = build.ndk_path() paths = orig_path.split(os.pathsep) From 578218a8b083c8e31be46d5c7f55b0cdbe1b3008 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 5 Aug 2015 15:18:22 +0200 Subject: [PATCH 4/8] build: set default config for .fdroid.* metadata The default config for .fdroid.* metadata that is included in a git repo is different than for the standard metadata/ layout because the expectations are different. In this case, the most common user will be the app developer working on the latest update of the app on their own machine. --- fdroidserver/build.py | 28 ++++++++++++++++++---------- fdroidserver/common.py | 12 +++++++++++- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index b64cc920..7f647b8f 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1002,17 +1002,25 @@ def main(): options, parser = parse_commandline() - metadata_files = glob.glob('.fdroid.*[a-z]') # ignore files ending in ~ - if os.path.isdir('metadata'): - pass - elif len(metadata_files) == 0: - raise FDroidException("No app metadata found, nothing to process!") - elif len(metadata_files) > 1: + # The defaults for .fdroid.* metadata that is included in a git repo are + # different than for the standard metadata/ layout because expectations + # are different. In this case, the most common user will be the app + # developer working on the latest update of the app on their own machine. + local_metadata_files = common.get_local_local_metadata_files() + if len(local_metadata_files) == 1: # there is local metadata in an app's source + config = dict(common.default_config) + # `fdroid build` should build only the latest version by default since + # most of the time the user will be building the most recent update + if not options.all: + options.latest = True + elif len(local_metadata_files) > 1: raise FDroidException("Only one local metadata file allowed! Found: " - + " ".join(metadata_files)) - - if not options.appid and not options.all: - parser.error("option %s: If you really want to build all the apps, use --all" % "all") + + " ".join(local_metadata_files)) + else: + if not os.path.isdir('metadata') and len(local_metadata_files) == 0: + raise FDroidException("No app metadata found, nothing to process!") + if not options.appid and not options.all: + parser.error("option %s: If you really want to build all the apps, use --all" % "all") config = common.read_config(options) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index df7977af..72af49d8 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -214,7 +214,7 @@ def read_config(opts, config_file='config.py'): with io.open(config_file, "rb") as f: code = compile(f.read(), config_file, 'exec') exec(code, None, config) - elif len(glob.glob('.fdroid.[a-z]*')) == 0: + elif len(get_local_metadata_files()) == 0: logging.critical("Missing config file - is this a repo directory?") sys.exit(2) @@ -361,6 +361,16 @@ def write_password_file(pwtype, password=None): config[pwtype + 'file'] = filename +def get_local_metadata_files(): + '''get any metadata files local to an app's source repo + + This tries to ignore anything that does not count as app metdata, + including emacs cruft ending in ~ and the .fdroid.key*pass.txt files. + + ''' + return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]') + + # Given the arguments in the form of multiple appid:[vc] strings, this returns # a dictionary with the set of vercodes specified for each package. def read_pkg_args(args, allow_vercodes=False): From 3b20153cd754c52caf5c00974548eb7f8103878e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 5 Aug 2015 15:19:14 +0200 Subject: [PATCH 5/8] document new `fdroid build` behavior with .fdroid.* metadata --- docs/fdroid.texi | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/fdroid.texi b/docs/fdroid.texi index 29cac7e4..f158147c 100644 --- a/docs/fdroid.texi +++ b/docs/fdroid.texi @@ -314,7 +314,7 @@ To build a single version of a single application, you could run the following: @example -./fdroid build org.fdroid.fdroid:16 +fdroid build org.fdroid.fdroid:16 @end example This attempts to build version code 16 (which is version 0.25) of the F-Droid @@ -336,7 +336,7 @@ tarball containing exactly the source that was used to generate the binary. If you were intending to publish these files, you could then run: @example -./fdroid publish +fdroid publish @end example The source tarball would move to the @code{repo} directory (which is the @@ -366,6 +366,26 @@ all such prebuilts are built either via the metadata or by a reputable third party. +@section Running "fdroid build" in your app's source + +Another option for using @code{fdroid build} is to use a metadata file +that is included in the app's source itself, rather than in a +@code{metadata/} folder with lots of other apps. This metadata file +should be in the root of your source repo, and be called +@code{.fdroid.json}, @code{.fdroid.xml}, @code{.fdroid.yaml}, or +@code{.fdroid.txt}, depending on your preferred data format: JSON, +XML, YAML, or F-Droid's @code{.txt} format. + +Once you have that setup, you can build the most recent version of +the app using the whole FDroid stack by running: + +@example +fdroid build +@end example + +If you want to build every single version, then specify @code{--all}. + + @section Direct Installation You can also build and install directly to a connected device or emulator @@ -386,7 +406,7 @@ will take a URL and optionally some other parameters, and attempt to construct as much information as possible by analysing the source code. Basic usage is: @example -./fdroid import --url=http://address.of.project +fdroid import --url=http://address.of.project @end example For this to work, the URL must point to a project format that the script From 994488ad47872d1d8aa10a90e28e30dda883da6e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 10 Aug 2015 22:29:17 +0200 Subject: [PATCH 6/8] rename metadata.write_metadata() to match metadata.parse_*_metadata() This changes the function name to include the format of the metadata file, and also changes the order of the args to match the parse_*_metadata() functions. --- fdroidserver/checkupdates.py | 3 +-- fdroidserver/import.py | 3 +-- fdroidserver/metadata.py | 23 +++++++++++++++-------- fdroidserver/rewritemeta.py | 3 +-- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index 7adfcbb3..e22a6e0f 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -492,8 +492,7 @@ def checkupdates_app(app, first=True): if commitmsg: metadatapath = os.path.join('metadata', app.id + '.txt') - with open(metadatapath, 'w') as f: - metadata.write_metadata('txt', f, app) + metadata.write_metadata(metadatapath, app) if options.commit: logging.info("Commiting update for " + metadatapath) gitcmd = ["git", "commit", "-m", commitmsg] diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 3b80e485..247a42c4 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -243,8 +243,7 @@ def main(): f.write(app.RepoType + ' ' + app.Repo) metadatapath = os.path.join('metadata', package + '.txt') - with open(metadatapath, 'w') as f: - metadata.write_metadata('txt', f, app) + metadata.write_metadata(metadatapath, app) logging.info("Wrote " + metadatapath) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index bbaccd91..722e1710 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -1270,7 +1270,7 @@ def write_plaintext_metadata(mf, app, w_comment, w_field, w_build): # # 'mf' - Writer interface (file, StringIO, ...) # 'app' - The app data -def write_txt_metadata(mf, app): +def write_txt(mf, app): def w_comment(line): mf.write("# %s\n" % line) @@ -1313,7 +1313,7 @@ def write_txt_metadata(mf, app): write_plaintext_metadata(mf, app, w_comment, w_field, w_build) -def write_yaml_metadata(mf, app): +def write_yaml(mf, app): def w_comment(line): mf.write("# %s\n" % line) @@ -1377,9 +1377,16 @@ def write_yaml_metadata(mf, app): write_plaintext_metadata(mf, app, w_comment, w_field, w_build) -def write_metadata(fmt, mf, app): - if fmt == 'txt': - return write_txt_metadata(mf, app) - if fmt == 'yaml': - return write_yaml_metadata(mf, app) - raise MetaDataException("Unknown metadata format given") +def write_metadata(metadatapath, app): + _, ext = fdroidserver.common.get_extension(metadatapath) + accepted = fdroidserver.common.config['accepted_formats'] + if ext not in accepted: + raise MetaDataException('Cannot write "%s", not an accepted format, use: %s' % ( + metadatapath, ', '.join(accepted))) + + with open(metadatapath, 'w') as mf: + if ext == 'txt': + return write_txt(mf, app) + elif ext == 'yaml': + return write_yaml(mf, app) + raise MetaDataException('Unknown metadata format: %s' % metadatapath) diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py index 971ab18c..75888a23 100644 --- a/fdroidserver/rewritemeta.py +++ b/fdroidserver/rewritemeta.py @@ -84,8 +84,7 @@ def main(): print(app.metadatapath) continue - with open(base + '.' + to_ext, 'w') as f: - metadata.write_metadata(to_ext, f, app) + metadata.write_metadata(base + '.' + to_ext, app) if ext != to_ext: os.remove(app.metadatapath) From 19189b9b041e23afc15fa0b0d06ee30e62b34208 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 21 Mar 2016 21:00:12 +0100 Subject: [PATCH 7/8] import app into fdroid directly from git clone This adds a new method for `fdroid import` that will generate the fdroidserver metadata based on a local git repo. This new mode generates the metadata in the new .fdroid.yaml format in the git repo itself. It is intended as a quick way to get starting building apps using the fdroidserver tools. --- docs/fdroid.texi | 21 +++++++++++--- fdroidserver/import.py | 65 ++++++++++++++++++++++++++++++------------ setup.py | 1 + 3 files changed, 65 insertions(+), 22 deletions(-) diff --git a/docs/fdroid.texi b/docs/fdroid.texi index f158147c..1cc011d6 100644 --- a/docs/fdroid.texi +++ b/docs/fdroid.texi @@ -401,19 +401,32 @@ the signed output directory were modified, you won't be notified. @node Importing Applications @chapter Importing Applications -To help with starting work on including a new application, @code{fdroid import} -will take a URL and optionally some other parameters, and attempt to construct -as much information as possible by analysing the source code. Basic usage is: +To help with starting work on including a new application, use +@code{fdroid import} to set up a new template project. It has two +modes of operation, starting with a cloned git repo: + +@example +git clone https://gitlab.com/fdroid/fdroidclient +cd fdroidclient +fdroid import +@end example + +Or starting with a URL to a project page: @example fdroid import --url=http://address.of.project @end example -For this to work, the URL must point to a project format that the script +When a URL is specified using the @code{--url=} flag, @code{fdroid +import} will use that URL to find out information about the project, +and if it finds a git repo, it will also clone that. For this to +work, the URL must point to a project format that the script understands. Currently this is limited to one of the following: @enumerate @item +GitLab - @code{https://gitlab.com/PROJECTNAME/REPONAME} +@item Gitorious - @code{https://gitorious.org/PROJECTNAME/REPONAME} @item Github - @code{https://github.com/USER/PROJECT} diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 247a42c4..4d342f04 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import binascii import sys import os import shutil @@ -180,12 +181,38 @@ def main(): root_dir = None build_dir = None - if options.url: - root_dir, build_dir = get_metadata_from_url(app, options.url) - elif os.path.isdir('.git'): - if options.url: - app.WebSite = options.url + local_metadata_files = common.get_local_metadata_files() + if local_metadata_files != []: + logging.error("This repo already has local metadata: %s" % local_metadata_files[0]) + sys.exit(1) + + if options.url is None and os.path.isdir('.git'): + app.AutoName = os.path.basename(os.getcwd()) + app.RepoType = 'git' + + build = {} root_dir = get_subdir(os.getcwd()) + if os.path.exists('build.gradle'): + build.gradle = ['yes'] + + import git + repo = git.repo.Repo(root_dir) # git repo + for remote in git.Remote.iter_items(repo): + if remote.name == 'origin': + url = repo.remotes.origin.url + if url.startswith('https://git'): # github, gitlab + app.SourceCode = url.rstrip('.git') + app.Repo = url + break + # repo.head.commit.binsha is a bytearray stored in a str + build.commit = binascii.hexlify(bytearray(repo.head.commit.binsha)) + write_local_file = True + elif options.url: + root_dir, build_dir = get_metadata_from_url(app, options.url) + build = metadata.Build() + build.commit = '?' + build.disable = 'Generated by import.py - check/set version fields and commit id' + write_local_file = False else: logging.error("Specify project url.") sys.exit(1) @@ -222,29 +249,31 @@ def main(): sys.exit(1) # Create a build line... - build = metadata.Build() build.version = version or '?' build.vercode = vercode or '?' - build.commit = '?' - build.disable = 'Generated by import.py - check/set version fields and commit id' if options.subdir: build.subdir = options.subdir if os.path.exists(os.path.join(root_dir, 'jni')): build.buildjni = ['yes'] + metadata.post_metadata_parse(app) + app.builds.append(build) - # Keep the repo directory to save bandwidth... - if not os.path.exists('build'): - os.mkdir('build') - if build_dir is not None: - shutil.move(build_dir, os.path.join('build', package)) - with open('build/.fdroidvcs-' + package, 'w') as f: - f.write(app.RepoType + ' ' + app.Repo) + if write_local_file: + metadata.write_metadata('.fdroid.yaml', app) + else: + # Keep the repo directory to save bandwidth... + if not os.path.exists('build'): + os.mkdir('build') + if build_dir is not None: + shutil.move(build_dir, os.path.join('build', package)) + with open('build/.fdroidvcs-' + package, 'w') as f: + f.write(app.RepoType + ' ' + app.Repo) - metadatapath = os.path.join('metadata', package + '.txt') - metadata.write_metadata(metadatapath, app) - logging.info("Wrote " + metadatapath) + metadatapath = os.path.join('metadata', package + '.txt') + metadata.write_metadata(metadatapath, app) + logging.info("Wrote " + metadatapath) if __name__ == "__main__": diff --git a/setup.py b/setup.py index 017cccd9..9c068b86 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ setup(name='fdroidserver', 'examples/fdroid-icon.png']), ], install_requires=[ + 'GitPython', 'mwclient', 'paramiko', 'Pillow', From 9cd6b444f848e2a2f8ed4ca16cebb96390ade179 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 21 Mar 2016 21:51:23 +0100 Subject: [PATCH 8/8] standardize on .yml as the file extension for YAML Though the YAML people recommend .yaml for the file extension, in Android land it seems clear that .yml has won out: * .travis.yml * .gitlab-ci.yml * .circle.yml * Ansible main.yml --- fdroidserver/common.py | 2 +- fdroidserver/import.py | 2 +- fdroidserver/metadata.py | 8 ++++---- fdroidserver/rewritemeta.py | 2 +- tests/metadata.TestCase | 2 +- tests/metadata/org.videolan.vlc.pickle | 2 +- .../{org.videolan.vlc.yaml => org.videolan.vlc.yml} | 0 tests/run-tests | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) rename tests/metadata/{org.videolan.vlc.yaml => org.videolan.vlc.yml} (100%) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 72af49d8..acaeaffd 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -63,7 +63,7 @@ default_config = { 'ant': "ant", 'mvn3': "mvn", 'gradle': 'gradle', - 'accepted_formats': ['txt', 'yaml'], + 'accepted_formats': ['txt', 'yml'], 'sync_from_local_copy_dir': False, 'per_app_repos': False, 'make_current_version_link': True, diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 4d342f04..8c7f5107 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -261,7 +261,7 @@ def main(): app.builds.append(build) if write_local_file: - metadata.write_metadata('.fdroid.yaml', app) + metadata.write_metadata('.fdroid.yml', app) else: # Keep the repo directory to save bandwidth... if not os.path.exists('build'): diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 722e1710..71da8453 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -779,10 +779,10 @@ def read_metadata(xref=True): for metadatapath in sorted(glob.glob(os.path.join('metadata', '*.txt')) + glob.glob(os.path.join('metadata', '*.json')) + glob.glob(os.path.join('metadata', '*.xml')) - + glob.glob(os.path.join('metadata', '*.yaml')) + + glob.glob(os.path.join('metadata', '*.yml')) + glob.glob('.fdroid.json') + glob.glob('.fdroid.xml') - + glob.glob('.fdroid.yaml')): + + glob.glob('.fdroid.yml')): app = parse_metadata(metadatapath) if app.id in apps: raise MetaDataException("Found multiple metadata files for " + app.id) @@ -950,7 +950,7 @@ def parse_metadata(metadatapath): parse_json_metadata(mf, app) elif ext == 'xml': parse_xml_metadata(mf, app) - elif ext == 'yaml': + elif ext == 'yml': parse_yaml_metadata(mf, app) else: raise MetaDataException('Unknown metadata format: %s' % metadatapath) @@ -1387,6 +1387,6 @@ def write_metadata(metadatapath, app): with open(metadatapath, 'w') as mf: if ext == 'txt': return write_txt(mf, app) - elif ext == 'yaml': + elif ext == 'yml': return write_yaml(mf, app) raise MetaDataException('Unknown metadata format: %s' % metadatapath) diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py index 75888a23..8335ed9c 100644 --- a/fdroidserver/rewritemeta.py +++ b/fdroidserver/rewritemeta.py @@ -64,7 +64,7 @@ def main(): if options.list and options.to is not None: parser.error("Cannot use --list and --to at the same time") - supported = ['txt', 'yaml'] + supported = ['txt', 'yml'] if options.to is not None and options.to not in supported: parser.error("Must give a valid format to --to") diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 9dfe2bbc..c287a212 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -33,7 +33,7 @@ class MetadataTest(unittest.TestCase): config = dict() config['sdk_path'] = '/opt/android-sdk' config['ndk_paths'] = dict() - config['accepted_formats'] = ['json', 'txt', 'xml', 'yaml'] + config['accepted_formats'] = ['json', 'txt', 'xml', 'yml'] fdroidserver.common.config = config apps = fdroidserver.metadata.read_metadata(xref=True) diff --git a/tests/metadata/org.videolan.vlc.pickle b/tests/metadata/org.videolan.vlc.pickle index 45af8916..a1810e60 100644 --- a/tests/metadata/org.videolan.vlc.pickle +++ b/tests/metadata/org.videolan.vlc.pickle @@ -4618,7 +4618,7 @@ sasS'FlattrID' p1324 NsS'metadatapath' p1325 -S'metadata/org.videolan.vlc.yaml' +S'metadata/org.videolan.vlc.yml' p1326 sS'Disabled' p1327 diff --git a/tests/metadata/org.videolan.vlc.yaml b/tests/metadata/org.videolan.vlc.yml similarity index 100% rename from tests/metadata/org.videolan.vlc.yaml rename to tests/metadata/org.videolan.vlc.yml diff --git a/tests/run-tests b/tests/run-tests index 8ec03c35..62d27aa7 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -163,7 +163,7 @@ cp $WORKSPACE/tests/metadata/org.smssecure.smssecure.txt $REPOROOT/metadata/ $fdroid readmeta # now make a fake duplicate -touch $REPOROOT/metadata/org.smssecure.smssecure.yaml +touch $REPOROOT/metadata/org.smssecure.smssecure.yml set +e $fdroid readmeta