diff --git a/docs/fdroid.texi b/docs/fdroid.texi index 29cac7e4..1cc011d6 100644 --- a/docs/fdroid.texi +++ b/docs/fdroid.texi @@ -314,7 +314,7 @@ To build a single version of a single application, you could run the following: @example -./fdroid build org.fdroid.fdroid:16 +fdroid build org.fdroid.fdroid:16 @end example This attempts to build version code 16 (which is version 0.25) of the F-Droid @@ -336,7 +336,7 @@ tarball containing exactly the source that was used to generate the binary. If you were intending to publish these files, you could then run: @example -./fdroid publish +fdroid publish @end example The source tarball would move to the @code{repo} directory (which is the @@ -366,6 +366,26 @@ all such prebuilts are built either via the metadata or by a reputable third party. +@section Running "fdroid build" in your app's source + +Another option for using @code{fdroid build} is to use a metadata file +that is included in the app's source itself, rather than in a +@code{metadata/} folder with lots of other apps. This metadata file +should be in the root of your source repo, and be called +@code{.fdroid.json}, @code{.fdroid.xml}, @code{.fdroid.yaml}, or +@code{.fdroid.txt}, depending on your preferred data format: JSON, +XML, YAML, or F-Droid's @code{.txt} format. + +Once you have that setup, you can build the most recent version of +the app using the whole FDroid stack by running: + +@example +fdroid build +@end example + +If you want to build every single version, then specify @code{--all}. + + @section Direct Installation You can also build and install directly to a connected device or emulator @@ -381,19 +401,32 @@ the signed output directory were modified, you won't be notified. @node Importing Applications @chapter Importing Applications -To help with starting work on including a new application, @code{fdroid import} -will take a URL and optionally some other parameters, and attempt to construct -as much information as possible by analysing the source code. Basic usage is: +To help with starting work on including a new application, use +@code{fdroid import} to set up a new template project. It has two +modes of operation, starting with a cloned git repo: @example -./fdroid import --url=http://address.of.project +git clone https://gitlab.com/fdroid/fdroidclient +cd fdroidclient +fdroid import @end example -For this to work, the URL must point to a project format that the script +Or starting with a URL to a project page: + +@example +fdroid import --url=http://address.of.project +@end example + +When a URL is specified using the @code{--url=} flag, @code{fdroid +import} will use that URL to find out information about the project, +and if it finds a git repo, it will also clone that. For this to +work, the URL must point to a project format that the script understands. Currently this is limited to one of the following: @enumerate @item +GitLab - @code{https://gitlab.com/PROJECTNAME/REPONAME} +@item Gitorious - @code{https://gitorious.org/PROJECTNAME/REPONAME} @item Github - @code{https://github.com/USER/PROJECT} diff --git a/fdroidserver/build.py b/fdroidserver/build.py index b059124d..7f647b8f 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -472,13 +472,7 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, logging.critical("Android NDK '%s' is not a directory!" % ndk_path) sys.exit(3) - # Set up environment vars that depend on each build - for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']: - common.env[n] = ndk_path - - common.reset_env_path() - # Set up the current NDK to the PATH - common.add_to_env_path(ndk_path) + common.set_FDroidPopen_env(build) # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, build, @@ -1008,17 +1002,25 @@ def main(): options, parser = parse_commandline() - metadata_files = glob.glob('.fdroid.*[a-z]') # ignore files ending in ~ - if os.path.isdir('metadata'): - pass - elif len(metadata_files) == 0: - raise FDroidException("No app metadata found, nothing to process!") - elif len(metadata_files) > 1: + # The defaults for .fdroid.* metadata that is included in a git repo are + # different than for the standard metadata/ layout because expectations + # are different. In this case, the most common user will be the app + # developer working on the latest update of the app on their own machine. + local_metadata_files = common.get_local_local_metadata_files() + if len(local_metadata_files) == 1: # there is local metadata in an app's source + config = dict(common.default_config) + # `fdroid build` should build only the latest version by default since + # most of the time the user will be building the most recent update + if not options.all: + options.latest = True + elif len(local_metadata_files) > 1: raise FDroidException("Only one local metadata file allowed! Found: " - + " ".join(metadata_files)) - - if not options.appid and not options.all: - parser.error("option %s: If you really want to build all the apps, use --all" % "all") + + " ".join(local_metadata_files)) + else: + if not os.path.isdir('metadata') and len(local_metadata_files) == 0: + raise FDroidException("No app metadata found, nothing to process!") + if not options.appid and not options.all: + parser.error("option %s: If you really want to build all the apps, use --all" % "all") config = common.read_config(options) diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index 7adfcbb3..e22a6e0f 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -492,8 +492,7 @@ def checkupdates_app(app, first=True): if commitmsg: metadatapath = os.path.join('metadata', app.id + '.txt') - with open(metadatapath, 'w') as f: - metadata.write_metadata('txt', f, app) + metadata.write_metadata(metadatapath, app) if options.commit: logging.info("Commiting update for " + metadatapath) gitcmd = ["git", "commit", "-m", commitmsg] diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 725c7961..acaeaffd 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -63,7 +63,7 @@ default_config = { 'ant': "ant", 'mvn3': "mvn", 'gradle': 'gradle', - 'accepted_formats': ['txt', 'yaml'], + 'accepted_formats': ['txt', 'yml'], 'sync_from_local_copy_dir': False, 'per_app_repos': False, 'make_current_version_link': True, @@ -194,25 +194,29 @@ def regsub_file(pattern, repl, path): def read_config(opts, config_file='config.py'): """Read the repository config - The config is read from config_file, which is in the current directory when - any of the repo management commands are used. + The config is read from config_file, which is in the current + directory when any of the repo management commands are used. If + there is a local metadata file in the git repo, then config.py is + not required, just use defaults. + """ - global config, options, env, orig_path + global config, options if config is not None: return config - if not os.path.isfile(config_file): - logging.critical("Missing config file - is this a repo directory?") - sys.exit(2) options = opts config = {} - logging.debug("Reading %s" % config_file) - with io.open(config_file, "rb") as f: - code = compile(f.read(), config_file, 'exec') - exec(code, None, config) + if os.path.isfile(config_file): + logging.debug("Reading %s" % config_file) + with io.open(config_file, "rb") as f: + code = compile(f.read(), config_file, 'exec') + exec(code, None, config) + elif len(get_local_metadata_files()) == 0: + logging.critical("Missing config file - is this a repo directory?") + sys.exit(2) # smartcardoptions must be a list since its command line args for Popen if 'smartcardoptions' in config: @@ -231,16 +235,6 @@ def read_config(opts, config_file='config.py'): fill_config_defaults(config) - # There is no standard, so just set up the most common environment - # variables - env = os.environ - orig_path = env['PATH'] - for n in ['ANDROID_HOME', 'ANDROID_SDK']: - env[n] = config['sdk_path'] - - for k, v in config['java_paths'].items(): - env['JAVA%s_HOME' % k] = v - for k in ["keystorepass", "keypass"]: if k in config: write_password_file(k) @@ -268,6 +262,21 @@ def read_config(opts, config_file='config.py'): return config +def get_ndk_path(version): + if config is None or 'ndk_paths' not in config: + ndk_path = os.getenv('ANDROID_NDK_HOME') + if ndk_path is None: + logging.error('No NDK found! Either set ANDROID_NDK_HOME or add ndk_path to your config.py') + else: + return ndk_path + if version is None: + version = 'r10e' # falls back to latest + paths = config['ndk_paths'] + if version not in paths: + return '' + return paths[version] or '' + + def find_sdk_tools_cmd(cmd): '''find a working path to a tool from the Android SDK''' @@ -352,6 +361,16 @@ def write_password_file(pwtype, password=None): config[pwtype + 'file'] = filename +def get_local_metadata_files(): + '''get any metadata files local to an app's source repo + + This tries to ignore anything that does not count as app metdata, + including emacs cruft ending in ~ and the .fdroid.key*pass.txt files. + + ''' + return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]') + + # Given the arguments in the form of multiple appid:[vc] strings, this returns # a dictionary with the set of vercodes specified for each package. def read_pkg_args(args, allow_vercodes=False): @@ -1639,6 +1658,8 @@ def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True): """ global env + if env is None: + set_FDroidPopen_env() if cwd: cwd = os.path.normpath(cwd) @@ -1780,25 +1801,40 @@ def remove_signing_keys(build_dir): logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path)) -def reset_env_path(): +def set_FDroidPopen_env(build=None): + ''' + set up the environment variables for the build environment + + There is only a weak standard, the variables used by gradle, so also set + up the most commonly used environment variables for SDK and NDK + ''' global env, orig_path - env['PATH'] = orig_path + if env is None: + env = os.environ + orig_path = env['PATH'] + for n in ['ANDROID_HOME', 'ANDROID_SDK']: + env[n] = config['sdk_path'] + for k, v in config['java_paths'].items(): + env['JAVA%s_HOME' % k] = v -def add_to_env_path(path): - global env - paths = env['PATH'].split(os.pathsep) - if path in paths: - return - paths.append(path) - env['PATH'] = os.pathsep.join(paths) + # Set up environment vars that depend on each build, only set the + # NDK env vars if the NDK is not already in the PATH + if build is not None: + path = build.ndk_path() + paths = orig_path.split(os.pathsep) + if path in paths: + return + paths.append(path) + env['PATH'] = os.pathsep.join(paths) + + for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']: + env[n] = build.ndk_path() def replace_config_vars(cmd, build): - global env cmd = cmd.replace('$$SDK$$', config['sdk_path']) - # env['ANDROID_NDK'] is set in build_local right before prepare_source - cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK']) + cmd = cmd.replace('$$NDK$$', get_ndk_path(build['ndk'])) cmd = cmd.replace('$$MVN3$$', config['mvn3']) if build is not None: cmd = cmd.replace('$$COMMIT$$', build.commit) diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 3b80e485..8c7f5107 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import binascii import sys import os import shutil @@ -180,12 +181,38 @@ def main(): root_dir = None build_dir = None - if options.url: - root_dir, build_dir = get_metadata_from_url(app, options.url) - elif os.path.isdir('.git'): - if options.url: - app.WebSite = options.url + local_metadata_files = common.get_local_metadata_files() + if local_metadata_files != []: + logging.error("This repo already has local metadata: %s" % local_metadata_files[0]) + sys.exit(1) + + if options.url is None and os.path.isdir('.git'): + app.AutoName = os.path.basename(os.getcwd()) + app.RepoType = 'git' + + build = {} root_dir = get_subdir(os.getcwd()) + if os.path.exists('build.gradle'): + build.gradle = ['yes'] + + import git + repo = git.repo.Repo(root_dir) # git repo + for remote in git.Remote.iter_items(repo): + if remote.name == 'origin': + url = repo.remotes.origin.url + if url.startswith('https://git'): # github, gitlab + app.SourceCode = url.rstrip('.git') + app.Repo = url + break + # repo.head.commit.binsha is a bytearray stored in a str + build.commit = binascii.hexlify(bytearray(repo.head.commit.binsha)) + write_local_file = True + elif options.url: + root_dir, build_dir = get_metadata_from_url(app, options.url) + build = metadata.Build() + build.commit = '?' + build.disable = 'Generated by import.py - check/set version fields and commit id' + write_local_file = False else: logging.error("Specify project url.") sys.exit(1) @@ -222,30 +249,31 @@ def main(): sys.exit(1) # Create a build line... - build = metadata.Build() build.version = version or '?' build.vercode = vercode or '?' - build.commit = '?' - build.disable = 'Generated by import.py - check/set version fields and commit id' if options.subdir: build.subdir = options.subdir if os.path.exists(os.path.join(root_dir, 'jni')): build.buildjni = ['yes'] + metadata.post_metadata_parse(app) + app.builds.append(build) - # Keep the repo directory to save bandwidth... - if not os.path.exists('build'): - os.mkdir('build') - if build_dir is not None: - shutil.move(build_dir, os.path.join('build', package)) - with open('build/.fdroidvcs-' + package, 'w') as f: - f.write(app.RepoType + ' ' + app.Repo) + if write_local_file: + metadata.write_metadata('.fdroid.yml', app) + else: + # Keep the repo directory to save bandwidth... + if not os.path.exists('build'): + os.mkdir('build') + if build_dir is not None: + shutil.move(build_dir, os.path.join('build', package)) + with open('build/.fdroidvcs-' + package, 'w') as f: + f.write(app.RepoType + ' ' + app.Repo) - metadatapath = os.path.join('metadata', package + '.txt') - with open(metadatapath, 'w') as f: - metadata.write_metadata('txt', f, app) - logging.info("Wrote " + metadatapath) + metadatapath = os.path.join('metadata', package + '.txt') + metadata.write_metadata(metadatapath, app) + logging.info("Wrote " + metadatapath) if __name__ == "__main__": diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index d69e9074..71da8453 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -22,6 +22,7 @@ import os import re import glob import cgi +import logging import textwrap import io @@ -778,7 +779,10 @@ def read_metadata(xref=True): for metadatapath in sorted(glob.glob(os.path.join('metadata', '*.txt')) + glob.glob(os.path.join('metadata', '*.json')) + glob.glob(os.path.join('metadata', '*.xml')) - + glob.glob(os.path.join('metadata', '*.yaml'))): + + glob.glob(os.path.join('metadata', '*.yml')) + + glob.glob('.fdroid.json') + + glob.glob('.fdroid.xml') + + glob.glob('.fdroid.yml')): app = parse_metadata(metadatapath) if app.id in apps: raise MetaDataException("Found multiple metadata files for " + app.id) @@ -824,6 +828,25 @@ def get_default_app_info(metadatapath=None): else: appid, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath)) + if appid == '.fdroid': # we have local metadata in the app's source + if os.path.exists('AndroidManifest.xml'): + manifestroot = fdroidserver.common.parse_xml('AndroidManifest.xml') + else: + pattern = re.compile(""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""") + for root, dirs, files in os.walk(os.getcwd()): + if 'build.gradle' in files: + p = os.path.join(root, 'build.gradle') + with open(p) as f: + data = f.read() + m = pattern.search(data) + if m: + logging.debug('Using: ' + os.path.join(root, 'AndroidManifest.xml')) + manifestroot = fdroidserver.common.parse_xml(os.path.join(root, 'AndroidManifest.xml')) + break + if manifestroot is None: + raise MetaDataException("Cannot find a packageName for {0}!".format(metadatapath)) + appid = manifestroot.attrib['package'] + app = App() app.metadatapath = metadatapath if appid is not None: @@ -927,7 +950,7 @@ def parse_metadata(metadatapath): parse_json_metadata(mf, app) elif ext == 'xml': parse_xml_metadata(mf, app) - elif ext == 'yaml': + elif ext == 'yml': parse_yaml_metadata(mf, app) else: raise MetaDataException('Unknown metadata format: %s' % metadatapath) @@ -1247,7 +1270,7 @@ def write_plaintext_metadata(mf, app, w_comment, w_field, w_build): # # 'mf' - Writer interface (file, StringIO, ...) # 'app' - The app data -def write_txt_metadata(mf, app): +def write_txt(mf, app): def w_comment(line): mf.write("# %s\n" % line) @@ -1290,7 +1313,7 @@ def write_txt_metadata(mf, app): write_plaintext_metadata(mf, app, w_comment, w_field, w_build) -def write_yaml_metadata(mf, app): +def write_yaml(mf, app): def w_comment(line): mf.write("# %s\n" % line) @@ -1354,9 +1377,16 @@ def write_yaml_metadata(mf, app): write_plaintext_metadata(mf, app, w_comment, w_field, w_build) -def write_metadata(fmt, mf, app): - if fmt == 'txt': - return write_txt_metadata(mf, app) - if fmt == 'yaml': - return write_yaml_metadata(mf, app) - raise MetaDataException("Unknown metadata format given") +def write_metadata(metadatapath, app): + _, ext = fdroidserver.common.get_extension(metadatapath) + accepted = fdroidserver.common.config['accepted_formats'] + if ext not in accepted: + raise MetaDataException('Cannot write "%s", not an accepted format, use: %s' % ( + metadatapath, ', '.join(accepted))) + + with open(metadatapath, 'w') as mf: + if ext == 'txt': + return write_txt(mf, app) + elif ext == 'yml': + return write_yaml(mf, app) + raise MetaDataException('Unknown metadata format: %s' % metadatapath) diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py index 971ab18c..8335ed9c 100644 --- a/fdroidserver/rewritemeta.py +++ b/fdroidserver/rewritemeta.py @@ -64,7 +64,7 @@ def main(): if options.list and options.to is not None: parser.error("Cannot use --list and --to at the same time") - supported = ['txt', 'yaml'] + supported = ['txt', 'yml'] if options.to is not None and options.to not in supported: parser.error("Must give a valid format to --to") @@ -84,8 +84,7 @@ def main(): print(app.metadatapath) continue - with open(base + '.' + to_ext, 'w') as f: - metadata.write_metadata(to_ext, f, app) + metadata.write_metadata(base + '.' + to_ext, app) if ext != to_ext: os.remove(app.metadatapath) diff --git a/setup.py b/setup.py index 017cccd9..9c068b86 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ setup(name='fdroidserver', 'examples/fdroid-icon.png']), ], install_requires=[ + 'GitPython', 'mwclient', 'paramiko', 'Pillow', diff --git a/tests/common.TestCase b/tests/common.TestCase index 48e1d29d..d7dca14e 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -64,7 +64,7 @@ class CommonTest(unittest.TestCase): def testIsApkDebuggable(self): config = dict() - config['sdk_path'] = os.getenv('ANDROID_HOME') + fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config self._set_build_tools() config['aapt'] = fdroidserver.common.find_sdk_tools_cmd('aapt') diff --git a/tests/import.TestCase b/tests/import.TestCase index cce85d68..c53b53d4 100755 --- a/tests/import.TestCase +++ b/tests/import.TestCase @@ -25,8 +25,9 @@ class ImportTest(unittest.TestCase): def test_import_gitlab(self): # FDroidPopen needs some config to work - fdroidserver.common.config = dict() - fdroidserver.common.config['sdk_path'] = '/fake/path/to/android-sdk' + config = dict() + fdroidserver.common.fill_config_defaults(config) + fdroidserver.common.config = config url = 'https://gitlab.com/fdroid/fdroidclient' app = fdroidserver.metadata.get_default_app_info() diff --git a/tests/install.TestCase b/tests/install.TestCase index d1ed93ff..ce516117 100755 --- a/tests/install.TestCase +++ b/tests/install.TestCase @@ -23,7 +23,7 @@ class InstallTest(unittest.TestCase): def test_devices(self): config = dict() - config['sdk_path'] = os.getenv('ANDROID_HOME') + fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config config['adb'] = fdroidserver.common.find_sdk_tools_cmd('adb') self.assertTrue(os.path.exists(config['adb'])) diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 9dfe2bbc..c287a212 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -33,7 +33,7 @@ class MetadataTest(unittest.TestCase): config = dict() config['sdk_path'] = '/opt/android-sdk' config['ndk_paths'] = dict() - config['accepted_formats'] = ['json', 'txt', 'xml', 'yaml'] + config['accepted_formats'] = ['json', 'txt', 'xml', 'yml'] fdroidserver.common.config = config apps = fdroidserver.metadata.read_metadata(xref=True) diff --git a/tests/metadata/org.videolan.vlc.pickle b/tests/metadata/org.videolan.vlc.pickle index 45af8916..a1810e60 100644 --- a/tests/metadata/org.videolan.vlc.pickle +++ b/tests/metadata/org.videolan.vlc.pickle @@ -4618,7 +4618,7 @@ sasS'FlattrID' p1324 NsS'metadatapath' p1325 -S'metadata/org.videolan.vlc.yaml' +S'metadata/org.videolan.vlc.yml' p1326 sS'Disabled' p1327 diff --git a/tests/metadata/org.videolan.vlc.yaml b/tests/metadata/org.videolan.vlc.yml similarity index 100% rename from tests/metadata/org.videolan.vlc.yaml rename to tests/metadata/org.videolan.vlc.yml diff --git a/tests/run-tests b/tests/run-tests index 8ec03c35..62d27aa7 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -163,7 +163,7 @@ cp $WORKSPACE/tests/metadata/org.smssecure.smssecure.txt $REPOROOT/metadata/ $fdroid readmeta # now make a fake duplicate -touch $REPOROOT/metadata/org.smssecure.smssecure.yaml +touch $REPOROOT/metadata/org.smssecure.smssecure.yml set +e $fdroid readmeta diff --git a/tests/update.TestCase b/tests/update.TestCase index 83349f5e..0cc93e5c 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -29,6 +29,10 @@ class UpdateTest(unittest.TestCase): if not os.path.exists(getsig_dir + "/getsig.class"): logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir) sys.exit(1) + # FDroidPopen needs some config to work + config = dict() + fdroidserver.common.fill_config_defaults(config) + fdroidserver.common.config = config p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'), 'getsig', os.path.join(os.getcwd(), apkfile)]) sig = None