From 10fa912c16e5b1df105792d7bbd869a77b28ab2d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 9 Jul 2020 09:34:04 +0200 Subject: [PATCH 1/4] deploy: remove git remotes not enabled in servergitmirrors If the repo operator removes an item from servergitmirrors, it should auto- matically remove the remote. --- fdroidserver/server.py | 14 ++++++++++---- tests/server.TestCase | 13 +++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index cc930eb6..d406da1d 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -45,6 +45,7 @@ BINARY_TRANSPARENCY_DIR = 'binary_transparency' AUTO_S3CFG = '.fdroid-server-update-s3cfg' USER_S3CFG = 's3cfg' +REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*') def update_awsbucket(repo_section): @@ -384,15 +385,17 @@ def update_servergitmirrors(servergitmirrors, repo_section): repo = git.Repo.init(git_mirror_path) + enabled_remotes = [] for remote_url in servergitmirrors: - hostname = re.sub(r'\W*\w+\W+(\w+).*', r'\1', remote_url) - r = git.remote.Remote(repo, hostname) + name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) + enabled_remotes.append(name) + r = git.remote.Remote(repo, name) if r in repo.remotes: - r = repo.remote(hostname) + r = repo.remote(name) if 'set_url' in dir(r): # force remote URL if using GitPython 2.x r.set_url(remote_url) else: - repo.create_remote(hostname, remote_url) + repo.create_remote(name, remote_url) logging.info('Mirroring to: ' + remote_url) # sadly index.add don't allow the --all parameter @@ -414,6 +417,9 @@ def update_servergitmirrors(servergitmirrors, repo_section): # push for every remote. This will overwrite the git history for remote in repo.remotes: + if remote.name not in enabled_remotes: + repo.delete_remote(remote) + continue if remote.name == 'gitlab': logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages') with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as out_file: diff --git a/tests/server.TestCase b/tests/server.TestCase index 8bd769ea..4236d6e5 100755 --- a/tests/server.TestCase +++ b/tests/server.TestCase @@ -148,6 +148,19 @@ class ServerTest(unittest.TestCase): virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY') fdroidserver.server.upload_to_virustotal('repo', virustotal_apikey) + def test_remote_hostname_regex(self): + for remote_url, name in ( + ('git@github.com:guardianproject/fdroid-repo', 'github'), + ('git@gitlab.com:guardianproject/fdroid-repo', 'gitlab'), + ('https://github.com:guardianproject/fdroid-repo', 'github'), + ('https://gitlab.com/guardianproject/fdroid-repo', 'gitlab'), + ('https://salsa.debian.org/foo/repo', 'salsa'), + ): + self.assertEqual( + name, + fdroidserver.server.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) + ) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 2858c73b871c539d67766309568cf78937e6e210 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 30 Jul 2020 14:55:46 +0200 Subject: [PATCH 2/4] gradlew-fdroid: more robust method for handling Windows linefeeds The bash pattern matching was getting confused by the Windows linefeeds, this strips them before the string hits the bash pattern tricks. * https://github.com/premnirmal/StockTicker/issues/145 --- gradlew-fdroid | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gradlew-fdroid b/gradlew-fdroid index 9e76fb35..a0b02d24 100755 --- a/gradlew-fdroid +++ b/gradlew-fdroid @@ -167,10 +167,10 @@ v_all=${plugin_v[@]} for f in {.,..}/gradle/wrapper/gradle-wrapper.properties; do [[ -f $f ]] || continue while IFS='' read -r line || [ -n "$line" ]; do + line=$(printf $line | tr -d '\r') # strip Windows linefeeds if [[ $line == 'distributionUrl='* ]]; then wrapper_ver=${line#*/gradle-} wrapper_ver=${wrapper_ver%-*.zip} - wrapper_ver=$(printf $wrapper_ver | tr -d '\r') # strip Windows linefeeds break 2 fi done < $f @@ -186,14 +186,13 @@ fi for f in {.,..}/build.gradle{,.kts}; do [[ -f $f ]] || continue while IFS='' read -r line || [ -n "$line" ]; do + line=$(printf $line | tr -d '\r') # strip Windows linefeeds if [[ -z "$plugin_pver" && $line == *'com.android.tools.build:gradle:'* ]]; then plugin_pver=${line#*[\'\"]com.android.tools.build:gradle:} plugin_pver=${plugin_pver%[\'\"]*} - plugin_pver=$(printf $plugin_pver | tr -d '\r') # strip Windows linefeeds elif [[ -z "$wrapper_ver" && $line == *'gradleVersion = '* ]]; then wrapper_ver=${line#*gradleVersion*=*[\'\"]} wrapper_ver=${wrapper_ver%[\'\"]*} - wrapper_ver=$(printf $wrapper_ver | tr -d '\r') # strip Windows linefeeds fi done < $f done From 960d31af2a27625cb0d1f0dac9b2fe9c9a92ba06 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 7 Jul 2020 10:20:03 +0200 Subject: [PATCH 3/4] import: standardize on 'appid' var name for Application ID The rest of the fdroidserver code uses 'appid' or 'packageName'. The official Android name is "Application ID". "Package Name" is the Java term, and it is used in Android in Java code. --- fdroidserver/import.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fdroidserver/import.py b/fdroidserver/import.py index a6098202..c7286e08 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -127,9 +127,9 @@ def main(): paths = common.get_all_gradle_and_manifests(tmp_importer_dir) subdir = common.get_gradle_subdir(tmp_importer_dir, paths) if paths: - versionName, versionCode, package = common.parse_androidmanifests(paths, app) - if not package: - raise FDroidException(_("Couldn't find package ID")) + versionName, versionCode, appid = common.parse_androidmanifests(paths, app) + if not appid: + raise FDroidException(_("Couldn't find Application ID")) if not versionName: logging.warning(_('Could not find latest version name')) if not versionCode: @@ -138,8 +138,8 @@ def main(): raise FDroidException(_("No gradle project could be found. Specify --subdir?")) # Make sure it's actually new... - if package in apps: - raise FDroidException("Package " + package + " already exists") + if appid in apps: + raise FDroidException(_('Package "{appid}" already exists').format(appid=appid)) # Create a build line... build.versionName = versionName or 'Unknown' @@ -205,17 +205,17 @@ def main(): # Keep the repo directory to save bandwidth... if not os.path.exists('build'): os.mkdir('build') - build_dir = os.path.join('build', package) + build_dir = os.path.join('build', appid) if os.path.exists(build_dir): logging.warning(_('{path} already exists, ignoring import results!') .format(path=build_dir)) sys.exit(1) elif tmp_importer_dir is not None: shutil.move(tmp_importer_dir, build_dir) - with open('build/.fdroidvcs-' + package, 'w') as f: + with open('build/.fdroidvcs-' + appid, 'w') as f: f.write(app.RepoType + ' ' + app.Repo) - metadatapath = os.path.join('metadata', package + '.yml') + metadatapath = os.path.join('metadata', appid + '.yml') metadata.write_metadata(metadatapath, app) logging.info("Wrote " + metadatapath) From a6c60a908aae94c146c3ab03c91751c58425b1f3 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 23 Jul 2020 13:03:20 +0200 Subject: [PATCH 4/4] import: add support for kivy buildozer projects --- fdroidserver/import.py | 43 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/fdroidserver/import.py b/fdroidserver/import.py index c7286e08..48af3b9c 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import configparser import git import json import os @@ -58,6 +59,44 @@ def clone_to_tmp_dir(app): return tmp_dir +def check_for_kivy_buildozer(tmp_importer_dir, app, build): + versionCode = None + buildozer_spec = os.path.join(tmp_importer_dir, 'buildozer.spec') + if os.path.exists(buildozer_spec): + config = configparser.ConfigParser() + config.read(buildozer_spec) + import pprint + pprint.pprint(sorted(config['app'].keys())) + app.id = config['app'].get('package.domain') + print(app.id) + app.AutoName = config['app'].get('package.name', app.AutoName) + app.License = config['app'].get('license', app.License) + app.Description = config['app'].get('description', app.Description) + build.versionName = config['app'].get('version') + build.output = 'bin/%s-$$VERSION$$-release-unsigned.apk' % app.AutoName + build.ndk = 'r17c' + build.srclibs = [ + 'buildozer@586152c', + 'python-for-android@ccb0f8e1', + ] + build.sudo = [ + 'apt-get update', + 'apt-get install -y build-essential libffi-dev libltdl-dev', + ] + build.prebuild = [ + 'sed -iE "/^[# ]*android\\.(ant|ndk|sdk)_path[ =]/d" buildozer.spec', + 'sed -iE "/^[# ]*android.accept_sdk_license[ =]+.*/d" buildozer.spec', + 'sed -iE "/^[# ]*android.skip_update[ =]+.*/d" buildozer.spec', + 'sed -iE "/^[# ]*p4a.source_dir[ =]+.*/d" buildozer.spec', + 'sed -i "s,\\[app\\],[app]\\n\\nandroid.sdk_path = $$SDK$$\\nandroid.ndk_path = $$NDK$$\\np4a.source_dir = $$python-for-android$$\\nandroid.accept_sdk_license = False\\nandroid.skip_update = True\\nandroid.ant_path = /usr/bin/ant\\n," buildozer.spec', + 'pip3 install --user --upgrade $$buildozer$$ Cython==0.28.6', + ] + build.build = [ + 'PATH="$HOME/.local/bin:$PATH" buildozer android release', + ] + return build.get('versionName'), versionCode, app.get('id') + + def main(): global config, options @@ -123,6 +162,8 @@ def main(): app.UpdateCheckMode = 'Tags' build.commit = common.get_head_commit_id(git_repo) + versionName, versionCode, appid = check_for_kivy_buildozer(tmp_importer_dir, app, build) + # Extract some information... paths = common.get_all_gradle_and_manifests(tmp_importer_dir) subdir = common.get_gradle_subdir(tmp_importer_dir, paths) @@ -134,7 +175,7 @@ def main(): logging.warning(_('Could not find latest version name')) if not versionCode: logging.warning(_('Could not find latest version code')) - else: + elif not appid: raise FDroidException(_("No gradle project could be found. Specify --subdir?")) # Make sure it's actually new...