diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dbfc219f..26c900ad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -84,7 +84,7 @@ debian_testing: - ./run-tests # Test using latest LTS set up with the PPA, including Recommends. -# bionic's apksigner, which comes from Recommends:, requires binfmt +# focal's apksigner, which comes from Recommends:, requires binfmt # support in the kernel. ubuntu_lts_ppa: image: ubuntu:latest @@ -106,13 +106,11 @@ ubuntu_lts_ppa: - cd tests - ./run-tests -# Test using Xenial LTS with all depends from pypi. The venv is used -# to isolate the dist tarball generation environment from the clean -# install environment. Xenial's pip is too old to install all the -# dependencies, so this has to uppgrade pip and setuptools in order to -# run the install. -ubuntu_xenial_pip: - image: ubuntu:xenial +# Test using Ubuntu/bionic LTS (supported til 2022) with all depends +# from pypi. The venv is used to isolate the dist tarball generation +# environment from the clean install environment. +ubuntu_bionic_pip: + image: ubuntu:bionic <<: *apt-template script: - apt-get install git default-jdk-headless python3-pip python3-venv rsync zipalign libarchive13 @@ -127,12 +125,11 @@ ubuntu_xenial_pip: - deactivate - tar tzf dist/fdroidserver-*.tar.gz | grep locale/de/LC_MESSAGES/fdroidserver.mo # back to bare machine to act as user's install machine - - $pip install --upgrade pip setuptools + - $pip install --upgrade pip setuptools wheel # make this go away: "error: invalid command 'bdist_wheel'" - $pip install dist/fdroidserver-*.tar.gz - - test -e /usr/share/locale/de/LC_MESSAGES/fdroidserver.mo - tar xzf dist/fdroidserver-*.tar.gz - cd fdroidserver-* - - ./tests/run-tests + - fdroid=`which fdroid` ./tests/run-tests # test install process on a bleeding edge distro with pip arch_pip_install: diff --git a/CHANGELOG.md b/CHANGELOG.md index b76fe549..7d6497e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,7 @@ milestone](https://gitlab.com/fdroid/fdroidserver/-/milestones/10) ([!692](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/692)) * Major refactoring on core parts of code to be more Pythonic ([!756](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/756)) +* `fdroid init` now works when installed with pip ### Removed * Removed all support for _.txt_ and _.json_ metadata diff --git a/MANIFEST.in b/MANIFEST.in index 48e4830a..4a14e8ad 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -13,7 +13,6 @@ include examples/fdroid_exportkeystore.py include examples/fdroid_export_keystore_to_nitrokey.py include examples/fdroid_extract_repo_pubkey.py include examples/fdroid_fetchsrclibs.py -include examples/fdroid-icon.png include examples/fdroid_nitrokeyimport.py include examples/makebuildserver.config.py include examples/opensc-fdroid.cfg diff --git a/examples/config.yml b/examples/config.yml index db93de9d..ac1e44a0 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -52,25 +52,36 @@ # The same policy is applied to the archive repo, if there is one. # repo_maxage: 0 -repo_url: https://MyFirstFDroidRepo.org/fdroid/repo -repo_name: My First F-Droid Repo Demo -repo_icon: fdroid-icon.png -repo_description: >- - This is a repository of apps to be used with F-Droid. Applications in this - repository are either official binaries built by the original application - developers, or are binaries built from source by the admin of f-droid.org - using the tools on https://gitlab.com/u/fdroid. +# repo_url: https://MyFirstFDroidRepo.org/fdroid/repo +# repo_name: My First F-Droid Repo Demo +# repo_description: >- +# This is a repository of apps to be used with F-Droid. Applications +# in this repository are either official binaries built by the +# original application developers, or are binaries built from source +# by the admin of f-droid.org using the tools on +# https://gitlab.com/fdroid. # As above, but for the archive repo. +# +# archive_url: https://f-droid.org/archive +# archive_name: My First F-Droid Archive Demo +# archive_description: >- +# The repository of older versions of packages from the main demo repository. + # archive_older sets the number of versions kept in the main repo, with all # older ones going to the archive. Set it to 0, and there will be no archive # repository, and no need to define the other archive_ values. -archive_older: 3 -archive_url: https://f-droid.org/archive -archive_name: My First F-Droid Archive Demo -archive_icon: fdroid-icon.png -archive_description: >- - The repository of older versions of packages from the main demo repository. +# +# archive_older: 3 + +# The repo's icon defaults to a file called 'icon.png' in the 'icons' +# folder for each section, e.g. repo/icons/icon.png and +# archive/icons/icon.png. To use a different filename for the icons, +# set the filename here. You must still copy it into place in +# repo/icons/ and/or archive/icons/. +# +# repo_icon: myicon.png +# archive_icon: myicon.png # This allows a specific kind of insecure APK to be included in the # 'repo' section. Since April 2017, APK signatures that use MD5 are diff --git a/examples/fdroid-icon.png b/examples/fdroid-icon.png deleted file mode 100644 index 0c0d4173..00000000 Binary files a/examples/fdroid-icon.png and /dev/null differ diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 22a4f3ff..3fe24c23 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -146,14 +146,10 @@ default_config = { }, 'keyaliases': {}, 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo", - 'repo_name': "My First FDroid Repo Demo", - 'repo_icon': "fdroid-icon.png", - 'repo_description': _(''' - This is a repository of apps to be used with FDroid. Applications in this - repository are either official binaries built by the original application - developers, or are binaries built from source by f-droid.org using the - tools on https://gitlab.com/fdroid. - '''), + 'repo_name': "My First F-Droid Repo Demo", + 'repo_icon': "icon.png", + 'repo_description': _("""This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/fdroid."""), + 'archive_name': 'My First F-Droid Archive Demo', 'archive_description': _('These are the apps that have been archived from the main repo.'), 'archive_older': 0, 'lint_licenses': fdroidserver.lint.APPROVED_LICENSES, @@ -228,7 +224,7 @@ def fill_config_defaults(thisconfig): return None return path - for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']: + for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore']: v = thisconfig[k] exp = expand_path(v) if exp is not None: diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 95b46ed5..d6d396a5 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -75,13 +75,14 @@ def make(apps, apks, repodir, archive): if archive: repodict['name'] = common.config['archive_name'] - repodict['icon'] = os.path.basename(common.config['archive_icon']) - repodict['address'] = common.config['archive_url'] + repodict['icon'] = common.config.get('archive_icon', common.default_config['repo_icon']) repodict['description'] = common.config['archive_description'] - urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path) + archive_url = common.config.get('archive_url', common.config['repo_url'][:-4] + 'archive') + repodict['address'] = archive_url + urlbasepath = os.path.basename(urllib.parse.urlparse(archive_url).path) else: repodict['name'] = common.config['repo_name'] - repodict['icon'] = os.path.basename(common.config['repo_icon']) + repodict['icon'] = common.config.get('repo_icon', common.default_config['repo_icon']) repodict['address'] = common.config['repo_url'] repodict['description'] = common.config['repo_description'] urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path) @@ -331,7 +332,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing doc.appendChild(root) repoel = doc.createElement("repo") - repoel.setAttribute("icon", os.path.basename(repodict['icon'])) + repoel.setAttribute("icon", repodict['icon']) if 'maxage' in repodict: repoel.setAttribute("maxage", str(repodict['maxage'])) repoel.setAttribute("name", repodict['name']) @@ -604,7 +605,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing if os.path.exists(repo_icon): shutil.copyfile(common.config['repo_icon'], iconfilename) else: - logging.warning(_('repo_icon %s does not exist, generating placeholder.') + logging.warning(_('repo_icon "repo/icons/%s" does not exist, generating placeholder.') % repo_icon) os.makedirs(os.path.dirname(iconfilename), exist_ok=True) try: diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 53eac796..7f9fc817 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -120,8 +120,17 @@ def main(): # 'metadata' and 'tmp' are created in fdroid if not os.path.exists('repo'): os.mkdir('repo') - shutil.copy(os.path.join(examplesdir, 'fdroid-icon.png'), fdroiddir) - shutil.copyfile(os.path.join(examplesdir, 'config.yml'), 'config.yml') + example_config_yml = os.path.join(examplesdir, 'config.yml') + if os.path.exists(example_config_yml): + shutil.copyfile(example_config_yml, 'config.yml') + else: + from pkg_resources import get_distribution + versionstr = get_distribution('fdroidserver').version + if not versionstr: + versionstr = 'master' + with open('config.yml', 'w') as fp: + fp.write('# see https://gitlab.com/fdroid/fdroidserver/blob/' + + versionstr + '/examples/config.yml\n') os.chmod('config.yml', 0o0600) # If android_home is None, test_config['sdk_path'] will be used and # "$ANDROID_HOME" may be used if the env var is set up correctly. diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 63e7d131..50239e20 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -192,7 +192,7 @@ def main(): readme = ''' # {repo_git_base} -[![{repo_url}](icon.png)]({repo_url}) +[![{repo_url}]({repo_url}/icons/icon.png)]({repo_url}) Last updated: {date}'''.format(repo_git_base=repo_git_base, repo_url=repo_url, @@ -202,16 +202,8 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, mirror_git_repo.git.add(all=True) mirror_git_repo.index.commit("update README") - icon_path = os.path.join(git_mirror_path, 'icon.png') - try: - import qrcode - qrcode.make(repo_url).save(icon_path) - except Exception: - exampleicon = os.path.join(common.get_examples_dir(), 'fdroid-icon.png') - shutil.copy(exampleicon, icon_path) mirror_git_repo.git.add(all=True) mirror_git_repo.index.commit("update repo/website icon") - shutil.copy(icon_path, repo_basedir) os.chdir(repo_basedir) if os.path.isdir(git_mirror_repodir): @@ -234,11 +226,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, config += "identity_file = '%s'\n" % ssh_private_key_file config += "repo_name = '%s'\n" % repo_git_base config += "repo_url = '%s'\n" % repo_url - config += "repo_icon = 'icon.png'\n" config += "repo_description = 'Nightly builds from %s'\n" % git_user_email config += "archive_name = '%s'\n" % (repo_git_base + ' archive') config += "archive_url = '%s'\n" % (repo_base + '/archive') - config += "archive_icon = 'icon.png'\n" config += "archive_description = 'Old nightly builds that have been archived.'\n" config += "archive_older = %i\n" % options.archive_older config += "servergitmirrors = '%s'\n" % servergitmirror diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 8679b134..983bdb79 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -2244,9 +2244,8 @@ def main(): for k in ['repo_icon', 'archive_icon']: if k in config: if not os.path.exists(config[k]): - logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.') - .format(name=k, path=config[k])) - sys.exit(1) + logging.warning(_('{name} "{section}/icons/{path}" does not exist! Check "config.yml".') + .format(name=k, section=k.split('_')[0], path=config[k])) # if the user asks to create a keystore, do it now, reusing whatever it can if options.create_key: diff --git a/tests/config.py b/tests/config.py index ead06f91..fde7e973 100644 --- a/tests/config.py +++ b/tests/config.py @@ -1,7 +1,6 @@ repo_url = "https://MyFirstFDroidRepo.org/fdroid/repo" repo_name = "My First F-Droid Repo Demo" -repo_icon = "fdroid-icon.png" repo_description = """ This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application @@ -12,7 +11,6 @@ using the tools on https://gitlab.com/u/fdroid. archive_older = 3 archive_url = "https://f-droid.org/archive" archive_name = "My First F-Droid Archive Demo" -archive_icon = "fdroid-icon.png" archive_description = """ The repository of older versions of applications from the main demo repository. """ diff --git a/tests/repo/index-v1.json b/tests/repo/index-v1.json index 5ad2ffe6..6ca6d559 100644 --- a/tests/repo/index-v1.json +++ b/tests/repo/index-v1.json @@ -3,9 +3,9 @@ "timestamp": 1502845383782, "version": 20000, "name": "My First F-Droid Repo Demo", - "icon": "fdroid-icon.png", + "icon": "icon.png", "address": "https://MyFirstFDroidRepo.org/fdroid/repo", - "description": "This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/u/fdroid.", + "description": "This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/fdroid.", "mirrors": [ "http://foobarfoobarfoobar.onion/fdroid/repo", "https://foo.bar/fdroid/repo" diff --git a/tests/repo/index.xml b/tests/repo/index.xml index 6f9ed101..69b09bcf 100644 --- a/tests/repo/index.xml +++ b/tests/repo/index.xml @@ -1,7 +1,7 @@ - - This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/u/fdroid. + + This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/fdroid. http://foobarfoobarfoobar.onion/fdroid/repo https://foo.bar/fdroid/repo diff --git a/tests/run-tests b/tests/run-tests index 935ceb8e..693c89e6 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -228,7 +228,6 @@ echo_header 'copy git import and run "fdroid scanner" on it' REPOROOT=`create_test_dir` cd $REPOROOT -cp $WORKSPACE/examples/fdroid-icon.png $REPOROOT/ mkdir metadata echo "AutoName: Just A Test" > metadata/org.fdroid.ci.test.app.yml echo "WebSite: " >> metadata/org.fdroid.ci.test.app.yml @@ -310,7 +309,7 @@ cp $WORKSPACE/tests/urzip.apk \ $WORKSPACE/tests/repo/com.politedroid_[0-9].apk \ $WORKSPACE/tests/repo/obb.main.twoversions_110161[357].apk \ repo/ -$sed -i.tmp 's,archive_older: [0-9],archive_older: 3,' config.yml +printf '\narchive_older: 3\n' >> config.yml $fdroid update --pretty --nosign if which apksigner; then @@ -333,7 +332,7 @@ if ! which apksigner; then cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata/ test -d repo || mkdir repo cp $WORKSPACE/tests/repo/com.politedroid_[0-9].apk repo/ - $sed -i.tmp 's,archive_older: [0-9],archive_older: 3,' config.yml + printf '\narchive_older: 3\n' >> config.yml $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 0 @@ -419,7 +418,7 @@ cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata/ $sed -i.tmp '/ArchivePolicy:/d' metadata/com.politedroid.yml test -d repo || mkdir repo cp $WORKSPACE/tests/repo/com.politedroid_[0-9].apk repo/ -$sed -i.tmp 's,archive_older: [0-9],archive_older: 3,' config.yml +printf '\narchive_older: 3\n' >> config.yml $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 1 @@ -495,7 +494,7 @@ REPOROOT=`create_test_dir` cd $REPOROOT fdroid_init_with_prebuilt_keystore echo 'allow_disabled_algorithms: true' >> config.yml -$sed -i.tmp 's,archive_older: [0-9],archive_older: 3,' config.yml +printf '\narchive_older: 3\n' >> config.yml test -d metadata || mkdir metadata cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata/ echo 'Summary: good test version of urzip' > metadata/info.guardianproject.urzip.yml @@ -616,6 +615,7 @@ echo_header "test for added date being set correctly for repo and archive" REPOROOT=`create_test_dir` cd $REPOROOT fdroid_init_with_prebuilt_keystore +printf '\narchive_older: 3\n' >> config.yml mkdir -p {repo,archive,metadata,stats} cp $WORKSPACE/tests/repo/com.politedroid_5.apk archive cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo @@ -942,7 +942,6 @@ echo_header "setup a new repo manually and generate a keystore" REPOROOT=`create_test_dir` KEYSTORE=$REPOROOT/keystore.p12 cd $REPOROOT -cp $WORKSPACE/examples/fdroid-icon.png $REPOROOT/ ! test -e $KEYSTORE set +e $fdroid update @@ -1127,6 +1126,7 @@ REPOROOT=`create_test_dir` GIT_MIRROR=$REPOROOT/git-mirror cd $REPOROOT fdroid_init_with_prebuilt_keystore +printf '\narchive_older: 3\n' >> config.yml echo "servergitmirrors: $SERVER_GIT_MIRROR" >> config.yml cp $WORKSPACE/tests/repo/com.politedroid_[345].apk repo/ @@ -1195,6 +1195,7 @@ if have_git_2_3; then cd $OFFLINE_ROOT fdroid_init_with_prebuilt_keystore + printf '\narchive_older: 3\n' >> config.yml cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $OFFLINE_ROOT/ echo "mirrors: ['http://foo.bar/fdroid', 'http://asdflkdsfjafdsdfhkjh.onion/fdroid']" >> config.yml @@ -1284,6 +1285,7 @@ if which wget; then # clean up kill -9 $http_server_pid rm -f 127.0.0.1\:${port}/repo/*.apk 127.0.0.1\:${port}/repo/*/*/*/*.png + sleep 1 # wait for webserver thread to quit else echo "WARNING: wget not installed, skipping" fi diff --git a/tests/update.TestCase b/tests/update.TestCase index ffc8fa5f..19b7436e 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -830,8 +830,17 @@ class UpdateTest(unittest.TestCase): # yaml.add_representer(fdroidserver.metadata.Build, _build_yaml_representer) # yaml.dump(apk, f, default_flow_style=False) + # CFullLoader doesn't always work + # https://github.com/yaml/pyyaml/issues/266#issuecomment-559116876 + TestLoader = FullLoader + try: + testyaml = '- !!python/object/new:fdroidserver.update.UsesPermission\n - test\n - null' + from_yaml = yaml.load(testyaml, Loader=TestLoader) + except yaml.constructor.ConstructorError: + from yaml import UnsafeLoader as TestLoader + with open(savepath, 'r') as f: - from_yaml = yaml.load(f, Loader=FullLoader) + from_yaml = yaml.load(f, Loader=TestLoader) self.maxDiff = None self.assertEqual(apk, from_yaml)