.bandit

@@ -1,3 +1,3 @@
 [bandit]
-skips: B110,B404,B408,B603,B607,B322
+skips: B110,B404,B408,B410,B603,B607
 targets: .

.gitignore

@@ -27,6 +27,7 @@ tmp/
 /tests/repo/status
 
 # files used in manual testing
+/config.py
 /config.yml
 /tmp/
 /logs/

.gitlab-ci.yml

@@ -1,22 +1,5 @@
 ---
 
-# Use merge request pipelines when a merge request is open for the branch.
-# Use branch pipelines when a merge request is not open for the branch.
-# https://docs.gitlab.com/ci/yaml/workflow/#switch-between-branch-pipelines-and-merge-request-pipelines
-workflow:
-  rules:
-    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
-    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
-      when: never
-    - if: $CI_COMMIT_BRANCH
-
-
-stages:
-  - lint
-  - test  # default for jobs that do not specify stage:
-  - deploy
-
-
 variables:
   pip: pip3 --timeout 100 --retries 10
   # speed up git checkout phase
@@ -67,7 +50,6 @@ metadata_v0:
     - git checkout $RELEASE_COMMIT_ID
     - cd ..
     - git clone --depth 1 https://gitlab.com/fdroid/fdroiddata.git
-    - rm -f fdroiddata/config.yml  # ignore config for this test
     - cd fdroiddata
     - ../tests/dump_internal_metadata_format.py
     - cd ..
@@ -77,7 +59,6 @@ metadata_v0:
     - ../tests/dump_internal_metadata_format.py
     - sed -i
           -e '/ArchivePolicy:/d'
-          -e '/FlattrID:/d'
           -e '/RequiresRoot:/d'
           metadata/dump_*/*.yaml
     - diff -uw metadata/dump_*
@@ -98,31 +79,12 @@ metadata_v0:
     # Ubuntu and other distros often lack https:// support
     - grep Debian /etc/issue.net
         && { find /etc/apt/sources.list* -type f | xargs sed -i s,http:,https:, ; }
-    # The official Debian docker images ship without ca-certificates,
-    # TLS certificates cannot be verified until that is installed. The
-    # following code turns off TLS verification, and enables HTTPS, so
-    # at least unverified TLS is used for apt-get instead of plain
-    # HTTP.  Once ca-certificates is installed, the CA verification is
-    # enabled by removing this config.  This set up makes the initial
-    # `apt-get update` and `apt-get install` look the same as verified
-    # TLS to the network observer and hides the metadata.
     - echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/99nocacertificates
     - apt-get update
     - apt-get install ca-certificates
     - rm /etc/apt/apt.conf.d/99nocacertificates
     - apt-get dist-upgrade
 
-# For jobs that only need to run when there are changes to Python files.
-.python-rules-changes: &python-rules-changes
-  rules:
-    - changes:
-        - .gitlab-ci.yml
-        - fdroid
-        - makebuildserver
-        - setup.py
-        - fdroidserver/*.py
-        - tests/*.py
-
 
 # Since F-Droid uses Debian as its default platform, from production
 # servers to CI to contributor machines, it is important to know when
@@ -131,8 +93,8 @@ metadata_v0:
 debian_testing:
   image: debian:testing
   <<: *apt-template
-  rules:
+  only:
-    - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver"
+    - master@fdroid/fdroidserver
   script:
     - apt-get install
         aapt
@@ -145,7 +107,6 @@ debian_testing:
         ipfs-cid
         python3-biplist
         python3-defusedxml
-        python3-libcloud
         python3-pycountry
         python3-setuptools
         sdkmanager
@@ -160,8 +121,8 @@ debian_testing:
 ubuntu_lts_ppa:
   image: ubuntu:latest
   <<: *apt-template
-  rules:
+  only:
-    - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver"
+    - master@fdroid/fdroidserver
   script:
     - export ANDROID_HOME=/usr/lib/android-sdk
     - apt-get install gnupg
@@ -186,46 +147,6 @@ ubuntu_lts_ppa:
     - ./run-tests
 
 
-# Test to see how rclone works with S3
-test_deploy_to_s3_with_rclone:
-  image: debian:bookworm-slim
-  <<: *apt-template
-  tags:
-    - saas-linux-small-amd64  # the shared runners are known to support Docker.
-  services:
-    - name: docker:dind
-      command: ["--tls=false"]
-  variables:
-    DOCKER_HOST: "tcp://docker:2375"
-    DOCKER_DRIVER: overlay2
-    DOCKER_TLS_CERTDIR: ""
-  before_script:
-    # ensure minio is up before executing tests
-    - apt-get update
-    - apt-get install -y
-        androguard
-        apksigner
-        curl
-        docker.io
-        git
-        python3-venv
-        rclone
-    # This job requires working docker but will silently fail if docker is not available
-    - docker info
-    - python3 -m venv --system-site-packages test-venv
-    - . test-venv/bin/activate
-    - pip install testcontainers[minio]
-    - pip install .
-  script:
-    - python3 -m unittest -k test_update_remote_storage_with_rclone --verbose
-  rules:
-    - changes:
-        - .gitlab-ci.yml
-        - fdroidserver/deploy.py
-        - tests/test_deploy.py
-        - tests/test_integration.py
-
-
 # Test using Ubuntu/jammy LTS (supported til April, 2027) with depends
 # from pypi and sdkmanager.  The venv is used to isolate the dist
 # tarball generation environment from the clean install environment.
@@ -245,7 +166,7 @@ ubuntu_jammy_pip:
     # back to bare machine to act as user's install machine
     - export ANDROID_HOME=/opt/android-sdk
     - $pip install sdkmanager
-    - sdkmanager 'build-tools;35.0.0'
+    - sdkmanager 'build-tools;33.0.0'
 
     # Install extras_require.optional from setup.py
     - $pip install biplist pycountry
@@ -253,16 +174,46 @@ ubuntu_jammy_pip:
     - $pip install dist/fdroidserver-*.tar.gz
     - tar xzf dist/fdroidserver-*.tar.gz
     - cd fdroidserver-*
-    - export PATH=$PATH:$ANDROID_HOME/build-tools/35.0.0
+    - export PATH=$PATH:$ANDROID_HOME/build-tools/33.0.0
     - fdroid=`which fdroid` ./tests/run-tests
 
     # check localization was properly installed
     - LANGUAGE='de' fdroid --help | grep 'Gültige Befehle sind'
 
 
+# test installation process on a bleeding edge distro with pip
+arch_pip_install:
+  image: archlinux
+  only:
+    - master@fdroid/fdroidserver
+  script:
+    - pacman --sync --sysupgrade --refresh --noconfirm gcc git grep python-pip python-virtualenv python-wheel tar
+    - python -m venv venv
+    - source venv/bin/activate
+    - pip install -e .[test]
+    - fdroid
+    - fdroid readmeta
+    - fdroid update --help
+
+
+# The gradlew-fdroid tests are isolated from the rest of the test
+# suite, so they run as their own job.
+gradlew-fdroid:
+  image: debian:bookworm-slim
+  <<: *apt-template
+  only:
+    changes:
+      - .gitlab-ci.yml
+      - gradlew-fdroid
+      - tests/test-gradlew-fdroid
+  script:
+    - apt-get install ca-certificates curl default-jdk-headless shellcheck unzip
+    - shellcheck --severity=error --color gradlew-fdroid tests/test-gradlew-fdroid
+    - ./tests/test-gradlew-fdroid
+
+
 # Run all the various linters and static analysis tools.
-hooks/pre-commit:
+lint_format_bandit_checks:
-  stage: lint
   image: debian:bookworm-slim
   variables:
     LANG: C.UTF-8
@@ -277,60 +228,35 @@ hooks/pre-commit:
           make
           pycodestyle
           pyflakes3
+          pylint
           python3-dev
           python3-git
           python3-nose
           python3-pip
           python3-yaml
-    - ./hooks/pre-commit
+          shellcheck
-
-bandit:
-  image: debian:bookworm-slim
-  <<: *python-rules-changes
-  <<: *apt-template
-  script:
-    - apt-get install python3-pip
     - $pip install --break-system-packages bandit
-    - bandit -r -ii --ini .bandit
+    - export EXITVALUE=0
-
+    - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
-pylint:
+    - ./hooks/pre-commit || set_error
-  stage: lint
+    - bandit
-  image: debian:bookworm-slim
+        -r
-  <<: *python-rules-changes
+        -ii
-  <<: *apt-template
+        --ini .bandit
-  script:
+        || set_error
-    - apt-get install pylint python3-pip
+    - pylint --output-format=colorized --reports=n
-    - $pip install --break-system-packages pylint-gitlab
-    - pylint --output-format=colorized,pylint_gitlab.GitlabCodeClimateReporter:pylint-report.json
             fdroid
             makebuildserver
             setup.py
             fdroidserver/*.py
             tests/*.py
-  artifacts:
+            tests/*.TestCase
-    reports:
+        || set_error
-      codequality: pylint-report.json
+    - shellcheck --exclude SC2046,SC2090 --severity=warning --color tests/run-tests
-    when: always
+        || set_error
+    - exit $EXITVALUE
 
 
-shellcheck:
-  stage: lint
-  image: debian:bookworm-slim
-  rules:
-    - changes:
-        - .gitlab-ci.yml
-        - hooks/install-hooks.sh
-        - hooks/pre-commit
-        - tests/run-tests
-  <<: *apt-template
-  script:
-    - apt-get install shellcheck
-    # TODO GitLab Code Quality report https://github.com/koalaman/shellcheck/issues/3155
-    - shellcheck --exclude SC2046,SC2090 --severity=warning --color
-        hooks/install-hooks.sh
-        hooks/pre-commit
-        tests/run-tests
-
 # Check all the dependencies in Debian to mirror production.  CVEs are
 # generally fixed in the latest versions in pip/pypi.org, so it isn't
 # so important to scan that kind of install in CI.
@@ -338,7 +264,10 @@ shellcheck:
 safety:
   image: debian:bookworm-slim
   rules:
-    - if: $SAFETY_API_KEY
+    # once only:/changes: are ported to rules:, this could be removed:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "push" && $SAFETY_API_KEY
       changes:
         - .gitlab-ci.yml
         - .safety-policy.yml
@@ -359,34 +288,8 @@ safety:
     - python3 -m safety --key "$SAFETY_API_KEY" --stage cicd scan
 
 
-# TODO tests/*/*/*.yaml are not covered
+# Run all the various linters and static analysis tools.
-yamllint:
-  stage: lint
-  image: debian:bookworm-slim
-  rules:
-    - changes:
-        - .gitlab-ci.yml
-        - .safety-policy.yml
-        - .yamllint
-        - tests/*.yml
-        - tests/*/*.yml
-        - tests/*/*/.*.yml
-  <<: *apt-template
-  variables:
-    LANG: C.UTF-8
-  script:
-    - apt-get install yamllint
-    - yamllint
-        .gitlab-ci.yml
-        .safety-policy.yml
-        .yamllint
-        tests/*.yml
-        tests/*/*.yml
-        tests/*/*/.*.yml
-
-
 locales:
-  stage: lint
   image: debian:bookworm-slim
   variables:
     LANG: C.UTF-8
@@ -405,7 +308,6 @@ locales:
 
 
 black:
-  stage: lint
   image: debian:bookworm-slim
   <<: *apt-template
   script:
@@ -452,14 +354,14 @@ fedora_latest:
     - chown -R testuser .
     - cd tests
     - su testuser --login --command
-        "cd `pwd`; export CI=$CI ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"
+        "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"
 
 
 macOS:
   tags:
     - saas-macos-medium-m1
-  rules:
+  only:
-    - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver"
+    - master@fdroid/fdroidserver
   script:
     - export HOMEBREW_CURL_RETRIES=10
     - brew update > /dev/null
@@ -470,7 +372,7 @@ macOS:
     - brew install --cask android-commandlinetools temurin  # temurin is a JDK
 
     # test suite dependencies
-    - brew install bash coreutils gnu-sed
+    - brew install dash bash coreutils gnu-sed
     # TODO port tests/run-tests to POSIX and gsed, it has a couple GNU-isms like du --bytes
     - export PATH="$(brew --prefix fdroidserver)/libexec/bin:$(brew --prefix coreutils)/libexec/gnubin:$PATH"
 
@@ -487,34 +389,48 @@ macOS:
 
     - echo "macOS sticks with bash 3.x because of licenses, so avoid new bash syntax"
     - /bin/bash --version
-    - /bin/bash -n tests/run-tests
+    - /bin/bash -n gradlew-fdroid tests/run-tests
+
+    # TODO remove the packages below once they are included in the Homebrew package
+    - $(brew --prefix fdroidserver)/libexec/bin/python3 -m pip install biplist oscrypto pycountry
 
     # test fdroidserver from git with current package's dependencies
     - fdroid="$(brew --prefix fdroidserver)/libexec/bin/python3 $PWD/fdroid" ./tests/run-tests
 
 
 gradle:
-  image: debian:trixie-slim
+  image: debian:bookworm-slim
   <<: *apt-template
-  rules:
+  variables:
-    - changes:
+    GIT_DEPTH: 1000
-        - .gitlab-ci.yml
-        - makebuildserver
   script:
     - apt-get install
           ca-certificates
           git
+          openssh-client
+          python3-bs4
           python3-colorama
+          python3-git
+          python3-gitlab
           python3-packaging
           python3-requests
+    # if this is a merge request fork, then only check if relevant files changed
+    - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then
+        git fetch https://gitlab.com/fdroid/fdroidserver.git;
+        for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do
+           test "$f" == "makebuildserver" && export CHANGED="yes";
+           test "$f" == "gradlew-fdroid" && export CHANGED="yes";
+        done;
+        test -z "$CHANGED" && exit;
+      fi
     - ./tests/gradle-release-checksums.py
 
 
 # Run an actual build in a simple, faked version of the buildserver guest VM.
 fdroid build:
   image: registry.gitlab.com/fdroid/fdroidserver:buildserver
-  rules:
+  only:
-    - changes:
+    changes:
       - .gitlab-ci.yml
       - fdroidserver/build.py
       - fdroidserver/common.py
@@ -523,9 +439,6 @@ fdroid build:
       - fdroidserver/net.py
       - fdroidserver/scanner.py
       - fdroidserver/vmtools.py
-        # for the docker: job which depends on this one
-        - makebuildserver
-        - buildserver/*
   cache:
     key: "$CI_JOB_NAME"
     paths:
@@ -561,8 +474,6 @@ fdroid build:
           env HOME=$home_vagrant
           fdroid"
 
-    - git -C $home_vagrant/gradlew-fdroid pull
-
     - chown -R vagrant $home_vagrant
     - chown -R vagrant $fdroidserver/.git
     - chown vagrant $fdroidserver/
@@ -588,8 +499,8 @@ fdroid build:
 plugin_fetchsrclibs:
   image: debian:bookworm-slim
   <<: *apt-template
-  rules:
+  only:
-    - changes:
+    changes:
       - .gitlab-ci.yml
       - examples/fdroid_fetchsrclibs.py
       - fdroidserver/__main__.py
@@ -632,8 +543,8 @@ plugin_fetchsrclibs:
 servergitmirrors:
   image: debian:bookworm-slim
   <<: *apt-template
-  rules:
+  only:
-    - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver"
+    - master@fdroid/fdroidserver
   script:
     - apt-get install
         default-jdk-headless
@@ -661,11 +572,11 @@ servergitmirrors:
     - ./tests/key-tricks.py
     - ssh-keyscan gitlab.com >> /root/.ssh/known_hosts
     - test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo
-    - cp tests/config.yml tests/keystore.jks /tmp/fdroid/
+    - cp tests/config.py tests/keystore.jks /tmp/fdroid/
     - cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/
     - cd /tmp/fdroid
     - touch fdroid-icon.png
-    - printf "\nservergitmirrors\x3a 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.yml
+    - printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py
     - $PYTHONPATH/fdroid update --verbose --create-metadata
     - $PYTHONPATH/fdroid deploy --verbose
     - export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json`
@@ -675,7 +586,6 @@ servergitmirrors:
 
 Build documentation:
   image: debian:bookworm-slim
-  <<: *python-rules-changes
   <<: *apt-template
   script:
     - apt-get install make python3-sphinx python3-numpydoc python3-pydata-sphinx-theme pydocstyle fdroidserver
@@ -695,8 +605,8 @@ Build documentation:
 Windows:
   tags:
     - windows
-  rules:
+  only:
-    - if: $CI_COMMIT_BRANCH == "windows"
+    - windows
   script:
     - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
     - choco install --no-progress -y git --force --params "/GitAndUnixToolsOnPath"
@@ -707,10 +617,10 @@ Windows:
     - python -m pip install --upgrade babel pip setuptools
     - python -m pip install -e .
 
-    - $files = @(Get-ChildItem tests\test_*.py)
+    - $files = @(Get-ChildItem tests\*.TestCase)
     - foreach ($f in $files) {
           write-output $f;
-          python -m unittest $f;
+          python $f;
           if( $LASTEXITCODE -eq 0 ) {
               write-output "SUCCESS $f";
           } else {
@@ -719,16 +629,15 @@ Windows:
       }
 
     # these are the tests that must pass
-    - python -m unittest -k
+    - python tests\checkupdates.TestCase
-        checkupdates
+    - python tests\exception.TestCase
-        exception
+    - python tests\import_subcommand.TestCase
-        import_subcommand
+    - python tests\init.TestCase
-        test_lint
+    - python tests\lint.TestCase
-        test_metadata
+    - python tests\main.TestCase
-        test_rewritemeta
+    - python tests\metadata.TestCase
-        test_vcs
+    - python tests\rewritemeta.TestCase
-        tests.test_init
+    - python tests\vcs.TestCase
-        tests.test_main
   after_script:
     - Copy-Item C:\ProgramData\chocolatey\logs\chocolatey.log
   artifacts:
@@ -747,9 +656,7 @@ pages:
   artifacts:
     paths:
       - public
-  needs:
+  needs: ["Build documentation"]
-    - job: "Build documentation"
-      optional: true
   rules:
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'  # only publish pages on default (master) branch
 
@@ -761,12 +668,13 @@ pages:
 docker:
   dependencies:
     - fdroid build
-  rules:
+  only:
-    - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver"
     changes:
       - .gitlab-ci.yml
       - makebuildserver
       - buildserver/*
+    variables:
+      - $CI_COMMIT_BRANCH == "master" || $CI_PROJECT_NAMESPACE != "fdroid"
   image: docker:dind
   services:
     - docker:dind
@@ -789,66 +697,3 @@ docker:
       fi
     - docker push $RELEASE_IMAGE
     - docker push $RELEASE_IMAGE-bookworm
-
-
-# PUBLISH is the signing server.  It has a very minimal manual setup.
-PUBLISH:
-  image: debian:bookworm-backports
-  <<: *python-rules-changes
-  script:
-    - apt-get update
-    - apt-get -qy upgrade
-    - apt-get -qy install --no-install-recommends -t bookworm-backports
-        androguard
-        apksigner
-        curl
-        default-jdk-headless
-        git
-        gpg
-        gpg-agent
-        python3-asn1crypto
-        python3-defusedxml
-        python3-git
-        python3-ruamel.yaml
-        python3-yaml
-        rsync
-
-    # Run only relevant parts of the test suite, other parts will fail
-    # because of this minimal base setup.
-    - python3 -m unittest
-        tests/test_gpgsign.py
-        tests/test_metadata.py
-        tests/test_publish.py
-        tests/test_signatures.py
-        tests/test_signindex.py
-
-    - cd tests
-    - mkdir archive
-    - mkdir unsigned
-    - cp urzip-release-unsigned.apk unsigned/info.guardianproject.urzip_100.apk
-    - grep '^key.*pass' config.yml | sed 's,\x3a ,=,' > $CI_PROJECT_DIR/variables
-    - sed -Ei 's,^(key.*pass|keystore)\x3a.*,\1\x3a {env\x3a \1},' config.yml
-    - printf '\ngpghome\x3a {env\x3a gpghome}\n' >> config.yml
-    - |
-      tee --append $CI_PROJECT_DIR/variables <<EOF
-          gpghome=$CI_PROJECT_DIR/tests/gnupghome
-          keystore=$CI_PROJECT_DIR/tests/keystore.jks
-          serverwebroot=/tmp
-          export gpghome keypass keystorepass keystore serverwebroot
-      EOF
-    - source $CI_PROJECT_DIR/variables
-    # silence warnings
-    - chmod 0600 config.yml config/*.yml config/*/*.yml
-    - chmod 0700 $gpghome
-
-    - export PATH=$CI_PROJECT_DIR:$PATH
-
-    # run signpkg.sh
-    - fdroid publish --verbose
-    - fdroid gpgsign --verbose
-    - rsync --progress repo/* $serverwebroot/
-
-    # run signindex.sh
-    - fdroid gpgsign --verbose
-    - fdroid signindex --verbose
-    - rsync --stats repo/* $serverwebroot/

.well-known/funding-manifest-urls

@@ -1 +0,0 @@
-https://f-droid.org/funding.json

CHANGELOG.md

@@ -4,203 +4,16 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
-## [2.5.0] - NEXT
+## unreleased
-
-### Removed
-
-* deploy: `awsaccesskeyid:` and `awssecretkey:` config items removed, use the
-  standard env vars: `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
-
-## [2.4.2] - 2025-06-24
-
-### Fixed
-
-* nightly: fix bug that clones nightly repo to wrong location
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1672
-* Sync translations for all supported languages: es pl ru
-
-## [2.4.1] - 2025-06-23
 
 ### Added
 
-* build: Clearer error messages when working with Git.
-* verify: generate <appid>.json files that list all reports
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1632
-
 ### Fixed
 
-* deploy: use master branch when working complete git-mirror repo
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1666
-* update: use ctime/mtime to control _strip_and_copy_image runs
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1665
-* update: If categories.yml only has icon:, then add name:
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1659
-* update: fix handling of Triple-T 1.0.0 graphics
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1652
-* update: never execute any VCS e.g. git
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1630
-* config: lazyload environment variables in config.yml
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1645
-* config: make localized name/description/icon optional
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1649
-* lint: add repo_key_sha256 to list of valid config keys
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1643
-* build: calculate all combinations of gradle flavors
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1638
-* build: set SOURCE_DATE_EPOCH from app's git otherwise fdroiddata metadata file
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1653
-* Sync translations for all supported languages: ca cs de fr ga ja pl pt pt_BR
-  pt_PT ru sq tr uk zh_Hans
-
 ### Removed
 
-## [2.4.0] - 2025-03-25
-
-### Added
-
-* lint: support the base _config.yml_.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1606
-
-### Fixed
-
-* Expand {env: foo} config syntax to be allowed any place a string is.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1610
-* Only show "unsafe permissions on config.yml" when secrets are present.
-* Standardized config files on ruamel.yaml with a YAML 1.2 data format.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1611
-* Brought back error when a package has multiple package types (e.g. xapk and
-  apk).  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1602
-* Reworked test suite to be entirely based on Python unittest (thanks @mindston).
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1587
-* publish/signindex/gpgsign no longer load the _qrcode_ and _requests_ modules,
-  and can operate without them installed.
-* scanner: add bun.lock as lock file of package.json
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1615
-* index: fail if user sets mirrors:isPrimary wrong
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1617
-  https://gitlab.com/fdroid/fdroidserver/-/issues/1125
-* Sync translations for all supported languages: bo ca cs de es fr ga hu it ja
-  ko nb_NO pl pt pt_BR pt_PT ro ru sq sr sw tr uk zh_Hans zh_Hant
-
-### Removed
-
-* checkupdates: remove auto_author: config, it is no longer used.
-* Purge support for the long-deprecated _config.py_ config file.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1607
-
-
-## [2.3.5] - 2025-01-20
-
-### Fixed
-
-* Fix issue where APKs with v1-only signatures and targetSdkVersion < 30 could
-  be maliciously crafted to bypass AllowedAPKSigningKeys
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1588
-* Ignore apksigner v33.x, it has bugs verifying APKs with v3/v3.1 sigs.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1593
-* Sync translations for: ca cs de es fr ga ja pt_BR pt_PT ru sq sr uk zh_Hans
-
-## [2.3.4] - 2024-12-12
-
-### Fixed
-
-* Fix localhost network tests on systems with IPv6.
-* lint: only error out on missing extlib on versions not archived.
-
-## [2.3.3] - 2024-12-11
-
-### Added
-
-* verify: `--clean-up-verified` to delete files used when verifying an APK if
-  the verification was successful.
-
-### Fixed
-
-* Support Python 3.13 in the full test suite.
-* Sync translations for: ca de fr ja pl ro ru sr ta
-* update: only generate _index.png_ when making _index.html_, allowing the repo
-  operator to set a different repo icon, e.g. not the QR Code.
-
-## [2.3.2] - 2024-11-26
-
-### Fixed
-
-* install: fix downloading from GitHub Releases and Maven Central.
-* Sync translations for: ca fa fr pt ru sr ta zh_Hant
-
-## [2.3.1] - 2024-11-25
-
-### Fixed
-
-* Sync all translations for: cs de es fr ga pt_BR ru sq zh_Hans.
-* Drop use of deprecated imghdr library to support Python 3.13.
-* Install biplist and pycountry by default on macOS.
-* Fixed running test suite out of dist tarball.
-
-## [2.3.0] - 2024-11-21
-
-### Added
-
-* YAML 1.2 as native format for all _.yml_ files, including metadata and config.
-* install: will now fetch _F-Droid.apk_ and install it via `adb`.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1546
-* scanner: scan APK Signing Block for known block types like Google Play
-  Signature aka "Frosting".
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1555
-* Support Rclone for deploying to many different cloud services.
-* deploy: support deploying to GitHub Releases.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1471
-* scanner: support libs.versions.toml
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1526
-* Consider subdir for triple-t metadata discovery in Flutter apps.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1541
-* deploy: added `index_only:` mode for mirroring the index to small hosting
-  locations. https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1420
-* Support publishing repos in AltStore format.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1465
-* Support indexing iOS IPA app files.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1413
-* deploy: _config/mirrors.yml_ file with support for adding per-mirror metadata,
-  like `countryCode:`.
-* Repo's categories are now set in the config files.
-* lint: check syntax of config files.
-* publish: `--error-on-failed` to exit when signing/verifying fails.
-* scanner: `--refresh` and `refresh_config:` to control triggering a refresh of
-  the rule sets.
-* Terminal output colorization and `--color` argument to control it.
-* New languages: Catalan (ca), Irish (ga), Japanese (ja), Serbian (sr), and
-  Swahili (sw).
-* Support donation links from `community_bridge`, `buy_me_a_coffee`.
-
-### Fixed
-
-* Use last modified time and file size for caching data about scanned APKs
-  instead of SHA-256 checksum.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1542
-* `repo_web_base_url:` config for generating per-app URLs for viewing in
-  browsers.  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1178
-* `fdroid scanner` flags WebAssembly binary _.wasm_ files.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1562
-* Test suite as standard Python `unittest` setup (thanks @ghost.adh).
-* scanner: error on dependency files without lock file.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1504
-* nightly: finding APKs in the wrong directory. (thanks @WrenIX)
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1512
-* `AllowedAPKSigningKeys` works with all single-signer APK signatures.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1466
-* Sync all translations for: cs de it ko pl pt pt_BR pt_PT ro ru sq tr uk
-  zh_Hans zh_Hant.
-* Support Androguard 4.x.
-* Support Python 3.12.
-
-### Removed
-
-* Drop all uses of _stats/known_apks.txt_ and the `update_stats:` config key.
-  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1547
 * The `maven:` field is now always a string, with `yes` as a legacy special
   value.  It is no longer treated like a boolean in any case.
-* scanner: jcenter is no longer an allowed Maven repo.
-* build: `--reset-server` removed (thanks @gotmi1k).
 
 ## [2.2.2] - 2024-04-24
 

MANIFEST.in

@@ -20,17 +20,14 @@ include examples/template.yml
 include examples/Vagrantfile.yaml
 include gradlew-fdroid
 include LICENSE
-include locale/ba/LC_MESSAGES/fdroidserver.po
 include locale/bo/LC_MESSAGES/fdroidserver.po
 include locale/ca/LC_MESSAGES/fdroidserver.po
 include locale/cs/LC_MESSAGES/fdroidserver.po
 include locale/de/LC_MESSAGES/fdroidserver.po
 include locale/es/LC_MESSAGES/fdroidserver.po
 include locale/fr/LC_MESSAGES/fdroidserver.po
-include locale/ga/LC_MESSAGES/fdroidserver.po
 include locale/hu/LC_MESSAGES/fdroidserver.po
 include locale/it/LC_MESSAGES/fdroidserver.po
-include locale/ja/LC_MESSAGES/fdroidserver.po
 include locale/ko/LC_MESSAGES/fdroidserver.po
 include locale/nb_NO/LC_MESSAGES/fdroidserver.po
 include locale/pl/LC_MESSAGES/fdroidserver.po
@@ -51,6 +48,7 @@ include README.md
 include tests/aosp_testkey_debug.keystore
 include tests/apk.embedded_1.apk
 include tests/bad-unicode-*.apk
+include tests/build.TestCase
 include tests/build-tools/17.0.0/aapt-output-com.moez.QKSMS_182.txt
 include tests/build-tools/17.0.0/aapt-output-com.politedroid_3.txt
 include tests/build-tools/17.0.0/aapt-output-com.politedroid_4.txt
@@ -60,10 +58,10 @@ include tests/build-tools/17.0.0/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/17.0.0/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/17.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/17.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/17.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/17.0.0/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/17.0.0/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/17.0.0/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/17.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/17.0.0/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/17.0.0/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/18.1.1/aapt-output-com.moez.QKSMS_182.txt
@@ -75,10 +73,10 @@ include tests/build-tools/18.1.1/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/18.1.1/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/18.1.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/18.1.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/18.1.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/18.1.1/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/18.1.1/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/18.1.1/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/18.1.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/18.1.1/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/18.1.1/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/19.0.0/aapt-output-com.moez.QKSMS_182.txt
@@ -90,10 +88,10 @@ include tests/build-tools/19.0.0/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/19.0.0/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/19.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/19.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/19.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/19.0.0/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/19.0.0/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/19.0.0/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/19.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/19.0.0/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/19.0.0/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/19.1.0/aapt-output-com.moez.QKSMS_182.txt
@@ -105,10 +103,10 @@ include tests/build-tools/19.1.0/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/19.1.0/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/19.1.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/19.1.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/19.1.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/19.1.0/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/19.1.0/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/19.1.0/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/19.1.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/19.1.0/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/19.1.0/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/20.0.0/aapt-output-com.moez.QKSMS_182.txt
@@ -120,10 +118,10 @@ include tests/build-tools/20.0.0/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/20.0.0/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/20.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/20.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/20.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/20.0.0/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/20.0.0/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/20.0.0/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/20.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/20.0.0/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/20.0.0/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/21.1.1/aapt-output-com.moez.QKSMS_182.txt
@@ -135,10 +133,10 @@ include tests/build-tools/21.1.1/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/21.1.1/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/21.1.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/21.1.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/21.1.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/21.1.1/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/21.1.1/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/21.1.1/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/21.1.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/21.1.1/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/21.1.1/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/21.1.2/aapt-output-com.moez.QKSMS_182.txt
@@ -150,10 +148,10 @@ include tests/build-tools/21.1.2/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/21.1.2/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/21.1.2/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/21.1.2/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/21.1.2/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/21.1.2/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/21.1.2/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/21.1.2/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/21.1.2/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/21.1.2/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/21.1.2/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/22.0.0/aapt-output-com.moez.QKSMS_182.txt
@@ -165,10 +163,10 @@ include tests/build-tools/22.0.0/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/22.0.0/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/22.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/22.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/22.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/22.0.0/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/22.0.0/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/22.0.0/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/22.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/22.0.0/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/22.0.0/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/22.0.1/aapt-output-com.moez.QKSMS_182.txt
@@ -180,10 +178,10 @@ include tests/build-tools/22.0.1/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/22.0.1/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/22.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/22.0.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/22.0.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/22.0.1/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/22.0.1/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/22.0.1/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/22.0.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/22.0.1/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/22.0.1/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/23.0.0/aapt-output-com.moez.QKSMS_182.txt
@@ -195,10 +193,10 @@ include tests/build-tools/23.0.0/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/23.0.0/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/23.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/23.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/23.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/23.0.0/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/23.0.0/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/23.0.0/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/23.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/23.0.0/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/23.0.0/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/23.0.1/aapt-output-com.moez.QKSMS_182.txt
@@ -210,10 +208,10 @@ include tests/build-tools/23.0.1/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/23.0.1/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/23.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/23.0.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/23.0.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/23.0.1/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/23.0.1/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/23.0.1/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/23.0.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/23.0.1/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/23.0.1/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/23.0.2/aapt-output-com.moez.QKSMS_182.txt
@@ -225,10 +223,10 @@ include tests/build-tools/23.0.2/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/23.0.2/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/23.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/23.0.2/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/23.0.2/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/23.0.2/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/23.0.2/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/23.0.2/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/23.0.2/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/23.0.2/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/23.0.2/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/23.0.3/aapt-output-com.moez.QKSMS_182.txt
@@ -240,10 +238,10 @@ include tests/build-tools/23.0.3/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/23.0.3/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/23.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/23.0.3/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/23.0.3/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/23.0.3/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/23.0.3/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/23.0.3/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/23.0.3/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/23.0.3/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/23.0.3/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/24.0.0/aapt-output-com.moez.QKSMS_182.txt
@@ -255,10 +253,10 @@ include tests/build-tools/24.0.0/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/24.0.0/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/24.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/24.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/24.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/24.0.0/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/24.0.0/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/24.0.0/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/24.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/24.0.0/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/24.0.0/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/24.0.1/aapt-output-com.moez.QKSMS_182.txt
@@ -270,10 +268,10 @@ include tests/build-tools/24.0.1/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/24.0.1/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/24.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/24.0.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/24.0.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/24.0.1/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/24.0.1/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/24.0.1/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/24.0.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/24.0.1/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/24.0.1/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/24.0.2/aapt-output-com.moez.QKSMS_182.txt
@@ -285,10 +283,10 @@ include tests/build-tools/24.0.2/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/24.0.2/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/24.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/24.0.2/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/24.0.2/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/24.0.2/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/24.0.2/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/24.0.2/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/24.0.2/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/24.0.2/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/24.0.2/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/24.0.3/aapt-output-com.moez.QKSMS_182.txt
@@ -300,10 +298,10 @@ include tests/build-tools/24.0.3/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/24.0.3/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/24.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/24.0.3/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/24.0.3/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/24.0.3/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/24.0.3/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/24.0.3/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/24.0.3/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/24.0.3/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/24.0.3/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/25.0.0/aapt-output-com.moez.QKSMS_182.txt
@@ -315,10 +313,10 @@ include tests/build-tools/25.0.0/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/25.0.0/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/25.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/25.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/25.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/25.0.0/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/25.0.0/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/25.0.0/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/25.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/25.0.0/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/25.0.0/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/25.0.1/aapt-output-com.moez.QKSMS_182.txt
@@ -330,10 +328,10 @@ include tests/build-tools/25.0.1/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/25.0.1/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/25.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/25.0.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/25.0.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/25.0.1/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/25.0.1/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/25.0.1/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/25.0.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/25.0.1/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/25.0.1/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/25.0.2/aapt-output-com.moez.QKSMS_182.txt
@@ -345,10 +343,10 @@ include tests/build-tools/25.0.2/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/25.0.2/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/25.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/25.0.2/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/25.0.2/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/25.0.2/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/25.0.2/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/25.0.2/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/25.0.2/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/25.0.2/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/25.0.2/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/25.0.3/aapt-output-com.moez.QKSMS_182.txt
@@ -360,10 +358,10 @@ include tests/build-tools/25.0.3/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/25.0.3/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/25.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/25.0.3/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/25.0.3/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/25.0.3/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/25.0.3/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/25.0.3/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/25.0.3/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/25.0.3/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/25.0.3/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/26.0.0/aapt-output-com.moez.QKSMS_182.txt
@@ -375,10 +373,10 @@ include tests/build-tools/26.0.0/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/26.0.0/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/26.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/26.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/26.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/26.0.0/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/26.0.0/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/26.0.0/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/26.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/26.0.0/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/26.0.0/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/26.0.1/aapt-output-com.moez.QKSMS_182.txt
@@ -390,10 +388,10 @@ include tests/build-tools/26.0.1/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/26.0.1/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/26.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/26.0.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/26.0.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/26.0.1/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/26.0.1/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/26.0.1/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/26.0.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/26.0.1/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/26.0.1/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/26.0.2/aapt-output-com.moez.QKSMS_182.txt
@@ -405,10 +403,10 @@ include tests/build-tools/26.0.2/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/26.0.2/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/26.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/26.0.2/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/26.0.2/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/26.0.2/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/26.0.2/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/26.0.2/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/26.0.2/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/26.0.2/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/26.0.2/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/26.0.3/aapt-output-com.moez.QKSMS_182.txt
@@ -420,10 +418,10 @@ include tests/build-tools/26.0.3/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/26.0.3/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/26.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/26.0.3/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/26.0.3/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/26.0.3/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/26.0.3/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/26.0.3/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/26.0.3/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/26.0.3/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/26.0.3/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/27.0.0/aapt-output-com.moez.QKSMS_182.txt
@@ -435,10 +433,10 @@ include tests/build-tools/27.0.0/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/27.0.0/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/27.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/27.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/27.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/27.0.0/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/27.0.0/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/27.0.0/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/27.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/27.0.0/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/27.0.0/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/27.0.1/aapt-output-com.moez.QKSMS_182.txt
@@ -450,10 +448,10 @@ include tests/build-tools/27.0.1/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/27.0.1/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/27.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/27.0.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/27.0.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/27.0.1/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/27.0.1/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/27.0.1/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/27.0.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/27.0.1/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/27.0.1/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/27.0.2/aapt-output-com.moez.QKSMS_182.txt
@@ -465,10 +463,10 @@ include tests/build-tools/27.0.2/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/27.0.2/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/27.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/27.0.2/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/27.0.2/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/27.0.2/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/27.0.2/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/27.0.2/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/27.0.2/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/27.0.2/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/27.0.2/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/27.0.3/aapt-output-com.moez.QKSMS_182.txt
@@ -480,10 +478,10 @@ include tests/build-tools/27.0.3/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/27.0.3/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/27.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/27.0.3/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/27.0.3/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/27.0.3/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/27.0.3/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/27.0.3/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/27.0.3/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/27.0.3/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/27.0.3/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/28.0.0/aapt-output-com.moez.QKSMS_182.txt
@@ -495,10 +493,10 @@ include tests/build-tools/28.0.0/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/28.0.0/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/28.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/28.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/28.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/28.0.0/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/28.0.0/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/28.0.0/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/28.0.0/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/28.0.0/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/28.0.0/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/28.0.1/aapt-output-com.moez.QKSMS_182.txt
@@ -510,10 +508,10 @@ include tests/build-tools/28.0.1/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/28.0.1/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/28.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/28.0.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/28.0.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/28.0.1/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/28.0.1/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/28.0.1/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/28.0.1/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/28.0.1/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/28.0.1/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/28.0.2/aapt-output-com.politedroid_3.txt
@@ -524,10 +522,10 @@ include tests/build-tools/28.0.2/aapt-output-duplicate.permisssions_9999999.txt
 include tests/build-tools/28.0.2/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/28.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/28.0.2/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/28.0.2/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/28.0.2/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/28.0.2/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/28.0.2/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/28.0.2/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/28.0.2/aapt-output-org.droidtr.keyboard_34.txt
 include tests/build-tools/28.0.2/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/28.0.3/aapt-output-com.example.test.helloworld_1.txt
@@ -540,15 +538,17 @@ include tests/build-tools/28.0.3/aapt-output-info.guardianproject.urzip_100.txt
 include tests/build-tools/28.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt
 include tests/build-tools/28.0.3/aapt-output-no.min.target.sdk_987.txt
 include tests/build-tools/28.0.3/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/28.0.3/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/28.0.3/aapt-output-obb.main.twoversions_1101613.txt
 include tests/build-tools/28.0.3/aapt-output-obb.main.twoversions_1101615.txt
 include tests/build-tools/28.0.3/aapt-output-obb.main.twoversions_1101617.txt
-include tests/build-tools/28.0.3/aapt-output-obb.mainpatch.current_1619.txt
 include tests/build-tools/28.0.3/aapt-output-souch.smsbypass_9.txt
 include tests/build-tools/generate.sh
 include tests/check-fdroid-apk
+include tests/checkupdates.TestCase
 include tests/com.fake.IpaApp_1000000000001.ipa
-include tests/config.yml
+include tests/common.TestCase
+include tests/config.py
 include tests/config/antiFeatures.yml
 include tests/config/categories.yml
 include tests/config/de/antiFeatures.yml
@@ -567,16 +567,18 @@ include tests/config/ic_antifeature_upstreamnonfree.xml
 include tests/config/ro/antiFeatures.yml
 include tests/config/zh-rCN/antiFeatures.yml
 include tests/corrupt-featureGraphic.png
+include tests/deploy.TestCase
 include tests/dummy-keystore.jks
 include tests/dump_internal_metadata_format.py
+include tests/exception.TestCase
 include tests/extra/manual-vmtools-test.py
 include tests/funding-usernames.yaml
 include tests/get_android_tools_versions/android-ndk-r10e/RELEASE.TXT
-include tests/get_android_tools_versions/android-sdk/ndk-bundle/package.xml
-include tests/get_android_tools_versions/android-sdk/ndk-bundle/source.properties
 include tests/get_android_tools_versions/android-sdk/ndk/11.2.2725575/source.properties
 include tests/get_android_tools_versions/android-sdk/ndk/17.2.4988734/source.properties
 include tests/get_android_tools_versions/android-sdk/ndk/21.3.6528147/source.properties
+include tests/get_android_tools_versions/android-sdk/ndk-bundle/package.xml
+include tests/get_android_tools_versions/android-sdk/ndk-bundle/source.properties
 include tests/get_android_tools_versions/android-sdk/patcher/v4/source.properties
 include tests/get_android_tools_versions/android-sdk/platforms/android-30/source.properties
 include tests/get_android_tools_versions/android-sdk/skiaparser/1/source.properties
@@ -587,6 +589,10 @@ include tests/gnupghome/secring.gpg
 include tests/gnupghome/trustdb.gpg
 include tests/gradle-maven-blocks.yaml
 include tests/gradle-release-checksums.py
+include tests/import_subcommand.TestCase
+include tests/index.TestCase
+include tests/init.TestCase
+include tests/install.TestCase
 include tests/IsMD5Disabled.java
 include tests/issue-1128-min-sdk-30-poc.apk
 include tests/issue-1128-poc1.apk
@@ -594,11 +600,10 @@ include tests/issue-1128-poc2.apk
 include tests/issue-1128-poc3a.apk
 include tests/issue-1128-poc3b.apk
 include tests/janus.apk
-include tests/key-tricks.py
 include tests/keystore.jks
-include tests/metadata-rewrite-yml/app.with.special.build.params.yml
+include tests/key-tricks.py
-include tests/metadata-rewrite-yml/fake.ota.update.yml
+include tests/lint.TestCase
-include tests/metadata-rewrite-yml/org.fdroid.fdroid.yml
+include tests/main.TestCase
 include tests/metadata/apk/info.guardianproject.urzip.yaml
 include tests/metadata/apk/org.dyndns.fules.ck.yaml
 include tests/metadata/app.with.special.build.params.yml
@@ -616,14 +621,13 @@ include tests/metadata/dump/org.smssecure.smssecure.yaml
 include tests/metadata/dump/org.videolan.vlc.yaml
 include tests/metadata/duplicate.permisssions.yml
 include tests/metadata/fake.ota.update.yml
-include tests/metadata/info.guardianproject.checkey.yml
 include tests/metadata/info.guardianproject.checkey/en-US/description.txt
 include tests/metadata/info.guardianproject.checkey/en-US/name.txt
 include tests/metadata/info.guardianproject.checkey/en-US/phoneScreenshots/checkey-phone.png
 include tests/metadata/info.guardianproject.checkey/en-US/phoneScreenshots/checkey.png
 include tests/metadata/info.guardianproject.checkey/en-US/summary.txt
 include tests/metadata/info.guardianproject.checkey/ja-JP/name.txt
-include tests/metadata/info.guardianproject.urzip.yml
+include tests/metadata/info.guardianproject.checkey.yml
 include tests/metadata/info.guardianproject.urzip/en-US/changelogs/100.txt
 include tests/metadata/info.guardianproject.urzip/en-US/changelogs/default.txt
 include tests/metadata/info.guardianproject.urzip/en-US/full_description.txt
@@ -632,26 +636,31 @@ include tests/metadata/info.guardianproject.urzip/en-US/images/icon.png
 include tests/metadata/info.guardianproject.urzip/en-US/short_description.txt
 include tests/metadata/info.guardianproject.urzip/en-US/title.txt
 include tests/metadata/info.guardianproject.urzip/en-US/video.txt
+include tests/metadata/info.guardianproject.urzip.yml
 include tests/metadata/info.zwanenburg.caffeinetile.yml
 include tests/metadata/no.min.target.sdk.yml
 include tests/metadata/obb.main.oldversion.yml
-include tests/metadata/obb.main.twoversions.yml
 include tests/metadata/obb.mainpatch.current.yml
+include tests/metadata/obb.main.twoversions.yml
 include tests/metadata/org.adaway.yml
 include tests/metadata/org.fdroid.ci.test.app.yml
 include tests/metadata/org.fdroid.fdroid.yml
-include tests/metadata/org.maxsdkversion.yml
-include tests/metadata/org.smssecure.smssecure.yml
 include tests/metadata/org.smssecure.smssecure/signatures/134/28969C09.RSA
 include tests/metadata/org.smssecure.smssecure/signatures/134/28969C09.SF
 include tests/metadata/org.smssecure.smssecure/signatures/134/MANIFEST.MF
 include tests/metadata/org.smssecure.smssecure/signatures/135/28969C09.RSA
 include tests/metadata/org.smssecure.smssecure/signatures/135/28969C09.SF
 include tests/metadata/org.smssecure.smssecure/signatures/135/MANIFEST.MF
+include tests/metadata/org.smssecure.smssecure.yml
 include tests/metadata/org.videolan.vlc.yml
 include tests/metadata/raw.template.yml
+include tests/metadata-rewrite-yml/app.with.special.build.params.yml
+include tests/metadata-rewrite-yml/fake.ota.update.yml
+include tests/metadata-rewrite-yml/org.fdroid.fdroid.yml
 include tests/metadata/souch.smsbypass.yml
+include tests/metadata.TestCase
 include tests/minimal_targetsdk_30_unsigned.apk
+include tests/nightly.TestCase
 include tests/Norway_bouvet_europe_2.obf.zip
 include tests/no_targetsdk_minsdk1_unsigned.apk
 include tests/no_targetsdk_minsdk30_unsigned.apk
@@ -662,6 +671,7 @@ include tests/org.bitbucket.tickytacky.mirrormirror_3.apk
 include tests/org.bitbucket.tickytacky.mirrormirror_4.apk
 include tests/org.dyndns.fules.ck_20.apk
 include tests/org.sajeg.fallingblocks_3.apk
+include tests/publish.TestCase
 include tests/repo/com.example.test.helloworld_1.apk
 include tests/repo/com.politedroid_3.apk
 include tests/repo/com.politedroid_4.apk
@@ -680,17 +690,16 @@ include tests/repo/main.1434483388.obb.main.oldversion.obb
 include tests/repo/main.1619.obb.mainpatch.current.obb
 include tests/repo/no.min.target.sdk_987.apk
 include tests/repo/obb.main.oldversion_1444412523.apk
-include tests/repo/obb.main.twoversions_1101613.apk
+include tests/repo/obb.mainpatch.current_1619_another-release-key.apk
-include tests/repo/obb.main.twoversions_1101615.apk
+include tests/repo/obb.mainpatch.current_1619.apk
-include tests/repo/obb.main.twoversions_1101617.apk
-include tests/repo/obb.main.twoversions_1101617_src.tar.gz
 include tests/repo/obb.mainpatch.current/en-US/featureGraphic.png
 include tests/repo/obb.mainpatch.current/en-US/icon.png
 include tests/repo/obb.mainpatch.current/en-US/phoneScreenshots/screenshot-main.png
 include tests/repo/obb.mainpatch.current/en-US/sevenInchScreenshots/screenshot-tablet-main.png
-include tests/repo/obb.mainpatch.current_1619.apk
+include tests/repo/obb.main.twoversions_1101613.apk
-include tests/repo/obb.mainpatch.current_1619_another-release-key.apk
+include tests/repo/obb.main.twoversions_1101615.apk
-include tests/repo/org.maxsdkversion_4.apk
+include tests/repo/obb.main.twoversions_1101617.apk
+include tests/repo/obb.main.twoversions_1101617_src.tar.gz
 include tests/repo/org.videolan.vlc/en-US/icon.png
 include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot10.png
 include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot12.png
@@ -702,16 +711,16 @@ include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot4.png
 include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot7.png
 include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot9.png
 include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot0.png
-include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot1.png
 include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot11.png
 include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot13.png
 include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot14.png
 include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot16.png
 include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot17.png
 include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot19.png
-include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot2.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot1.png
 include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot21.png
 include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot23.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot2.png
 include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot3.png
 include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot5.png
 include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot6.png
@@ -720,25 +729,18 @@ include tests/repo/patch.1619.obb.mainpatch.current.obb
 include tests/repo/souch.smsbypass_9.apk
 include tests/repo/urzip-*.apk
 include tests/repo/v1.v2.sig_1020.apk
+include tests/rewritemeta.TestCase
 include tests/run-tests
 include tests/SANAPPSI.RSA
 include tests/SANAPPSI.SF
-include tests/shared_test_code.py
+include tests/scanner.TestCase
-include tests/signindex/guardianproject-v1.jar
+include tests/signatures.TestCase
+include tests/signindex.TestCase
 include tests/signindex/guardianproject.jar
+include tests/signindex/guardianproject-v1.jar
 include tests/signindex/testy.jar
 include tests/signindex/unsigned.jar
 include tests/source-files/at.bitfire.davdroid/build.gradle
-include tests/source-files/catalog.test/app/build.gradle
-include tests/source-files/catalog.test/build.gradle.kts
-include tests/source-files/catalog.test/buildSrc/build.gradle.kts
-include tests/source-files/catalog.test/buildSrc/settings.gradle.kts
-include tests/source-files/catalog.test/buildSrc2/build.gradle.kts
-include tests/source-files/catalog.test/buildSrc2/settings.gradle.kts
-include tests/source-files/catalog.test/core/build.gradle
-include tests/source-files/catalog.test/gradle/libs.versions.toml
-include tests/source-files/catalog.test/libs.versions.toml
-include tests/source-files/catalog.test/settings.gradle.kts
 include tests/source-files/cn.wildfirechat.chat/avenginekit/build.gradle
 include tests/source-files/cn.wildfirechat.chat/build.gradle
 include tests/source-files/cn.wildfirechat.chat/chat/build.gradle
@@ -755,11 +757,6 @@ include tests/source-files/com.anpmech.launcher/app/src/main/AndroidManifest.xml
 include tests/source-files/com.anpmech.launcher/build.gradle
 include tests/source-files/com.anpmech.launcher/settings.gradle
 include tests/source-files/com.github.jameshnsears.quoteunquote/build.gradle
-include tests/source-files/com.github.shadowsocks/core/build.gradle.kts
-include tests/source-files/com.github.shadowsocks/mobile/build.gradle.kts
-include tests/source-files/com.infomaniak.mail/Core/gradle/core.versions.toml
-include tests/source-files/com.infomaniak.mail/gradle/libs.versions.toml
-include tests/source-files/com.infomaniak.mail/settings.gradle
 include tests/source-files/com.integreight.onesheeld/build.gradle
 include tests/source-files/com.integreight.onesheeld/gradle/wrapper/gradle-wrapper.properties
 include tests/source-files/com.integreight.onesheeld/localeapi/build.gradle
@@ -773,21 +770,16 @@ include tests/source-files/com.integreight.onesheeld/pullToRefreshlibrary/src/ma
 include tests/source-files/com.integreight.onesheeld/quickReturnHeader/build.gradle
 include tests/source-files/com.integreight.onesheeld/quickReturnHeader/src/main/AndroidManifest.xml
 include tests/source-files/com.integreight.onesheeld/settings.gradle
-include tests/source-files/com.jens.automation2/app/build.gradle
 include tests/source-files/com.jens.automation2/build.gradle
+include tests/source-files/com.jens.automation2/app/build.gradle
 include tests/source-files/com.kunzisoft.testcase/build.gradle
-include tests/source-files/com.lolo.io.onelist/app/build.gradle.kts
+include tests/source-files/com.nextcloud.client/build.gradle
-include tests/source-files/com.lolo.io.onelist/build.gradle.kts
-include tests/source-files/com.lolo.io.onelist/gradle/libs.versions.toml
-include tests/source-files/com.lolo.io.onelist/gradle/wrapper/gradle-wrapper.properties
-include tests/source-files/com.lolo.io.onelist/settings.gradle
 include tests/source-files/com.nextcloud.client.dev/src/generic/fastlane/metadata/android/en-US/full_description.txt
 include tests/source-files/com.nextcloud.client.dev/src/generic/fastlane/metadata/android/en-US/short_description.txt
 include tests/source-files/com.nextcloud.client.dev/src/generic/fastlane/metadata/android/en-US/title.txt
 include tests/source-files/com.nextcloud.client.dev/src/versionDev/fastlane/metadata/android/en-US/full_description.txt
 include tests/source-files/com.nextcloud.client.dev/src/versionDev/fastlane/metadata/android/en-US/short_description.txt
 include tests/source-files/com.nextcloud.client.dev/src/versionDev/fastlane/metadata/android/en-US/title.txt
-include tests/source-files/com.nextcloud.client/build.gradle
 include tests/source-files/com.nextcloud.client/src/generic/fastlane/metadata/android/en-US/full_description.txt
 include tests/source-files/com.nextcloud.client/src/generic/fastlane/metadata/android/en-US/short_description.txt
 include tests/source-files/com.nextcloud.client/src/generic/fastlane/metadata/android/en-US/title.txt
@@ -807,26 +799,21 @@ include tests/source-files/firebase-allowlisted/app/build.gradle
 include tests/source-files/firebase-allowlisted/build.gradle
 include tests/source-files/firebase-suspect/app/build.gradle
 include tests/source-files/firebase-suspect/build.gradle
-include tests/source-files/flavor.test/build.gradle
 include tests/source-files/info.guardianproject.ripple/build.gradle
-include tests/source-files/lockfile.test/flutter/.dart_tool/flutter_gen/pubspec.yaml
 include tests/source-files/lockfile.test/flutter/pubspec.lock
 include tests/source-files/lockfile.test/flutter/pubspec.yaml
 include tests/source-files/lockfile.test/javascript/package.json
 include tests/source-files/lockfile.test/javascript/yarn.lock
+include tests/source-files/lockfile.test/rust/subdir2/Cargo.toml
 include tests/source-files/lockfile.test/rust/subdir/Cargo.lock
 include tests/source-files/lockfile.test/rust/subdir/Cargo.toml
 include tests/source-files/lockfile.test/rust/subdir/subdir/subdir/Cargo.toml
-include tests/source-files/lockfile.test/rust/subdir2/Cargo.toml
 include tests/source-files/open-keychain/open-keychain/build.gradle
 include tests/source-files/open-keychain/open-keychain/OpenKeychain/build.gradle
 include tests/source-files/org.mozilla.rocket/app/build.gradle
 include tests/source-files/org.noise_planet.noisecapture/app/build.gradle
 include tests/source-files/org.noise_planet.noisecapture/settings.gradle
 include tests/source-files/org.noise_planet.noisecapture/sosfilter/build.gradle
-include tests/source-files/org.piepmeyer.gauguin/build.gradle.kts
-include tests/source-files/org.piepmeyer.gauguin/libs.versions.toml
-include tests/source-files/org.piepmeyer.gauguin/settings.gradle.kts
 include tests/source-files/org.tasks/app/build.gradle.kts
 include tests/source-files/org.tasks/build.gradle
 include tests/source-files/org.tasks/build.gradle.kts
@@ -834,7 +821,6 @@ include tests/source-files/org.tasks/buildSrc/build.gradle.kts
 include tests/source-files/org.tasks/settings.gradle.kts
 include tests/source-files/osmandapp/osmand/build.gradle
 include tests/source-files/osmandapp/osmand/gradle/wrapper/gradle-wrapper.properties
-include tests/source-files/OtakuWorld/build.gradle
 include tests/source-files/realm/react-native/android/build.gradle
 include tests/source-files/se.manyver/android/app/build.gradle
 include tests/source-files/se.manyver/android/build.gradle
@@ -852,33 +838,11 @@ include tests/source-files/ut.ewh.audiometrytest/settings.gradle
 include tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties
 include tests/source-files/Zillode/syncthing-silk/build.gradle
 include tests/SpeedoMeterApp.main_1.apk
-include tests/test_build.py
+include tests/stats/known_apks.txt
-include tests/test_checkupdates.py
+include tests/testcommon.py
-include tests/test_common.py
+include tests/test-gradlew-fdroid
-include tests/test_deploy.py
-include tests/test_exception.py
-include tests/test_gradlew-fdroid
-include tests/test_import_subcommand.py
-include tests/test_index.py
-include tests/test_init.py
-include tests/test_install.py
-include tests/test_lint.py
-include tests/test_main.py
-include tests/test_metadata.py
-include tests/test_nightly.py
-include tests/test_publish.py
-include tests/test_rewritemeta.py
-include tests/test_scanner.py
-include tests/test_signatures.py
-include tests/test_signindex.py
-include tests/test_update.py
-include tests/test_vcs.py
-include tests/triple-t-1-graphics/build/de.wivewa.dialer/app/src/main/play/en-US/listing/featureGraphic/play_store_feature_graphic.png
-include tests/triple-t-1-graphics/build/de.wivewa.dialer/app/src/main/play/en-US/listing/icon/icon.png
-include tests/triple-t-1-graphics/build/de.wivewa.dialer/app/src/main/play/en-US/listing/phoneScreenshots/1.png
-include tests/triple-t-1-graphics/metadata/de.wivewa.dialer.yml
-include tests/triple-t-2/build/org.piwigo.android/app/.gitignore
 include tests/triple-t-2/build/org.piwigo.android/app/build.gradle
+include tests/triple-t-2/build/org.piwigo.android/app/.gitignore
 include tests/triple-t-2/build/org.piwigo.android/app/src/debug/res/values/constants.xml
 include tests/triple-t-2/build/org.piwigo.android/app/src/debug/res/values/strings.xml
 include tests/triple-t-2/build/org.piwigo.android/app/src/main/java/org/piwigo/PiwigoApplication.java
@@ -933,13 +897,14 @@ include tests/triple-t-multiple/build/ch.admin.bag.covidcertificate.wallet/verif
 include tests/triple-t-multiple/build/ch.admin.bag.covidcertificate.wallet/wallet/src/main/play/listings/en-US/title.txt
 include tests/triple-t-multiple/metadata/ch.admin.bag.covidcertificate.verifier.yml
 include tests/triple-t-multiple/metadata/ch.admin.bag.covidcertificate.wallet.yml
+include tests/update.TestCase
+include tests/urzip.apk
 include tests/urzip-badcert.apk
 include tests/urzip-badsig.apk
-include tests/urzip-release-unsigned.apk
 include tests/urzip-release.apk
-include tests/urzip.apk
+include tests/urzip-release-unsigned.apk
 include tests/v2.only.sig_2.apk
 include tests/valid-package-names/random-package-names
 include tests/valid-package-names/RandomPackageNames.java
 include tests/valid-package-names/test.py
-include tests/__init__.py
+include tests/vcs.TestCase

README.md

@@ -53,13 +53,13 @@ To run the full test suite:
 
     tests/run-tests
 
-To run the tests for individual Python modules, see the `tests/test_*.py` files, e.g.:
+To run the tests for individual Python modules, see the _.TestCase_ files, e.g.:
 
-    python -m unittest tests/test_metadata.py
+    tests/metadata.TestCase
 
 It is also possible to run individual tests:
 
-    python -m unittest tests.test_metadata.MetadataTest.test_rewrite_yaml_special_build_params
+    tests/metadata.TestCase MetadataTest.test_rewrite_yaml_special_build_params
 
 There is a growing test suite that has good coverage on a number of key parts of
 this code base.  It does not yet cover all the code, and there are some parts

buildserver/Dockerfile

@@ -37,22 +37,11 @@ RUN useradd --create-home -s /bin/bash vagrant && echo -n 'vagrant:vagrant' | ch
 #
 # Ensure fdroidserver's dependencies are marked manual before purging
 # unneeded packages, otherwise, all its dependencies get purged.
-#
-# The official Debian docker images ship without ca-certificates, so
-# TLS certificates cannot be verified until that is installed. The
-# following code temporarily turns off TLS verification, and enables
-# HTTPS, so at least unverified TLS is used for apt-get instead of
-# plain HTTP.  Once ca-certificates is installed, the CA verification
-# is enabled by removing the newly created config file.  This set up
-# makes the initial `apt-get update` and `apt-get install` look the
-# same as verified TLS to the network observer and hides the metadata.
 RUN printf "path-exclude=/usr/share/locale/*\npath-exclude=/usr/share/man/*\npath-exclude=/usr/share/doc/*\npath-include=/usr/share/doc/*/copyright\n" >/etc/dpkg/dpkg.cfg.d/01_nodoc \
 	&& mkdir -p /usr/share/man/man1 \
-	&& echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/99nocacertificates \
-	&& find /etc/apt/sources.list* -type f -exec sed -i s,http:,https:, {} \; \
 	&& apt-get update \
 	&& apt-get install ca-certificates \
-	&& rm /etc/apt/apt.conf.d/99nocacertificates \
+        && sed -i 's,http:,https:,' /etc/apt/sources.list.d/debian.sources \
 	&& apt-get upgrade \
 	&& apt-get dist-upgrade \
 	&& apt-get install openssh-client iproute2 python3 openssh-server sudo \

buildserver/Vagrantfile

@@ -14,7 +14,7 @@ configfile = {
 
 srvpath = Pathname.new(File.dirname(__FILE__)).realpath
 configpath = File.join(srvpath, "/Vagrantfile.yaml")
-if File.exist? configpath
+if File.exists? configpath
   c = YAML.load_file(configpath)
   if c and not c.empty?
     c.each do |k,v|
@@ -86,7 +86,7 @@ Vagrant.configure("2") do |config|
   # Make sure dir exists to mount to, since buildserver/ is
   # automatically mounted as /vagrant in the guest VM. This is more
   # necessary with 9p synced folders
-  Dir.mkdir('cache') unless File.exist?('cache')
+  Dir.mkdir('cache') unless File.exists?('cache')
 
   # Root partition needs to be resized to the new allocated space
   config.vm.provision "shell", inline: <<-SHELL

buildserver/provision-apt-get-install

@@ -104,7 +104,6 @@ apt-get upgrade
 apt-get update || apt-get update
 
 packages="
- androguard/bookworm-backports
  apksigner
  default-jdk-headless
  default-jre-headless
@@ -115,7 +114,6 @@ packages="
  gnupg
  mercurial
  patch
- python3-magic
  python3-packaging
  rsync
  sdkmanager/bookworm-backports

buildserver/provision-disable-analytics

@@ -10,6 +10,6 @@ cat <<EOF > $flutter_conf
   "enabled": false
 }
 EOF
-chown -R vagrant:vagrant $flutter_conf
+chown -R vagrant.vagrant $flutter_conf
 chmod -R 0644 $flutter_conf
 

buildserver/provision-gradle

@@ -25,14 +25,9 @@ fi
 chmod -R a+rX /opt/gradle
 
 test -e /opt/gradle/bin || mkdir -p /opt/gradle/bin
-git clone --depth 1 https://gitlab.com/fdroid/gradlew-fdroid.git /home/vagrant/gradlew-fdroid/
+ln -fs /home/vagrant/fdroidserver/gradlew-fdroid /opt/gradle/bin/gradle
-chmod 0755 /home/vagrant/gradlew-fdroid/gradlew-fdroid
+chown -h vagrant.vagrant /opt/gradle/bin/gradle
-chmod -R u+rwX,a+rX,go-w /home/vagrant/gradlew-fdroid/
+chown vagrant.vagrant /opt/gradle/versions
-ln -fs /home/vagrant/gradlew-fdroid/gradlew-fdroid /opt/gradle/bin/gradle
-ln -fs /home/vagrant/gradlew-fdroid/gradlew-fdroid /usr/local/bin/
-
-chown -h vagrant:vagrant /opt/gradle/bin/gradle
-chown vagrant:vagrant /opt/gradle/versions
 chmod 0755 /opt/gradle/versions
 
 GRADLE_HOME=/home/vagrant/.gradle
@@ -49,5 +44,5 @@ systemProp.org.gradle.internal.http.connectionTimeout=600000
 systemProp.org.gradle.internal.http.socketTimeout=600000
 EOF
 
-chown -R vagrant:vagrant $GRADLE_HOME/
+chown -R vagrant.vagrant $GRADLE_HOME/
 chmod -R a+rX $GRADLE_HOME/

completion/bash-completion

@@ -109,8 +109,8 @@ __complete_gpgsign() {
 }
 
 __complete_install() {
-	opts="-v -q -a -p -n -y"
+	opts="-v -q"
-	lopts="--verbose --quiet --all --color --no-color --privacy-mode --no-privacy-mode --no --yes"
+	lopts="--verbose --quiet --all"
 	case "${cur}" in
 		-*)
 			__complete_options
@@ -251,7 +251,7 @@ __complete_btlog() {
 
 __complete_mirror() {
 	opts="-v"
-	lopts="--all --archive --build-logs --color --no-color --pgp-signatures --src-tarballs --output-dir"
+	lopts="--all --archive --build-logs --pgp-signatures --src-tarballs --output-dir"
 	__complete_options
 }
 
@@ -270,7 +270,7 @@ __complete_deploy() {
 
 __complete_signatures() {
 	opts="-v -q"
-	lopts="--verbose --color --no-color --no-check-https"
+	lopts="--verbose --no-check-https"
 	case "${cur}" in
 		-*)
 			__complete_options
@@ -289,7 +289,7 @@ __complete_signindex() {
 __complete_init() {
 	opts="-v -q -d"
 	lopts="--verbose --quiet --distinguished-name --keystore
- --repo-keyalias --android-home --no-prompt --color --no-color"
+ --repo-keyalias --android-home --no-prompt"
 	__complete_options
 }
 

docs/source/conf.py

@@ -12,7 +12,6 @@
 #
 import os
 import sys
-
 sys.path.insert(0, os.path.abspath('../../fdroidserver'))
 
 # -- Project information -----------------------------------------------------

examples/config.yml

@@ -51,13 +51,6 @@
 # Canonical URL of the repositoy, needs to end in /repo. Is is used to identity
 # the repo in the client, as well.
 # repo_url: https://MyFirstFDroidRepo.org/fdroid/repo
-#
-# Base URL for per-package pages on the website of this repo,
-# i.e. https://f-droid.org/packages/<appid>/ This should be accessible
-# with a browser. Setting it to null or not setting this disables the
-# feature.
-# repo_web_base_url: https://MyFirstFDroidRepo.org/packages/
-#
 # repo_name: My First F-Droid Repo Demo
 # repo_description: >-
 #   This is a repository of apps to be used with F-Droid. Applications
@@ -69,7 +62,6 @@
 # As above, but for the archive repo.
 #
 # archive_url: https://f-droid.org/archive
-# archive_web_base_url:
 # archive_name: My First F-Droid Archive Demo
 # archive_description: >-
 #   The repository of older versions of packages from the main demo repository.
@@ -305,33 +297,70 @@
 #
 # sync_from_local_copy_dir: true
 
-# To deploy to an AWS S3 "bucket" in the US East region, set the
+
-# bucket name in the config, then set the environment variables
+# To upload the repo to an Amazon S3 bucket using `fdroid deploy'
-# AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY using the values from
+# . rclone, s3cmd and apache libcloud are the available options.
-# the AWS Management Console. See
+# If rclone and s3cmd are not installed, apache libcloud is used.
-# https://rclone.org/s3/#authentication
+# To use apache libcloud, add the following options to this file
+# (config.yml)
 #
-# awsbucket: myawsfdroidbucket
+# awsbucket: myawsfdroid
-
+# awsaccesskeyid: SEE0CHAITHEIMAUR2USA
-
+# awssecretkey: {env: awssecretkey}
-# For extended options for syncing to cloud drive and object store
-# services, `fdroid deploy' wraps Rclone. Rclone is a full featured
-# sync tool for a huge variety of cloud services. Set up your services
-# using `rclone config`, then specify each config name to deploy the
-# awsbucket: to.  Using rclone_config: overrides the default AWS S3 US
-# East setup, and will only sync to the services actually specified.
 #
-# awsbucket: myawsfdroidbucket
+# In case s3cmd is installed and rclone is not installed,
-# rclone_config:
+# s3cmd will be the preferred sync option.
-#   - aws-sample-config
+# It will delete and recreate the whole fdroid directory each time.
-#   - rclone-supported-service-config
+# To customize how s3cmd interacts with the cloud
-
+# provider, create a 's3cfg' file next to this file (config.yml), and
-
+# those settings will be used instead of any 'aws' variable below.
-# By default Rclone uses the user's default configuration file at
+# Secrets can be fetched from environment variables to ensure that
-# ~/.config/rclone/rclone.conf To specify a custom configuration file,
+# they are not leaked as part of this file.
-# please add the full path to the configuration file as below.
+#
+# awsbucket: myawsfdroid
+# awsaccesskeyid: SEE0CHAITHEIMAUR2USA
+# awssecretkey: {env: awssecretkey}
+#
+# In case rclone is installed and s3cmd is not installed,
+# rclone will be the preferred sync option.
+# It will sync the local folders with remote folders without
+# deleting anything in one go.
+# To ensure success, install rclone as per
+# the instructions at https://rclone.org/install/ and also configure for
+# object storage services as detailed at https://rclone.org/s3/#configuration
+# By default rclone uses the configuration file at ~/.config/rclone/rclone.conf
+# To specify a custom configuration file, please add the full path to the
+# configuration file as below
 #
 # path_to_custom_rclone_config: /home/mycomputer/somedir/example.conf
+#
+# This setting will ignore the default rclone config found at
+# ~/.config/rclone/rclone.conf
+#
+# Please note that rclone_config can be assigned a string or list
+#
+# awsbucket: myawsfdroid
+# rclone_config: aws-sample-config
+#
+# or
+#
+# awsbucket: myawsfdroid
+# rclone_config: [aws-sample-config, rclone-supported-service-config]
+#
+# In case both rclone and s3cmd are installed, the preferred sync
+# tool can be specified in this file (config.yml)
+# if s3cmd is preferred, set it as below
+#
+# s3cmd: true
+#
+# if rclone is preferred, set it as below
+#
+# rclone: true
+#
+# Please note that only one can be set to true at any time
+# Also, in the event that both s3cmd and rclone are installed
+# and both are missing from the config.yml file, the preferred
+# tool will be s3cmd.
 
 
 # If you want to force 'fdroid server' to use a non-standard serverwebroot.
@@ -364,6 +393,11 @@
 # configured to allow push access (e.g. ssh key, username/password, etc)
 # binary_transparency_remote: git@gitlab.com:fdroid/binary-transparency-log.git
 
+# If you want to keep the "added" and "last updated" dates for each
+# app and APK in your repo, enable this.  The name comes from an old
+# system for tracking statistics that is no longer included.
+# update_stats: true
+
 # Set this to true to always use a build server. This saves specifying the
 # --server option on dedicated secure build server hosts.
 # build_server_always: true

examples/fdroid_clean_repos.py

@@ -6,6 +6,7 @@ import argparse
 import logging
 
 from fdroidserver import _, common, metadata
+
 from fdroidserver.exception import VCSException
 
 fdroid_summary = 'reset app VCSs to the latest version'
@@ -23,9 +24,9 @@ def main():
     )
     metadata.add_metadata_arguments(parser)
     options = common.parse_args(parser)
-    apps = common.read_app_args(
+    pkgs = common.read_pkg_args(options.appid, True)
-        options.appid, allow_version_codes=True, sort_by_time=True
+    allapps = metadata.read_metadata(pkgs)
-    )
+    apps = common.read_app_args(options.appid, allapps, True)
     common.read_config()
 
     for appid, app in apps.items():

examples/fdroid_export_keystore_to_nitrokey.py

@@ -4,7 +4,6 @@
 
 import os
 from argparse import ArgumentParser
-
 from fdroidserver import common
 from fdroidserver.common import FDroidPopen
 from fdroidserver.exception import BuildException

examples/fdroid_exportkeystore.py

@@ -4,7 +4,6 @@
 
 import os
 from argparse import ArgumentParser
-
 from fdroidserver import common
 from fdroidserver.common import FDroidPopen
 from fdroidserver.exception import BuildException

examples/fdroid_extract_repo_pubkey.py

@@ -4,7 +4,6 @@
 #
 
 from argparse import ArgumentParser
-
 from fdroidserver import common, index
 
 fdroid_summary = 'export the keystore in standard PEM format'

examples/fdroid_fetchsrclibs.py

@@ -8,7 +8,6 @@
 import argparse
 import os
 import pprint
-
 from fdroidserver import _, common, metadata
 
 fdroid_summary = 'prepare the srclibs for `fdroid build --on-server`'
@@ -20,7 +19,9 @@ def main():
     parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
     metadata.add_metadata_arguments(parser)
     options = common.parse_args(parser)
-    apps = common.read_app_args(options.appid, allow_version_codes=True, sort_by_time=True)
+    pkgs = common.read_pkg_args(options.appid, True)
+    allapps = metadata.read_metadata(pkgs)
+    apps = common.read_app_args(options.appid, allapps, True)
     common.read_config()
     srclib_dir = os.path.join('build', 'srclib')
     os.makedirs(srclib_dir, exist_ok=True)

examples/fdroid_nitrokeyimport.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 from argparse import ArgumentParser
-
 from fdroidserver import common
 from fdroidserver.common import FDroidPopen
 from fdroidserver.exception import BuildException

examples/mirror-to-mirror.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# This script syncs the entire repo to the primary mirrors.  It is
+# meant to run in a cronjob quite frequently, as often as there are
+# files to send.
+#
+# This script expects the receiving side to have the following
+# preceeding the ssh key entry in ~/.ssh/authorized_keys:
+#   command="rsync --server -logDtpre.iLsfx --log-format=X --delete --delay-updates . /path/to/htdocs/fdroid/",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty
+#
+set -e
+(
+flock -n 200
+set -e
+cd /home/fdroid
+for section in repo archive; do
+    echo "Started $section at `date`:"
+    for host in fdroid@ftp-push.lysator.liu.se fdroid@plug-mirror.rcac.purdue.edu fdroid@ftp.agdsn.de; do
+        set -x
+        # be super careful with the trailing slashes here! if one is wrong, it'll delete the entire section!
+        rsync --archive --delay-updates --progress --delete \
+              --timeout=3600 \
+              /home/fdroid/public_html/${section} \
+              ${host}:/srv/fdroid-mirror.at.or.at/htdocs/fdroid/ &
+        set +x
+    done
+    wait
+done
+) 200>/var/lock/root_fdroidmirrortomirror

fdroidserver/__init__.py

@@ -3,6 +3,7 @@ import glob
 import os
 import sys
 
+
 # support running straight from git and standard installs
 rootpaths = [
     os.path.realpath(os.path.join(os.path.dirname(__file__), '..')),
@@ -14,10 +15,7 @@ rootpaths = [
 
 localedir = None
 for rootpath in rootpaths:
-    found_mo = glob.glob(
+    if len(glob.glob(os.path.join(rootpath, 'locale', '*', 'LC_MESSAGES', 'fdroidserver.mo'))) > 0:
-        os.path.join(rootpath, 'locale', '*', 'LC_MESSAGES', 'fdroidserver.mo')
-    )
-    if len(found_mo) > 0:
         localedir = os.path.join(rootpath, 'locale')
         break
 
@@ -26,52 +24,39 @@ gettext.textdomain('fdroidserver')
 _ = gettext.gettext
 
 
-from fdroidserver.exception import (
+from fdroidserver.exception import (FDroidException,
-    FDroidException,
                                     MetaDataException,
-    VerificationException,  # NOQA: E402
+                                    VerificationException)  # NOQA: E402
-)
-
 FDroidException  # NOQA: B101
 MetaDataException  # NOQA: B101
 VerificationException  # NOQA: B101
 
-from fdroidserver.common import genkeystore as generate_keystore  # NOQA: E402
+from fdroidserver.common import (verify_apk_signature,
-from fdroidserver.common import verify_apk_signature
+                                 genkeystore as generate_keystore)  # NOQA: E402
-
 verify_apk_signature  # NOQA: B101
 generate_keystore  # NOQA: B101
-from fdroidserver.index import (
+from fdroidserver.index import (download_repo_index,
-    download_repo_index,
                                 download_repo_index_v1,
                                 download_repo_index_v2,
                                 get_mirror_service_urls,
-)
+                                make as make_index)  # NOQA: E402
-from fdroidserver.index import make as make_index  # NOQA: E402
-
 download_repo_index  # NOQA: B101
 download_repo_index_v1  # NOQA: B101
 download_repo_index_v2  # NOQA: B101
 get_mirror_service_urls  # NOQA: B101
 make_index  # NOQA: B101
-from fdroidserver.update import (
+from fdroidserver.update import (process_apk,
-    process_apk,
                                  process_apks,
                                  scan_apk,
-    scan_repo_files,  # NOQA: E402
+                                 scan_repo_files)  # NOQA: E402
-)
-
 process_apk  # NOQA: B101
 process_apks  # NOQA: B101
 scan_apk  # NOQA: B101
 scan_repo_files  # NOQA: B101
-from fdroidserver.deploy import (
+from fdroidserver.deploy import (update_awsbucket,
-    update_awsbucket,
                                  update_servergitmirrors,
-    update_serverwebroot,  # NOQA: E402
                                  update_serverwebroots,
-)
+                                 update_serverwebroot)  # NOQA: E402
-
 update_awsbucket  # NOQA: B101
 update_servergitmirrors  # NOQA: B101
 update_serverwebroots  # NOQA: B101

fdroidserver/__main__.py

@@ -18,20 +18,20 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import importlib.metadata
-import logging
-import os
-import pkgutil
 import re
 import sys
-from argparse import ArgumentError
+import os
-from collections import OrderedDict
+import pkgutil
+import logging
+import importlib.metadata
 
 import git
-
 import fdroidserver.common
 import fdroidserver.metadata
 from fdroidserver import _
+from argparse import ArgumentError
+from collections import OrderedDict
+
 
 COMMANDS = OrderedDict([
     ("build", _("Build a package from source")),

fdroidserver/_yaml.py

@@ -1,64 +0,0 @@
-# Copyright (C) 2025, Hans-Christoph Steiner <hans@eds.org>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Standard YAML parsing and dumping.
-
-YAML 1.2 is the preferred format for all data files.  When loading
-F-Droid formats like config.yml and <Application ID>.yml, YAML 1.2 is
-forced, and older YAML constructs should be considered an error.
-
-It is OK to load and dump files in other YAML versions if they are
-externally defined formats, like FUNDING.yml.  In those cases, these
-common instances might not be appropriate to use.
-
-There is a separate instance for dumping based on the "round trip" aka
-"rt" mode.  The "rt" mode maintains order while the "safe" mode sorts
-the output.  Also, yaml.version is not forced in the dumper because that
-makes it write out a "%YAML 1.2" header.  F-Droid's formats are
-explicitly defined as YAML 1.2 and meant to be human-editable.  So that
-header gets in the way.
-
-"""
-
-import ruamel.yaml
-
-yaml = ruamel.yaml.YAML(typ='safe')
-yaml.version = (1, 2)
-
-yaml_dumper = ruamel.yaml.YAML(typ='rt')
-
-
-def config_dump(config, fp=None):
-    """Dump config data in YAML 1.2 format without headers.
-
-    This outputs YAML in a string that is suitable for use in regexps
-    and string replacements, as well as complete files.  It is therefore
-    explicitly set up to avoid writing out headers and footers.
-
-    This is modeled after PyYAML's yaml.dump(), which can dump to a file
-    or return a string.
-
-    https://yaml.dev/doc/ruamel.yaml/example/#Output_of_%60dump()%60_as_a_string
-
-    """
-    dumper = ruamel.yaml.YAML(typ='rt')
-    dumper.default_flow_style = False
-    dumper.explicit_start = False
-    dumper.explicit_end = False
-    if fp is None:
-        with ruamel.yaml.compat.StringIO() as fp:
-            dumper.dump(config, fp)
-            return fp.getvalue()
-    dumper.dump(config, fp)

fdroidserver/apksigcopier.py

@@ -68,18 +68,9 @@ import struct
 import sys
 import zipfile
 import zlib
+
 from collections import namedtuple
-from typing import (
+from typing import Any, BinaryIO, Callable, Dict, Iterable, Iterator, Optional, Tuple, Union
-    Any,
-    BinaryIO,
-    Callable,
-    Dict,
-    Iterable,
-    Iterator,
-    Optional,
-    Tuple,
-    Union,
-)
 
 __version__ = "1.1.1"
 NAME = "apksigcopier"

fdroidserver/btlog.py

@@ -28,21 +28,22 @@
 # the F-Droid client.
 
 import collections
+import defusedxml.minidom
+import git
 import glob
+import os
 import json
 import logging
-import os
+import requests
 import shutil
 import tempfile
 import zipfile
 from argparse import ArgumentParser
 from typing import Optional
 
-import defusedxml.minidom
+from . import _
-import git
+from . import common
-import requests
+from . import deploy
-
-from . import _, common, deploy
 from .exception import FDroidException
 
 

fdroidserver/build.py

@@ -18,27 +18,31 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import argparse
-import glob
-import logging
 import os
+import shutil
+import glob
+import subprocess
 import posixpath
 import re
-import shutil
-import subprocess
 import tarfile
-import tempfile
 import threading
-import time
 import traceback
+import time
+import requests
+import tempfile
+import argparse
+import logging
 from gettext import ngettext
 from pathlib import Path
 
-import requests
+from . import _
-
+from . import common
-from . import _, common, metadata, net, scanner, vmtools
+from . import net
+from . import metadata
+from . import scanner
+from . import vmtools
 from .common import FDroidPopen
-from .exception import BuildException, FDroidException, VCSException
+from .exception import FDroidException, BuildException, VCSException
 
 try:
     import paramiko
@@ -151,7 +155,9 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         ftp.mkdir('fdroidserver')
         ftp.chdir('fdroidserver')
         ftp.put(os.path.join(serverpath, '..', 'fdroid'), 'fdroid')
+        ftp.put(os.path.join(serverpath, '..', 'gradlew-fdroid'), 'gradlew-fdroid')
         ftp.chmod('fdroid', 0o755)  # nosec B103 permissions are appropriate
+        ftp.chmod('gradlew-fdroid', 0o755)  # nosec B103 permissions are appropriate
         send_dir(os.path.join(serverpath))
         ftp.chdir(homedir)
 
@@ -300,7 +306,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         else:
             ftp.chdir(posixpath.join(homedir, 'unsigned'))
         apkfile = common.get_release_filename(app, build)
-        tarball = common.get_src_tarball_name(app.id, build.versionCode)
+        tarball = common.getsrcname(app, build)
         try:
             ftp.get(apkfile, os.path.join(output_dir, apkfile))
             if not options.notarball:
@@ -473,7 +479,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
             logging.critical("Android NDK '%s' is not a directory!" % ndk_path)
             raise FDroidException()
 
-    common.set_FDroidPopen_env(app, build)
+    common.set_FDroidPopen_env(build)
 
     # create ..._toolsversion.log when running in builder vm
     if onserver:
@@ -535,13 +541,13 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
         if build.preassemble:
             gradletasks += build.preassemble
 
-        flavors = build.gradle
+        flavours = build.gradle
-        if flavors == ['yes']:
+        if flavours == ['yes']:
-            flavors = []
+            flavours = []
 
-        flavors_cmd = ''.join([transform_first_char(flav, str.upper) for flav in flavors])
+        flavours_cmd = ''.join([transform_first_char(flav, str.upper) for flav in flavours])
 
-        gradletasks += ['assemble' + flavors_cmd + 'Release']
+        gradletasks += ['assemble' + flavours_cmd + 'Release']
 
         cmd = [config['gradle']]
         if build.gradleprops:
@@ -613,7 +619,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
     if not options.notarball:
         # Build the source tarball right before we build the release...
         logging.info("Creating source tarball...")
-        tarname = common.get_src_tarball_name(app.id, build.versionCode)
+        tarname = common.getsrcname(app, build)
         tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
 
         def tarexc(t):
@@ -715,7 +721,8 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
         bindir = os.path.join(root_dir, 'bin')
 
     if os.path.isdir(os.path.join(build_dir, '.git')):
-        commit_id = str(common.get_head_commit_id(build_dir))
+        import git
+        commit_id = common.get_head_commit_id(git.repo.Repo(build_dir))
     else:
         commit_id = build.commit
 
@@ -757,11 +764,11 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
             # really old path
             os.path.join(root_dir, 'build', 'apk'),
             ]
-        # If we build with gradle flavors with gradle plugin >= 3.0 the APK will be in
+        # If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in
-        # a subdirectory corresponding to the flavor command used, but with different
+        # a subdirectory corresponding to the flavour command used, but with different
         # capitalization.
-        if flavors_cmd:
+        if flavours_cmd:
-            apk_dirs.append(os.path.join(root_dir, 'build', 'outputs', 'apk', transform_first_char(flavors_cmd, str.lower), 'release'))
+            apk_dirs.append(os.path.join(root_dir, 'build', 'outputs', 'apk', transform_first_char(flavours_cmd, str.lower), 'release'))
         for apks_dir in apk_dirs:
             for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']:
                 apks = glob.glob(os.path.join(apks_dir, apkglob))
@@ -1110,7 +1117,10 @@ def main():
     srclib_dir = os.path.join(build_dir, 'srclib')
     extlib_dir = os.path.join(build_dir, 'extlib')
 
-    apps = common.read_app_args(options.appid, allow_version_codes=True, sort_by_time=True)
+    # Read all app and srclib metadata
+    pkgs = common.read_pkg_args(options.appid, True)
+    allapps = metadata.read_metadata(pkgs, sort_by_time=True)
+    apps = common.read_app_args(options.appid, allapps, True)
 
     for appid, app in list(apps.items()):
         if (app.get('Disabled') and not options.force) or not app.get('RepoType') or not app.get('Builds', []):

fdroidserver/checkupdates.py

@@ -18,34 +18,26 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import configparser
-import copy
-import logging
 import os
 import re
+import urllib.request
+import urllib.error
+import time
 import subprocess
 import sys
-import time
-import traceback
-import urllib.error
-import urllib.parse
-import urllib.request
 from argparse import ArgumentParser
+import traceback
+import logging
+import copy
+import urllib.parse
 from pathlib import Path
 from typing import Optional
 
-import git
+from . import _
-
+from . import common
-from . import _, common, metadata, net
+from . import metadata
-from .exception import (
+from . import net
-    FDroidException,
+from .exception import VCSException, NoSubmodulesException, FDroidException, MetaDataException
-    MetaDataException,
-    NoSubmodulesException,
-    VCSException,
-)
-
-# https://gitlab.com/fdroid/checkupdates-runner/-/blob/1861899262a62a4ed08fa24e5449c0368dfb7617/.gitlab-ci.yml#L36
-BOT_EMAIL = 'fdroidci@bubu1.eu'
 
 
 def check_http(app: metadata.App) -> tuple[Optional[str], Optional[int]]:
@@ -93,7 +85,7 @@ def check_http(app: metadata.App) -> tuple[Optional[str], Optional[int]]:
 
     m = re.search(codeex, page)
     if not m:
-        raise FDroidException("No RE match for versionCode")
+        raise FDroidException("No RE match for version code")
     vercode = common.version_code_string_to_int(m.group(1).strip())
 
     if urlver != '.':
@@ -213,7 +205,7 @@ def check_tags(app: metadata.App, pattern: str) -> tuple[str, int, str]:
             if codeex:
                 m = re.search(codeex, filecontent)
                 if not m:
-                    logging.debug(f"UpdateCheckData regex {codeex} for versionCode"
+                    logging.debug(f"UpdateCheckData regex {codeex} for version code"
                                   f" has no match in tag {tag}")
                     continue
 
@@ -233,7 +225,7 @@ def check_tags(app: metadata.App, pattern: str) -> tuple[str, int, str]:
             if verex:
                 m = re.search(verex, filecontent)
                 if not m:
-                    logging.debug(f"UpdateCheckData regex {verex} for versionName"
+                    logging.debug(f"UpdateCheckData regex {verex} for version name"
                                   f" has no match in tag {tag}")
                     continue
 
@@ -383,8 +375,7 @@ def dirs_with_manifest(startdir: str):
         A directory that contains a manifest file of an Android project, None if
         no directory could be found
     """
-    for root, dirs, files in os.walk(startdir):
+    for root, _dirs, files in os.walk(startdir):
-        dirs.sort()
         if any(m in files for m in [
                 'AndroidManifest.xml', 'pom.xml', 'build.gradle', 'build.gradle.kts']):
             yield Path(root)
@@ -677,6 +668,8 @@ def checkupdates_app(app: metadata.App, auto: bool, commit: bool = False) -> Non
         if commit:
             logging.info("Commiting update for " + app.metadatapath)
             gitcmd = ["git", "commit", "-m", commitmsg]
+            if 'auto_author' in config:
+                gitcmd.extend(['--author', config['auto_author']])
             gitcmd.extend(["--", app.metadatapath])
             if subprocess.call(gitcmd) != 0:
                 raise FDroidException("Git commit failed")
@@ -690,184 +683,6 @@ def get_last_build_from_app(app: metadata.App) -> metadata.Build:
         return metadata.Build()
 
 
-def get_upstream_main_branch(git_repo):
-    refs = list()
-    for ref in git_repo.remotes.upstream.refs:
-        if ref.name != 'upstream/HEAD':
-            refs.append(ref.name)
-    if len(refs) == 1:
-        return refs[0]
-    for name in ('upstream/main', 'upstream/master'):
-        if name in refs:
-            return name
-    try:
-        with git_repo.config_reader() as reader:
-            return 'upstream/%s' % reader.get_value('init', 'defaultBranch')
-    except configparser.NoSectionError:
-        return 'upstream/main'
-
-
-def checkout_appid_branch(appid):
-    """Prepare the working branch named after the appid.
-
-    This sets up everything for checkupdates_app() to run and add
-    commits.  If there is an existing branch named after the appid,
-    and it has commits from users other than the checkupdates-bot,
-    then this will return False.  Otherwise, it returns True.
-
-    The checkupdates-runner must set the committer email address in
-    the git config.  Then any commit with a committer or author that
-    does not match that will be considered to have human edits.  That
-    email address is currently set in:
-    https://gitlab.com/fdroid/checkupdates-runner/-/blob/1861899262a62a4ed08fa24e5449c0368dfb7617/.gitlab-ci.yml#L36
-
-    """
-    logging.debug(f'Creating merge request branch for {appid}')
-    git_repo = git.Repo.init('.')
-    upstream_main = get_upstream_main_branch(git_repo)
-    for remote in git_repo.remotes:
-        remote.fetch()
-    try:
-        git_repo.remotes.origin.fetch(f'{appid}:refs/remotes/origin/{appid}')
-    except Exception as e:
-        logging.debug('"%s" branch not found on origin remote:\n\t%s', appid, e)
-    if appid in git_repo.remotes.origin.refs:
-        start_point = f"origin/{appid}"
-        for commit in git_repo.iter_commits(
-            f'{upstream_main}...{start_point}', right_only=True
-        ):
-            if commit.committer.email != BOT_EMAIL or commit.author.email != BOT_EMAIL:
-                return False
-    else:
-        start_point = upstream_main
-    git_repo.git.checkout('-B', appid, start_point)
-    git_repo.git.rebase(upstream_main, strategy_option='ours', kill_after_timeout=120)
-    return True
-
-
-def get_changes_versus_ref(git_repo, ref, f):
-    changes = []
-    for m in re.findall(
-        r"^[+-].*", git_repo.git.diff(f"{ref}", '--', f), flags=re.MULTILINE
-    ):
-        if not re.match(r"^(\+\+\+|---) ", m):
-            changes.append(m)
-    return changes
-
-
-def push_commits(branch_name='checkupdates'):
-    """Make git branch then push commits as merge request.
-
-    The appid is parsed from the actual file that was changed so that
-    only the right branch is ever updated.
-
-    This uses the appid as the standard branch name so that there is
-    only ever one open merge request per-app.  If multiple apps are
-    included in the branch, then 'checkupdates' is used as branch
-    name.  This is to support the old way operating, e.g. in batches.
-
-    This uses GitLab "Push Options" to create a merge request. Git
-    Push Options are config data that can be sent via `git push
-    --push-option=... origin foo`.
-
-    References
-    ----------
-    * https://docs.gitlab.com/ee/user/project/push_options.html
-
-    """
-    if branch_name != "checkupdates":
-        if callable(getattr(git.SymbolicReference, "_check_ref_name_valid", None)):
-            git.SymbolicReference._check_ref_name_valid(branch_name)
-
-    git_repo = git.Repo.init('.')
-    upstream_main = get_upstream_main_branch(git_repo)
-    files = set()
-    for commit in git_repo.iter_commits(f'{upstream_main}...HEAD', right_only=True):
-        files.update(commit.stats.files.keys())
-
-    files = list(files)
-    if len(files) == 1:
-        m = re.match(r'metadata/(\S+)\.yml', files[0])
-        if m:
-            branch_name = m.group(1)  # appid
-    if not files:
-        return
-
-    # https://git-scm.com/docs/git-check-ref-format Git refname can't end with .lock
-    if branch_name.endswith(".lock"):
-        branch_name = f"{branch_name}_"
-
-    remote = git_repo.remotes.origin
-    if branch_name in remote.refs:
-        if not get_changes_versus_ref(git_repo, f'origin/{branch_name}', files[0]):
-            return
-
-    git_repo.create_head(branch_name, force=True)
-    push_options = [
-        'merge_request.create',
-        'merge_request.remove_source_branch',
-        'merge_request.title=bot: ' + git_repo.branches[branch_name].commit.summary,
-        'merge_request.description='
-        + '~%s checkupdates-bot run %s' % (branch_name, os.getenv('CI_JOB_URL')),
-    ]
-
-    # mark as draft if there are only changes to CurrentVersion:
-    current_version_only = True
-    for m in get_changes_versus_ref(git_repo, upstream_main, files[0]):
-        if not re.match(r"^[-+]CurrentVersion", m):
-            current_version_only = False
-            break
-    if current_version_only:
-        push_options.append('merge_request.draft')
-
-    progress = git.RemoteProgress()
-
-    pushinfos = remote.push(
-        f"HEAD:refs/heads/{branch_name}",
-        progress=progress,
-        force=True,
-        set_upstream=True,
-        push_option=push_options,
-    )
-
-    for pushinfo in pushinfos:
-        logging.info(pushinfo.summary)
-        # Show potentially useful messages from git remote
-        if progress:
-            for line in progress.other_lines:
-                logging.info(line)
-        if pushinfo.flags & (
-            git.remote.PushInfo.ERROR
-            | git.remote.PushInfo.REJECTED
-            | git.remote.PushInfo.REMOTE_FAILURE
-            | git.remote.PushInfo.REMOTE_REJECTED
-        ):
-            raise FDroidException(
-                f'{remote.url} push failed: {pushinfo.flags} {pushinfo.summary}'
-            )
-        else:
-            logging.info(remote.url + ': ' + pushinfo.summary)
-
-
-def prune_empty_appid_branches(git_repo=None, main_branch='main'):
-    """Remove empty branches from checkupdates-bot git remote."""
-    if git_repo is None:
-        git_repo = git.Repo.init('.')
-    upstream_main = get_upstream_main_branch(git_repo)
-    main_branch = upstream_main.split('/')[1]
-
-    remote = git_repo.remotes.origin
-    remote.update(prune=True)
-    merged_branches = git_repo.git().branch(remotes=True, merged=upstream_main).split()
-    for remote_branch in merged_branches:
-        if not remote_branch or '/' not in remote_branch:
-            continue
-        if remote_branch.split('/')[1] not in (main_branch, 'HEAD'):
-            for ref in git_repo.remotes.origin.refs:
-                if remote_branch == ref.name:
-                    remote.push(':%s' % ref.remote_head, force=True)  # rm remote branch
-
-
 def status_update_json(processed: list, failed: dict) -> None:
     """Output a JSON file with metadata about this run."""
     logging.debug(_('Outputting JSON'))
@@ -901,8 +716,6 @@ def main():
                         help=_("Only process apps with auto-updates"))
     parser.add_argument("--commit", action="store_true", default=False,
                         help=_("Commit changes"))
-    parser.add_argument("--merge-request", action="store_true", default=False,
-                        help=_("Commit changes, push, then make a merge request"))
     parser.add_argument("--allow-dirty", action="store_true", default=False,
                         help=_("Run on git repo that has uncommitted changes"))
     metadata.add_metadata_arguments(parser)
@@ -917,11 +730,10 @@ def main():
             logging.error(_('Build metadata git repo has uncommited changes!'))
             sys.exit(1)
 
-    if options.merge_request and not (options.appid and len(options.appid) == 1):
+    # Get all apps...
-        logging.error(_('--merge-request only runs on a single appid!'))
+    allapps = metadata.read_metadata()
-        sys.exit(1)
 
-    apps = common.read_app_args(options.appid)
+    apps = common.read_app_args(options.appid, allapps, False)
 
     processed = []
     failed = dict()
@@ -936,17 +748,7 @@ def main():
         logging.info(msg)
 
         try:
-            if options.merge_request:
+            checkupdates_app(app, options.auto, options.commit)
-                if not checkout_appid_branch(appid):
-                    msg = _("...checkupdate failed for {appid} : {error}").format(
-                        appid=appid,
-                        error='Open merge request with human edits, skipped.',
-                    )
-                    logging.warning(msg)
-                    failed[appid] = msg
-                    continue
-
-            checkupdates_app(app, options.auto, options.commit or options.merge_request)
             processed.append(appid)
         except Exception as e:
             msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e)
@@ -955,10 +757,6 @@ def main():
             failed[appid] = str(e)
             exit_code = 1
 
-    if options.appid and options.merge_request:
-        push_commits()
-        prune_empty_appid_branches()
-
     status_update_json(processed, failed)
     sys.exit(exit_code)
 

fdroidserver/deploy.py

@@ -16,28 +16,29 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import configparser
-import glob
-import json
-import logging
-import os
-import pathlib
-import re
-import shutil
-import subprocess
 import sys
+import glob
+import hashlib
+import json
+import os
+import re
+import subprocess
 import time
 import urllib
-from argparse import ArgumentParser
 from typing import Dict, List
-
-import git
-import yaml
 from git import Repo
-
+import yaml
+from argparse import ArgumentParser
+import logging
+from shlex import split
+import pathlib
+import shutil
+import git
 import fdroidserver.github
 
-from . import _, common, index
+from . import _
+from . import common
+from . import index
 from .exception import FDroidException
 
 config = None
@@ -47,10 +48,11 @@ GIT_BRANCH = 'master'
 
 BINARY_TRANSPARENCY_DIR = 'binary_transparency'
 
+AUTO_S3CFG = '.fdroid-deploy-s3cfg'
+USER_S3CFG = 's3cfg'
+USER_RCLONE_CONF = None
 REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*')
 
-EMBEDDED_RCLONE_CONF = 'rclone.conf'
-
 
 def _get_index_file_paths(base_dir):
     """Return the list of files to be synced last, since they finalize the deploy.
@@ -59,15 +61,8 @@ def _get_index_file_paths(base_dir):
     services can take a while.  So the index files should be updated
     last.  That ensures that the package files are available when the
     client learns about them from the new index files.
-
-    signer-index.* are only published in the repo/ section.
-
     """
-    return [
+    return [os.path.join(base_dir, filename) for filename in common.INDEX_FILES]
-        os.path.join(base_dir, filename)
-        for filename in common.INDEX_FILES
-        if not (filename.startswith('signer-index.') and base_dir.endswith('archive'))
-    ]
 
 
 def _get_index_excludes(base_dir):
@@ -97,161 +92,399 @@ def _remove_missing_files(files: List[str]) -> List[str]:
     return existing
 
 
-def _generate_rclone_include_pattern(files):
-    """Generate a pattern for rclone's --include flag (https://rclone.org/filtering/)."""
-    return "{" + ",".join(sorted(set(files))) + "}"
-
-
 def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=False):
-    """Sync the directory `repo_section` (including subdirectories) to AWS S3 US East.
+    """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
 
-    This is a shim function for public API compatibility.
+    The contents of that subdir of the
-
+    bucket will first be deleted.
-    Requires AWS credentials set as environment variables:
-    https://rclone.org/s3/#authentication
 
+    Requires AWS credentials set in config.yml: awsaccesskeyid, awssecretkey
     """
+    logging.debug(
+        f'''Syncing "{repo_section}" to Amazon S3 bucket "{config['awsbucket']}"'''
+    )
+
+    if common.set_command_in_config('s3cmd') and common.set_command_in_config('rclone'):
+        logging.info(
+            'Both rclone and s3cmd are installed. Checking config.yml for preference.'
+        )
+        if config['s3cmd'] is not True and config['rclone'] is not True:
+            logging.warning(
+                'No syncing tool set in config.yml!. Defaulting to using s3cmd'
+            )
+            update_awsbucket_s3cmd(repo_section, is_index_only)
+        if config['s3cmd'] is True and config['rclone'] is True:
+            logging.warning(
+                'Both syncing tools set in config.yml!. Defaulting to using s3cmd'
+            )
+            update_awsbucket_s3cmd(repo_section, is_index_only)
+        if config['s3cmd'] is True and config['rclone'] is not True:
+            update_awsbucket_s3cmd(repo_section, is_index_only)
+        if config['rclone'] is True and config['s3cmd'] is not True:
+            update_remote_storage_with_rclone(
+                repo_section, is_index_only, verbose, quiet
+            )
+
+    elif common.set_command_in_config('s3cmd'):
+        update_awsbucket_s3cmd(repo_section, is_index_only)
+    elif common.set_command_in_config('rclone'):
         update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet)
+    else:
+        update_awsbucket_libcloud(repo_section, is_index_only)
+
+
+def update_awsbucket_s3cmd(repo_section, is_index_only=False):
+    """Upload using the CLI tool s3cmd, which provides rsync-like sync.
+
+    The upload is done in multiple passes to reduce the chance of
+    interfering with an existing client-server interaction.  In the
+    first pass, only new files are uploaded.  In the second pass,
+    changed files are uploaded, overwriting what is on the server.  On
+    the third/last pass, the indexes are uploaded, and any removed
+    files are deleted from the server.  The last pass is the only pass
+    to use a full MD5 checksum of all files to detect changes.
+    """
+    logging.debug(_('Using s3cmd to sync with: {url}').format(url=config['awsbucket']))
+
+    if os.path.exists(USER_S3CFG):
+        logging.info(_('Using "{path}" for configuring s3cmd.').format(path=USER_S3CFG))
+        configfilename = USER_S3CFG
+    else:
+        fd = os.open(AUTO_S3CFG, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
+        logging.debug(
+            _('Creating "{path}" for configuring s3cmd.').format(path=AUTO_S3CFG)
+        )
+        os.write(fd, '[default]\n'.encode('utf-8'))
+        os.write(
+            fd, ('access_key = ' + config['awsaccesskeyid'] + '\n').encode('utf-8')
+        )
+        os.write(fd, ('secret_key = ' + config['awssecretkey'] + '\n').encode('utf-8'))
+        os.close(fd)
+        configfilename = AUTO_S3CFG
+
+    s3bucketurl = 's3://' + config['awsbucket']
+    s3cmd = [config['s3cmd'], '--config=' + configfilename]
+    if subprocess.call(s3cmd + ['info', s3bucketurl]) != 0:
+        logging.warning(_('Creating new S3 bucket: {url}').format(url=s3bucketurl))
+        if subprocess.call(s3cmd + ['mb', s3bucketurl]) != 0:
+            logging.error(
+                _('Failed to create S3 bucket: {url}').format(url=s3bucketurl)
+            )
+            raise FDroidException()
+
+    s3cmd_sync = s3cmd + ['sync', '--acl-public']
+    options = common.get_options()
+    if options and options.verbose:
+        s3cmd_sync += ['--verbose']
+    if options and options.quiet:
+        s3cmd_sync += ['--quiet']
+
+    s3url = s3bucketurl + '/fdroid/'
+
+    logging.debug(
+        _('s3cmd sync indexes {path} to {url} and delete').format(
+            path=repo_section, url=s3url
+        )
+    )
+
+    if is_index_only:
+        logging.debug(
+            _('s3cmd syncs indexes from {path} to {url} and deletes removed').format(
+                path=repo_section, url=s3url
+            )
+        )
+        sync_indexes_flags = []
+        sync_indexes_flags.extend(_get_index_includes(repo_section))
+        sync_indexes_flags.append('--delete-removed')
+        sync_indexes_flags.append('--delete-after')
+        if options.no_checksum:
+            sync_indexes_flags.append('--no-check-md5')
+        else:
+            sync_indexes_flags.append('--check-md5')
+        returncode = subprocess.call(
+            s3cmd_sync + sync_indexes_flags + [repo_section, s3url]
+        )
+        if returncode != 0:
+            raise FDroidException()
+    else:
+        logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
+        logging.debug(_('Running first pass with MD5 checking disabled'))
+        excludes = _get_index_excludes(repo_section)
+        returncode = subprocess.call(
+            s3cmd_sync
+            + excludes
+            + ['--no-check-md5', '--skip-existing', repo_section, s3url]
+        )
+        if returncode != 0:
+            raise FDroidException()
+        logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
+        returncode = subprocess.call(
+            s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
+        )
+        if returncode != 0:
+            raise FDroidException()
+
+        logging.debug(
+            _('s3cmd sync indexes {path} to {url} and delete').format(
+                path=repo_section, url=s3url
+            )
+        )
+        s3cmd_sync.append('--delete-removed')
+        s3cmd_sync.append('--delete-after')
+        if options.no_checksum:
+            s3cmd_sync.append('--no-check-md5')
+        else:
+            s3cmd_sync.append('--check-md5')
+        if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0:
+            raise FDroidException()
 
 
 def update_remote_storage_with_rclone(
-    repo_section,
+    repo_section, is_index_only=False, verbose=False, quiet=False
-    awsbucket,
-    is_index_only=False,
-    verbose=False,
-    quiet=False,
-    checksum=False,
 ):
-    """Sync the directory `repo_section` (including subdirectories) to configed cloud services.
+    """
+    Upload fdroid repo folder to remote storage using rclone sync.
 
     Rclone sync can send the files to any supported remote storage
-    service once without numerous polling.  If remote storage is S3 e.g
+    service once without numerous polling.
-    AWS S3, Wasabi, Filebase, etc, then path will be
+    If remote storage is s3 e.g aws s3, wasabi, filebase then path will be
-    bucket_name/fdroid/repo where bucket_name will be an S3 bucket. If
+    bucket_name/fdroid/repo where bucket_name will be an s3 bucket
-    remote storage is storage drive/sftp e.g google drive, rsync.net the
+    If remote storage is storage drive/sftp e.g google drive, rsync.net
-    new path will be bucket_name/fdroid/repo where bucket_name will be a
+    the new path will be bucket_name/fdroid/repo where bucket_name
-    folder
+    will be a folder
-
-    See https://rclone.org/docs/#config-config-file
-
-    rclone filtering works differently than rsync.  For example,
-    "--include" implies "--exclude **" at the end of an rclone internal
-    filter list.
-
-    If rclone.conf is in the root of the repo, then it will be preferred
-    over the rclone default config paths.
 
+    Better than the s3cmd command as it does the syncing in one command
+    Check https://rclone.org/docs/#config-config-file (optional config file)
     """
-    logging.debug(_('Using rclone to sync to "{name}"').format(name=awsbucket))
+    logging.debug(_('Using rclone to sync with: {url}').format(url=config['awsbucket']))
 
-    rclone_config = config.get('rclone_config', [])
+    if config.get('path_to_custom_rclone_config') is not None:
-    if rclone_config and isinstance(rclone_config, str):
+        USER_RCLONE_CONF = config['path_to_custom_rclone_config']
-        rclone_config = [rclone_config]
+        if os.path.exists(USER_RCLONE_CONF):
-
+            logging.info("'path_to_custom_rclone_config' found in config.yml")
-    path = config.get('path_to_custom_rclone_config')
+            logging.info(
-    if path:
+                _('Using "{path}" for syncing with remote storage.').format(
-        if not os.path.exists(path):
+                    path=USER_RCLONE_CONF
-            logging.error(
-                _('path_to_custom_rclone_config: "{path}" does not exist!').format(
-                    path=path
                 )
             )
-            sys.exit(1)
+            configfilename = USER_RCLONE_CONF
-        configfilename = path
-    elif os.path.exists(EMBEDDED_RCLONE_CONF):
-        path = EMBEDDED_RCLONE_CONF  # in this case, only for display
-        configfilename = EMBEDDED_RCLONE_CONF
-        if not rclone_config:
-            raise FDroidException(_("'rclone_config' must be set in config.yml!"))
         else:
+            logging.info('Custom configuration not found.')
+            logging.info(
+                'Using default configuration at {}'.format(
+                    subprocess.check_output(split("rclone config file")).decode("utf-8")
+                )
+            )
+            configfilename = None
+    else:
+        logging.warning("'path_to_custom_rclone_config' not found in config.yml")
+        logging.info('Custom configuration not found.')
+        logging.info(
+            'Using default configuration at {}'.format(
+                subprocess.check_output(split("rclone config file")).decode("utf-8")
+            )
+        )
         configfilename = None
-        output = subprocess.check_output(['rclone', 'config', 'file'], text=True)
-        default_config_path = output.split('\n')[-2]
-        if os.path.exists(default_config_path):
-            path = default_config_path
-    if path:
-        logging.info(_('Using "{path}" for rclone config.').format(path=path))
 
     upload_dir = 'fdroid/' + repo_section
 
-    if not rclone_config:
+    if not config.get('rclone_config') or not config.get('awsbucket'):
-        env = os.environ
-        # Check both canonical and backup names, but only tell user about canonical.
-        if not env.get("AWS_SECRET_ACCESS_KEY") and not env.get("AWS_SECRET_KEY"):
         raise FDroidException(
-                _(
+            _('To use rclone, rclone_config and awsbucket must be set in config.yml!')
-                    """"AWS_SECRET_ACCESS_KEY" must be set as an environmental variable!"""
-                )
-            )
-        if not env.get("AWS_ACCESS_KEY_ID") and not env.get('AWS_ACCESS_KEY'):
-            raise FDroidException(
-                _(""""AWS_ACCESS_KEY_ID" must be set as an environmental variable!""")
         )
 
-        default_remote = "AWS-S3-US-East-1"
+    if is_index_only:
-        env_rclone_config = configparser.ConfigParser()
+        sources = _get_index_file_paths(repo_section)
-        env_rclone_config.add_section(default_remote)
+        sources = _remove_missing_files(sources)
-        env_rclone_config.set(
+    else:
-            default_remote,
+        sources = [repo_section]
-            '; = This file is auto-generated by fdroid deploy, do not edit!',
+
-            '',
+    for source in sources:
+        if isinstance(config['rclone_config'], str):
+            rclone_sync_command = (
+                'rclone sync '
+                + source
+                + ' '
+                + config['rclone_config']
+                + ':'
+                + config['awsbucket']
+                + '/'
+                + upload_dir
             )
-        env_rclone_config.set(default_remote, "type", "s3")
-        env_rclone_config.set(default_remote, "provider", "AWS")
-        env_rclone_config.set(default_remote, "region", "us-east-1")
-        env_rclone_config.set(default_remote, "env_auth", "true")
 
-        configfilename = ".fdroid-deploy-rclone.conf"
+            rclone_sync_command = split(rclone_sync_command)
-        with open(configfilename, "w", encoding="utf-8") as autoconfigfile:
-            env_rclone_config.write(autoconfigfile)
-        rclone_config = [default_remote]
-
-    rclone_sync_command = ['rclone', 'sync', '--delete-after']
-    if configfilename:
-        rclone_sync_command += ['--config', configfilename]
-
-    if checksum:
-        rclone_sync_command.append('--checksum')
 
             if verbose:
                 rclone_sync_command += ['--verbose']
             elif quiet:
                 rclone_sync_command += ['--quiet']
 
-    # TODO copying update_serverwebroot rsync algo
+            if configfilename:
-    for remote_config in rclone_config:
+                rclone_sync_command += split('--config=' + configfilename)
-        complete_remote_path = f'{remote_config}:{awsbucket}/{upload_dir}'
+
-        logging.info(f'rclone sync to {complete_remote_path}')
+            complete_remote_path = (
-        if is_index_only:
+                config['rclone_config'] + ':' + config['awsbucket'] + '/' + upload_dir
-            index_only_files = common.INDEX_FILES + ['diff/*.*']
-            include_pattern = _generate_rclone_include_pattern(index_only_files)
-            cmd = rclone_sync_command + [
-                '--include',
-                include_pattern,
-                '--delete-excluded',
-                repo_section,
-                complete_remote_path,
-            ]
-            logging.info(cmd)
-            if subprocess.call(cmd) != 0:
-                raise FDroidException()
-        else:
-            cmd = (
-                rclone_sync_command
-                + _get_index_excludes(repo_section)
-                + [
-                    repo_section,
-                    complete_remote_path,
-                ]
             )
-            if subprocess.call(cmd) != 0:
+
+            logging.debug(
+                "rclone sync all files in " + source + ' to ' + complete_remote_path
+            )
+
+            if subprocess.call(rclone_sync_command) != 0:
                 raise FDroidException()
-            cmd = rclone_sync_command + [
+
-                repo_section,
+        if isinstance(config['rclone_config'], list):
-                complete_remote_path,
+            for remote_config in config['rclone_config']:
+                rclone_sync_command = (
+                    'rclone sync '
+                    + source
+                    + ' '
+                    + remote_config
+                    + ':'
+                    + config['awsbucket']
+                    + '/'
+                    + upload_dir
+                )
+
+                rclone_sync_command = split(rclone_sync_command)
+
+                if verbose:
+                    rclone_sync_command += ['--verbose']
+                elif quiet:
+                    rclone_sync_command += ['--quiet']
+
+                if configfilename:
+                    rclone_sync_command += split('--config=' + configfilename)
+
+                complete_remote_path = (
+                    remote_config + ':' + config['awsbucket'] + '/' + upload_dir
+                )
+
+                logging.debug(
+                    "rclone sync all files in " + source + ' to ' + complete_remote_path
+                )
+
+                if subprocess.call(rclone_sync_command) != 0:
+                    raise FDroidException()
+
+
+def update_awsbucket_libcloud(repo_section, is_index_only=False):
+    """No summary.
+
+    Upload the contents of the directory `repo_section` (including
+    subdirectories) to the AWS S3 "bucket".
+
+    The contents of that subdir of the
+    bucket will first be deleted.
+
+    Requires AWS credentials set in config.yml: awsaccesskeyid, awssecretkey
+    """
+    logging.debug(
+        _('using Apache libcloud to sync with {url}').format(url=config['awsbucket'])
+    )
+
+    import libcloud.security
+
+    libcloud.security.VERIFY_SSL_CERT = True
+    from libcloud.storage.types import Provider, ContainerDoesNotExistError
+    from libcloud.storage.providers import get_driver
+
+    if not config.get('awsaccesskeyid') or not config.get('awssecretkey'):
+        raise FDroidException(
+            _(
+                'To use awsbucket, awssecretkey and awsaccesskeyid must also be set in config.yml!'
+            )
+        )
+    awsbucket = config['awsbucket']
+
+    if os.path.exists(USER_S3CFG):
+        raise FDroidException(
+            _('"{path}" exists but s3cmd is not installed!').format(path=USER_S3CFG)
+        )
+
+    cls = get_driver(Provider.S3)
+    driver = cls(config['awsaccesskeyid'], config['awssecretkey'])
+    try:
+        container = driver.get_container(container_name=awsbucket)
+    except ContainerDoesNotExistError:
+        container = driver.create_container(container_name=awsbucket)
+        logging.info(_('Created new container "{name}"').format(name=container.name))
+
+    upload_dir = 'fdroid/' + repo_section
+    objs = dict()
+    for obj in container.list_objects():
+        if obj.name.startswith(upload_dir + '/'):
+            objs[obj.name] = obj
+
+    if is_index_only:
+        index_files = [
+            f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)
         ]
-            if subprocess.call(cmd) != 0:
+        files_to_upload = [
-                raise FDroidException()
+            os.path.join(root, name)
+            for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section))
+            for name in files
+        ]
+        files_to_upload = list(set(files_to_upload) & set(index_files))
+        files_to_upload = _remove_missing_files(files_to_upload)
+
+    else:
+        files_to_upload = [
+            os.path.join(root, name)
+            for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section))
+            for name in files
+        ]
+
+    for file_to_upload in files_to_upload:
+        upload = False
+        object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
+        if object_name not in objs:
+            upload = True
+        else:
+            obj = objs.pop(object_name)
+            if obj.size != os.path.getsize(file_to_upload):
+                upload = True
+            else:
+                # if the sizes match, then compare by MD5
+                md5 = hashlib.md5()  # nosec AWS uses MD5
+                with open(file_to_upload, 'rb') as f:
+                    while True:
+                        data = f.read(8192)
+                        if not data:
+                            break
+                        md5.update(data)
+                if obj.hash != md5.hexdigest():
+                    s3url = 's3://' + awsbucket + '/' + obj.name
+                    logging.info(' deleting ' + s3url)
+                    if not driver.delete_object(obj):
+                        logging.warning('Could not delete ' + s3url)
+                    upload = True
+
+        if upload:
+            logging.debug(' uploading "' + file_to_upload + '"...')
+            extra = {'acl': 'public-read'}
+            if file_to_upload.endswith('.sig'):
+                extra['content_type'] = 'application/pgp-signature'
+            elif file_to_upload.endswith('.asc'):
+                extra['content_type'] = 'application/pgp-signature'
+            path = os.path.relpath(file_to_upload)
+            logging.info(f' uploading {path} to s3://{awsbucket}/{object_name}')
+            with open(file_to_upload, 'rb') as iterator:
+                obj = driver.upload_object_via_stream(
+                    iterator=iterator,
+                    container=container,
+                    object_name=object_name,
+                    extra=extra,
+                )
+    # delete the remnants in the bucket, they do not exist locally
+    while objs:
+        object_name, obj = objs.popitem()
+        s3url = 's3://' + awsbucket + '/' + object_name
+        if object_name.startswith(upload_dir):
+            logging.warning(' deleting ' + s3url)
+            driver.delete_object(obj)
+        else:
+            logging.info(' skipping ' + s3url)
 
 
 def update_serverwebroot(serverwebroot, repo_section):
@@ -421,13 +654,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
     For history, there is the archive section, and there is the binary
     transparency log.
 
-    This will attempt to use the existing remote branch so that it does
-    not have to push all of the files in the repo each time.  Old setups
-    or runs of `fdroid nightly` might use the "master" branch.  For the
-    "index only" mode, it will recreate the branch from scratch each
-    time since usually all the files are changed.  In any case, the
-    index files are small compared to the full repo.
-
     """
     from clint.textui import progress
 
@@ -498,7 +724,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             if is_index_only:
                 local_branch_name = 'index_only'
             else:
-                local_branch_name = GIT_BRANCH
+                local_branch_name = 'full'
             if local_branch_name in repo.heads:
                 repo.git.switch(local_branch_name)
             else:
@@ -623,7 +849,6 @@ def upload_to_servergitmirror(
                 | git.remote.PushInfo.REMOTE_REJECTED
             ):
                 # Show potentially useful messages from git remote
-                if progress:
                 for line in progress.other_lines:
                     if line.startswith('remote:'):
                         logging.debug(line)
@@ -658,9 +883,8 @@ def upload_to_android_observatory(repo_section):
 def upload_apk_to_android_observatory(path):
     # depend on requests and lxml only if users enable AO
     import requests
-    from lxml.html import fromstring
-
     from . import net
+    from lxml.html import fromstring
 
     apkfilename = os.path.basename(path)
     r = requests.post(
@@ -971,7 +1195,7 @@ def upload_to_github_releases_repo(repo_conf, release_infos, global_gh_token):
     if not token:
         logging.warning(
             _(
-                "One of the 'github_releases' config items is missing the "
+                "One of the 'github_releases' config itmes is missing the "
                 "'token' value. skipping ..."
             )
         )
@@ -982,7 +1206,7 @@ def upload_to_github_releases_repo(repo_conf, release_infos, global_gh_token):
     if not conf_package_names:
         logging.warning(
             _(
-                "One of the 'github_releases' config items is missing the "
+                "One of the 'github_releases' config itmes is missing the "
                 "'packageNames' value. skipping ..."
             )
         )
@@ -1146,16 +1370,8 @@ def main():
             # update_servergitmirrors will take care of multiple mirrors so don't need a foreach
             update_servergitmirrors(config['servergitmirrors'], repo_section)
         if config.get('awsbucket'):
-            awsbucket = config['awsbucket']
             index_only = config.get('awsbucket_index_only')
-            update_remote_storage_with_rclone(
+            update_awsbucket(repo_section, index_only, options.verbose, options.quiet)
-                repo_section,
-                awsbucket,
-                index_only,
-                options.verbose,
-                options.quiet,
-                not options.no_checksum,
-            )
         if config.get('androidobservatory'):
             upload_to_android_observatory(repo_section)
         if config.get('virustotal_apikey'):

fdroidserver/exception.py

@@ -35,10 +35,6 @@ class VCSException(FDroidException):
     pass
 
 
-class NoVersionCodeException(FDroidException):
-    pass
-
-
 class NoSubmodulesException(VCSException):
     pass
 

fdroidserver/github.py

@@ -18,20 +18,17 @@
 
 import json
 import pathlib
-import urllib.parse
 import urllib.request
+import urllib.parse
 
 
 class GithubApi:
-    """Wrapper for some select calls to GitHub Json/REST API.
+    """
+    Warpper for some select calls to GitHub Json/REST API.
 
     This class wraps some calls to api.github.com. This is not intended to be a
     general API wrapper. Instead it's purpose is to return pre-filtered and
     transformed data that's playing well with other fdroidserver functions.
-
-    With the GitHub API, the token is optional, but it has pretty
-    severe rate limiting.
-
     """
 
     def __init__(self, api_token, repo_path):
@@ -44,10 +41,9 @@ class GithubApi:
     def _req(self, url, data=None):
         h = {
             "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {self._api_token}",
             "X-GitHub-Api-Version": "2022-11-28",
         }
-        if self._api_token:
-            h["Authorization"] = f"Bearer {self._api_token}"
         return urllib.request.Request(
             url,
             headers=h,
@@ -69,17 +65,6 @@ class GithubApi:
         released_tags = self.list_released_tags()
         return [x for x in all_tags if x not in released_tags]
 
-    def get_latest_apk(self):
-        req = self._req(
-            f"https://api.github.com/repos/{self._repo_path}/releases/latest"
-        )
-        with urllib.request.urlopen(req) as resp:  # nosec CWE-22 disable bandit warning
-            assets = json.load(resp)['assets']
-            for asset in assets:
-                url = asset.get('browser_download_url')
-                if url and url.endswith('.apk'):
-                    return url
-
     def tag_exists(self, tag):
         """
         Check if git tag is present on github.

fdroidserver/gpgsign.py

@@ -16,13 +16,14 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import glob
-import logging
 import os
-import time
+import glob
 from argparse import ArgumentParser
+import logging
+import time
 
-from . import _, common
+from . import _
+from . import common
 from .common import FDroidPopen
 from .exception import FDroidException
 

fdroidserver/import_subcommand.py

@@ -18,64 +18,34 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import json
-import logging
 import os
 import re
-import shutil
 import stat
-import sys
 import urllib
-from argparse import ArgumentParser
-from pathlib import Path
-from typing import Optional
 
 import git
+import json
+import shutil
+import sys
 import yaml
+from argparse import ArgumentParser
+import logging
+from pathlib import Path
+from typing import Optional
 
 try:
     from yaml import CSafeLoader as SafeLoader
 except ImportError:
     from yaml import SafeLoader
 
-from . import _, common, metadata
+from . import _
+from . import common
+from . import metadata
 from .exception import FDroidException
 
+
 config = None
 
-SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?')
-GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''')
-APPLICATION_ID_REGEX = re.compile(r'''\s*applicationId\s=?\s?['"].*['"]''')
-
-
-def get_all_gradle_and_manifests(build_dir):
-    paths = []
-    for root, dirs, files in os.walk(build_dir):
-        for f in sorted(files):
-            if f == 'AndroidManifest.xml' or f.endswith(('.gradle', '.gradle.kts')):
-                full = Path(root) / f
-                paths.append(full)
-    return paths
-
-
-def get_gradle_subdir(build_dir, paths):
-    """Get the subdir where the gradle build is based."""
-    first_gradle_dir = None
-    for path in paths:
-        if not first_gradle_dir:
-            first_gradle_dir = path.parent.relative_to(build_dir)
-        if path.exists() and SETTINGS_GRADLE_REGEX.match(path.name):
-            for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text(encoding='utf-8')):
-                for f in (path.parent / m.group(1)).glob('build.gradle*'):
-                    with f.open(encoding='utf-8') as fp:
-                        for line in fp:
-                            if common.ANDROID_PLUGIN_REGEX.match(
-                                line
-                            ) or APPLICATION_ID_REGEX.match(line):
-                                return f.parent.relative_to(build_dir)
-    if first_gradle_dir and first_gradle_dir != Path('.'):
-        return first_gradle_dir
-
 
 def handle_retree_error_on_windows(function, path, excinfo):
     """Python can't remove a readonly file on Windows so chmod first."""
@@ -130,7 +100,6 @@ def getrepofrompage(url: str) -> tuple[Optional[str], str]:
         The found repository type or None if an error occured.
     address_or_reason
         The address to the found repository or the reason if an error occured.
-
     """
     if not url.startswith('http'):
         return (None, _('{url} does not start with "http"!'.format(url=url)))
@@ -199,7 +168,6 @@ def get_app_from_url(url: str) -> metadata.App:
         If the VCS type could not be determined.
     :exc:`ValueError`
         If the URL is invalid.
-
     """
     parsed = urllib.parse.urlparse(url)
     invalid_url = False
@@ -275,29 +243,18 @@ def main():
     # Parse command line...
     parser = ArgumentParser()
     common.setup_global_opts(parser)
-    parser.add_argument("-u", "--url", help=_("Project URL to import from."))
+    parser.add_argument("-u", "--url", default=None,
-    parser.add_argument(
+                        help=_("Project URL to import from."))
-        "-s",
+    parser.add_argument("-s", "--subdir", default=None,
-        "--subdir",
+                        help=_("Path to main Android project subdirectory, if not in root."))
-        help=_("Path to main Android project subdirectory, if not in root."),
+    parser.add_argument("-c", "--categories", default=None,
-    )
+                        help=_("Comma separated list of categories."))
-    parser.add_argument(
+    parser.add_argument("-l", "--license", default=None,
-        "-c",
+                        help=_("Overall license of the project."))
-        "--categories",
+    parser.add_argument("--omit-disable", action="store_true", default=False,
-        help=_("Comma separated list of categories."),
+                        help=_("Do not add 'disable:' to the generated build entries"))
-    )
+    parser.add_argument("--rev", default=None,
-    parser.add_argument("-l", "--license", help=_("Overall license of the project."))
+                        help=_("Allows a different revision (or git branch) to be specified for the initial import"))
-    parser.add_argument(
-        "--omit-disable",
-        action="store_true",
-        help=_("Do not add 'disable:' to the generated build entries"),
-    )
-    parser.add_argument(
-        "--rev",
-        help=_(
-            "Allows a different revision (or git branch) to be specified for the initial import"
-        ),
-    )
     metadata.add_metadata_arguments(parser)
     options = common.parse_args(parser)
     metadata.warnings_action = options.W
@@ -311,20 +268,24 @@ def main():
 
     local_metadata_files = common.get_local_metadata_files()
     if local_metadata_files:
-        raise FDroidException(
+        raise FDroidException(_("This repo already has local metadata: %s") % local_metadata_files[0])
-            _("This repo already has local metadata: %s") % local_metadata_files[0]
-        )
 
     build = metadata.Build()
-    app = metadata.App()
     if options.url is None and Path('.git').is_dir():
+        app = metadata.App()
+        app.AutoName = Path.cwd().name
         app.RepoType = 'git'
-        tmp_importer_dir = Path.cwd()
+
-        git_repo = git.Repo(tmp_importer_dir)
+        if Path('build.gradle').exists() or Path('build.gradle.kts').exists():
+            build.gradle = ['yes']
+
+        git_repo = git.Repo(Path.cwd())
         for remote in git.Remote.iter_items(git_repo):
             if remote.name == 'origin':
                 url = git_repo.remotes.origin.url
-                app = get_app_from_url(url)
+                if url.startswith('https://git'):  # github, gitlab
+                    app.SourceCode = url.rstrip('.git')
+                app.Repo = url
                 break
         write_local_file = True
     elif options.url:
@@ -333,28 +294,25 @@ def main():
         git_repo = git.Repo(tmp_importer_dir)
 
         if not options.omit_disable:
-            build.disable = (
+            build.disable = 'Generated by `fdroid import` - check version fields and commitid'
-                'Generated by `fdroid import` - check version fields and commitid'
-            )
         write_local_file = False
     else:
         raise FDroidException("Specify project url.")
 
-    app.AutoUpdateMode = 'Version'
     app.UpdateCheckMode = 'Tags'
-    build.commit = common.get_head_commit_id(tmp_importer_dir)
+    build.commit = common.get_head_commit_id(git_repo)
 
     # Extract some information...
-    paths = get_all_gradle_and_manifests(tmp_importer_dir)
+    paths = common.get_all_gradle_and_manifests(tmp_importer_dir)
-    gradle_subdir = get_gradle_subdir(tmp_importer_dir, paths)
+    subdir = common.get_gradle_subdir(tmp_importer_dir, paths)
     if paths:
         versionName, versionCode, appid = common.parse_androidmanifests(paths, app)
         if not appid:
             raise FDroidException(_("Couldn't find Application ID"))
         if not versionName:
-            logging.warning(_('Could not find latest versionName'))
+            logging.warning(_('Could not find latest version name'))
         if not versionCode:
-            logging.warning(_('Could not find latest versionCode'))
+            logging.warning(_('Could not find latest version code'))
     else:
         raise FDroidException(_("No gradle project could be found. Specify --subdir?"))
 
@@ -364,15 +322,16 @@ def main():
 
     # Create a build line...
     build.versionName = versionName or 'Unknown'
-    app.CurrentVersion = build.versionName
     build.versionCode = versionCode or 0
-    app.CurrentVersionCode = build.versionCode
     if options.subdir:
         build.subdir = options.subdir
-    elif gradle_subdir:
+        build.gradle = ['yes']
-        build.subdir = gradle_subdir.as_posix()
+    elif subdir:
+        build.subdir = subdir.as_posix()
+        build.gradle = ['yes']
+    else:
         # subdir might be None
-    subdir = Path(tmp_importer_dir / build.subdir) if build.subdir else tmp_importer_dir
+        subdir = Path()
 
     if options.license:
         app.License = options.license
@@ -380,23 +339,23 @@ def main():
         app.Categories = options.categories.split(',')
     if (subdir / 'jni').exists():
         build.buildjni = ['yes']
-    if (subdir / 'build.gradle').exists() or (subdir / 'build.gradle.kts').exists():
+    if (subdir / 'build.gradle').exists() or (subdir / 'build.gradle').exists():
         build.gradle = ['yes']
 
-    app.AutoName = common.fetch_real_name(subdir, build.gradle)
-
     package_json = tmp_importer_dir / 'package.json'  # react-native
     pubspec_yaml = tmp_importer_dir / 'pubspec.yaml'  # flutter
     if package_json.exists():
         build.sudo = [
             'sysctl fs.inotify.max_user_watches=524288 || true',
-            'apt-get update',
+            'curl -Lo node.tar.gz https://nodejs.org/download/release/v19.3.0/node-v19.3.0-linux-x64.tar.gz',
-            'apt-get install -y npm',
+            'echo "b525028ae5bb71b5b32cb7fce903ccce261dbfef4c7dd0f3e0ffc27cd6fc0b3f node.tar.gz" | sha256sum -c -',
+            'tar xzf node.tar.gz --strip-components=1 -C /usr/local/',
+            'npm -g install yarn',
         ]
         build.init = ['npm install --build-from-source']
         with package_json.open() as fp:
             data = json.load(fp)
-        app.AutoName = app.AutoName or data.get('name')
+        app.AutoName = data.get('name', app.AutoName)
         app.License = data.get('license', app.License)
         app.Description = data.get('description', app.Description)
         app.WebSite = data.get('homepage', app.WebSite)
@@ -406,11 +365,11 @@ def main():
         if app_json.exists():
             with app_json.open() as fp:
                 data = json.load(fp)
-            app.AutoName = app.AutoName or data.get('name')
+            app.AutoName = data.get('name', app.AutoName)
     if pubspec_yaml.exists():
         with pubspec_yaml.open() as fp:
             data = yaml.load(fp, Loader=SafeLoader)
-        app.AutoName = app.AutoName or data.get('name')
+        app.AutoName = data.get('name', app.AutoName)
         app.License = data.get('license', app.License)
         app.Description = data.get('description', app.Description)
         app.UpdateCheckData = 'pubspec.yaml|version:\\s.+\\+(\\d+)|.|version:\\s(.+)\\+'
@@ -446,11 +405,8 @@ def main():
         Path('build').mkdir(exist_ok=True)
         build_dir = Path('build') / appid
         if build_dir.exists():
-            logging.warning(
+            logging.warning(_('{path} already exists, ignoring import results!')
-                _('{path} already exists, ignoring import results!').format(
+                            .format(path=build_dir))
-                    path=build_dir
-                )
-            )
             sys.exit(1)
         elif tmp_importer_dir:
             # For Windows: Close the repo or a git.exe instance holds handles to repo

fdroidserver/index.py

@@ -20,49 +20,32 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Process the index files.
-
-This module is loaded by all fdroid subcommands since it is loaded in
-fdroidserver/__init__.py.  Any narrowly used dependencies should be
-imported where they are used to limit dependencies for subcommands
-like publish/signindex/gpgsign.  This eliminates the need to have
-these installed on the signing server.
-
-"""
-
-import calendar
 import collections
 import hashlib
 import json
 import logging
 import os
 import re
+import ruamel.yaml
 import shutil
-import sys
 import tempfile
 import urllib.parse
 import zipfile
+import calendar
+import qrcode
 from binascii import hexlify, unhexlify
 from datetime import datetime, timezone
 from pathlib import Path
 from xml.dom.minidom import Document
 
-from fdroidserver._yaml import yaml
+from . import _
-from fdroidserver.common import (
+from . import common
-    ANTIFEATURES_CONFIG_NAME,
+from . import metadata
-    CATEGORIES_CONFIG_NAME,
+from . import net
-    CONFIG_CONFIG_NAME,
+from . import signindex
-    DEFAULT_LOCALE,
+from fdroidserver.common import ANTIFEATURES_CONFIG_NAME, CATEGORIES_CONFIG_NAME, CONFIG_CONFIG_NAME, MIRRORS_CONFIG_NAME, RELEASECHANNELS_CONFIG_NAME, DEFAULT_LOCALE, FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
-    MIRRORS_CONFIG_NAME,
-    RELEASECHANNELS_CONFIG_NAME,
-    FDroidPopen,
-    FDroidPopenBytes,
-    load_publish_signer_fingerprints,
-)
 from fdroidserver.exception import FDroidException, VerificationException
 
-from . import _, common, metadata, signindex
-
 
 def make(apps, apks, repodir, archive):
     """Generate the repo index files.
@@ -94,7 +77,7 @@ def make(apps, apks, repodir, archive):
         sortedapps[appid] = apps[appid]
 
     repodict = collections.OrderedDict()
-    repodict['timestamp'] = datetime.now(timezone.utc)
+    repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc)
     repodict['version'] = METADATA_VERSION
 
     if common.config['repo_maxage'] != 0:
@@ -133,13 +116,14 @@ def make(apps, apks, repodir, archive):
                 raise TypeError(_('only accepts strings, lists, and tuples'))
         requestsdict[command] = packageNames
 
-    signer_fingerprints = load_publish_signer_fingerprints()
+    fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
 
-    make_v0(sortedapps, apks, repodir, repodict, requestsdict, signer_fingerprints)
+    make_v0(sortedapps, apks, repodir, repodict, requestsdict,
-    make_v1(sortedapps, apks, repodir, repodict, requestsdict, signer_fingerprints)
+            fdroid_signing_key_fingerprints)
-    make_v2(
+    make_v1(sortedapps, apks, repodir, repodict, requestsdict,
-        sortedapps, apks, repodir, repodict, requestsdict, signer_fingerprints, archive
+            fdroid_signing_key_fingerprints)
-    )
+    make_v2(sortedapps, apks, repodir, repodict, requestsdict,
+            fdroid_signing_key_fingerprints, archive)
     make_website(sortedapps, repodir, repodict)
     make_altstore(
         sortedapps,
@@ -160,25 +144,23 @@ def _should_file_be_generated(path, magic_string):
 
 
 def make_website(apps, repodir, repodict):
+    _ignored, repo_pubkey_fingerprint = extract_pubkey()
+    repo_pubkey_fingerprint_stripped = repo_pubkey_fingerprint.replace(" ", "")
+    link = repodict["address"]
+    link_fingerprinted = ('{link}?fingerprint={fingerprint}'
+                          .format(link=link, fingerprint=repo_pubkey_fingerprint_stripped))
     # do not change this string, as it will break updates for files with older versions of this string
     autogenerate_comment = "auto-generated - fdroid index updates will overwrite this file"
 
     if not os.path.exists(repodir):
         os.makedirs(repodir)
 
+    qrcode.make(link_fingerprinted).save(os.path.join(repodir, "index.png"))
+
     html_name = 'index.html'
     html_file = os.path.join(repodir, html_name)
 
     if _should_file_be_generated(html_file, autogenerate_comment):
-        import qrcode
-
-        _ignored, repo_pubkey_fingerprint = extract_pubkey()
-        repo_pubkey_fingerprint_stripped = repo_pubkey_fingerprint.replace(" ", "")
-        link = repodict["address"]
-        link_fingerprinted = '{link}?fingerprint={fingerprint}'.format(
-            link=link, fingerprint=repo_pubkey_fingerprint_stripped
-        )
-        qrcode.make(link_fingerprinted).save(os.path.join(repodir, "index.png"))
         with open(html_file, 'w') as f:
             name = repodict["name"]
             description = repodict["description"]
@@ -527,6 +509,7 @@ def package_metadata(app, repodir):
         "AuthorPhone",
         "AuthorWebSite",
         "Bitcoin",
+        "FlattrID",
         "Liberapay",
         "Litecoin",
         "OpenCollective",
@@ -595,10 +578,7 @@ def convert_version(version, app, repodir):
         ver["file"]["ipfsCIDv1"] = ipfsCIDv1
 
     if "srcname" in version:
-        ver["src"] = common.file_entry(
+        ver["src"] = common.file_entry(os.path.join(repodir, version["srcname"]))
-            os.path.join(repodir, version["srcname"]),
-            version["srcnameSha256"],
-        )
 
     if "obbMainFile" in version:
         ver["obbMainFile"] = common.file_entry(
@@ -694,13 +674,9 @@ def v2_repo(repodict, repodir, archive):
 
     config = common.load_localized_config(CONFIG_CONFIG_NAME, repodir)
     if config:
-        localized_config = config["archive" if archive else "repo"]
+        repo["name"] = config["archive" if archive else "repo"]["name"]
-        if "name" in localized_config:
+        repo["description"] = config["archive" if archive else "repo"]["description"]
-            repo["name"] = localized_config["name"]
+        repo["icon"] = config["archive" if archive else "repo"]["icon"]
-        if "description" in localized_config:
-            repo["description"] = localized_config["description"]
-        if "icon" in localized_config:
-            repo["icon"] = localized_config["icon"]
 
     repo["address"] = repodict["address"]
     if "mirrors" in repodict:
@@ -725,7 +701,7 @@ def v2_repo(repodict, repodir, archive):
     return repo
 
 
-def make_v2(apps, packages, repodir, repodict, requestsdict, signer_fingerprints, archive):
+def make_v2(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints, archive):
 
     def _index_encoder_default(obj):
         if isinstance(obj, set):
@@ -747,7 +723,7 @@ def make_v2(apps, packages, repodir, repodict, requestsdict, signer_fingerprints
         output["repo"]["requests"] = requestsdict
 
     # establish sort order of the index
-    sort_package_versions(packages, signer_fingerprints)
+    v1_sort_packages(packages, fdroid_signing_key_fingerprints)
 
     output_packages = collections.OrderedDict()
     output['packages'] = output_packages
@@ -786,9 +762,7 @@ def make_v2(apps, packages, repodir, repodict, requestsdict, signer_fingerprints
     # include definitions for "auto-defined" categories, e.g. just used in app metadata
     for category in sorted(categories_used_by_apps):
         if category not in output['repo'][CATEGORIES_CONFIG_NAME]:
-            output['repo'][CATEGORIES_CONFIG_NAME][category] = dict()
+            output['repo'][CATEGORIES_CONFIG_NAME][category] = {"name": {DEFAULT_LOCALE: category}}
-        if 'name' not in output['repo'][CATEGORIES_CONFIG_NAME][category]:
-            output['repo'][CATEGORIES_CONFIG_NAME][category]['name'] = {DEFAULT_LOCALE: category}
     # do not include defined categories if no apps use them
     for category in list(output['repo'].get(CATEGORIES_CONFIG_NAME, list())):
         if category not in categories_used_by_apps:
@@ -864,7 +838,7 @@ def make_v2(apps, packages, repodir, repodict, requestsdict, signer_fingerprints
         signindex.sign_index(repodir, json_name)
 
 
-def make_v1(apps, packages, repodir, repodict, requestsdict, signer_fingerprints):
+def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
 
     def _index_encoder_default(obj):
         if isinstance(obj, set):
@@ -894,7 +868,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, signer_fingerprints
         output['repo']['mirrors'] = mirrors
 
     # establish sort order of the index
-    sort_package_versions(packages, signer_fingerprints)
+    v1_sort_packages(packages, fdroid_signing_key_fingerprints)
 
     appslist = []
     output['apps'] = appslist
@@ -967,7 +941,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, signer_fingerprints
         for k, v in sorted(package.items()):
             if not v:
                 continue
-            if k in ('icon', 'icons', 'icons_src', 'ipfsCIDv1', 'name', 'srcnameSha256'):
+            if k in ('icon', 'icons', 'icons_src', 'ipfsCIDv1', 'name'):
                 continue
             if k == 'antiFeatures':
                 d[k] = sorted(v.keys())
@@ -1002,8 +976,8 @@ def _copy_to_local_copy_dir(repodir, f):
                               .format(path=local_copy_dir))
 
 
-def sort_package_versions(packages, signer_fingerprints):
+def v1_sort_packages(packages, fdroid_signing_key_fingerprints):
-    """Sort to ensure a deterministic order for package versions in the index file.
+    """Sort the supplied list to ensure a deterministic sort order for package entries in the index file.
 
     This sort-order also expresses
     installation preference to the clients.
@@ -1028,7 +1002,7 @@ def sort_package_versions(packages, signer_fingerprints):
         if dev_signer and dev_signer == signer:
             group = GROUP_DEV_SIGNED
         else:
-            fdroid_signer = signer_fingerprints.get(packageName, {}).get('signer')
+            fdroid_signer = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
             if fdroid_signer and fdroid_signer == signer:
                 group = GROUP_FDROID_SIGNED
 
@@ -1041,7 +1015,7 @@ def sort_package_versions(packages, signer_fingerprints):
     packages.sort(key=v1_sort_keys)
 
 
-def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints):
+def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
     """Aka index.jar aka index.xml."""
     doc = Document()
 
@@ -1140,7 +1114,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints):
                 if name_from_apk is None:
                     name_from_apk = apk.get('name')
         for versionCode, apksforver in apksbyversion.items():
-            fdroid_signer = signer_fingerprints.get(appid, {}).get('signer')
+            fdroid_signer = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
             fdroid_signed_apk = None
             name_match_apk = None
             for x in apksforver:
@@ -1195,6 +1169,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints):
         addElementNonEmpty('donate', app.Donate, doc, apel)
         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
+        addElementNonEmpty('flattr', app.FlattrID, doc, apel)
         addElementNonEmpty('openCollective', app.OpenCollective, doc, apel)
 
         # These elements actually refer to the current version (i.e. which
@@ -1337,29 +1312,6 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints):
                         os.remove(siglinkname)
                     os.symlink(sigfile_path, siglinkname)
 
-    if sys.version_info.minor >= 13:
-        # Python 3.13 changed minidom so it no longer converts " to an XML entity.
-        # https://github.com/python/cpython/commit/154477be722ae5c4e18d22d0860e284006b09c4f
-        # This just puts back the previous implementation, with black code format.
-        import inspect
-        import xml.dom.minidom
-
-        def _write_data(writer, text, attr):  # pylint: disable=unused-argument
-            if text:
-                text = (
-                    text.replace('&', '&amp;')
-                    .replace('<', '&lt;')
-                    .replace('"', '&quot;')
-                    .replace('>', '&gt;')
-                )
-            writer.write(text)
-
-        argnames = tuple(inspect.signature(xml.dom.minidom._write_data).parameters)
-        if argnames == ('writer', 'text', 'attr'):
-            xml.dom.minidom._write_data = _write_data
-        else:
-            logging.warning('Failed to monkey patch minidom for index.xml support!')
-
     if common.options.pretty:
         output = doc.toprettyxml(encoding='utf-8')
     else:
@@ -1405,15 +1357,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints):
                         % repo_icon)
         os.makedirs(os.path.dirname(iconfilename), exist_ok=True)
         try:
-            import qrcode
-
             qrcode.make(common.config['repo_url']).save(iconfilename)
-        except ModuleNotFoundError as e:
-            raise ModuleNotFoundError(
-                _(
-                    'The "qrcode" Python package is not installed (e.g. apt-get install python3-qrcode)!'
-                )
-            ) from e
         except Exception:
             exampleicon = os.path.join(common.get_examples_dir(),
                                        common.default_config['repo_icon'])
@@ -1480,7 +1424,7 @@ def add_mirrors_to_repodict(repo_section, repodict):
                 )
             )
         with mirrors_yml.open() as fp:
-            mirrors_config = yaml.load(fp)
+            mirrors_config = ruamel.yaml.YAML(typ='safe').load(fp)
         if not isinstance(mirrors_config, list):
             msg = _('{path} is not list, but a {datatype}!')
             raise TypeError(
@@ -1534,7 +1478,6 @@ def add_mirrors_to_repodict(repo_section, repodict):
     repodict['mirrors'] = []
     canonical_url = repodict['address']
     found_primary = False
-    errors = 0
     for mirror in mirrors:
         if canonical_url == mirror['url']:
             found_primary = True
@@ -1543,19 +1486,9 @@ def add_mirrors_to_repodict(repo_section, repodict):
             for k in sorted(mirror.keys()):
                 sortedmirror[k] = mirror[k]
             repodict['mirrors'].insert(0, sortedmirror)
-        elif mirror.get('isPrimary'):
-            errors += 1
-            logging.error(
-                _('Mirror config for {url} contains "isPrimary" key!').format(
-                    url=mirror['url']
-                )
-            )
         else:
             repodict['mirrors'].append(mirror)
 
-    if errors:
-        raise FDroidException(_('"isPrimary" key should not be added to mirrors!'))
-
     if repodict['mirrors'] and not found_primary:
         repodict['mirrors'].insert(0, {'isPrimary': True, 'url': repodict['address']})
 
@@ -1670,8 +1603,6 @@ def download_repo_index_v1(url_str, etag=None, verify_fingerprint=True, timeout=
         - The new eTag as returned by the HTTP request
 
     """
-    from . import net
-
     url = urllib.parse.urlsplit(url_str)
 
     fingerprint = None
@@ -1704,7 +1635,7 @@ def download_repo_index_v1(url_str, etag=None, verify_fingerprint=True, timeout=
         return index, new_etag
 
 
-def download_repo_index_v2(url_str, etag=None, verify_fingerprint=True, timeout=None):
+def download_repo_index_v2(url_str, etag=None, verify_fingerprint=True, timeout=600):
     """Download and verifies index v2 file, then returns its data.
 
     Downloads the repository index from the given :param url_str and
@@ -1723,15 +1654,8 @@ def download_repo_index_v2(url_str, etag=None, verify_fingerprint=True, timeout=
         - The new eTag as returned by the HTTP request
 
     """
-    from . import net
-
-    etag  # etag is unused but needs to be there to keep the same API as the earlier functions.
-
     url = urllib.parse.urlsplit(url_str)
 
-    if timeout is not None:
-        logging.warning('"timeout" argument of download_repo_index_v2() is deprecated!')
-
     fingerprint = None
     if verify_fingerprint:
         query = urllib.parse.parse_qs(url.query)
@@ -1743,22 +1667,29 @@ def download_repo_index_v2(url_str, etag=None, verify_fingerprint=True, timeout=
         path = url.path.rsplit('/', 1)[0]
     else:
         path = url.path.rstrip('/')
-    url = urllib.parse.SplitResult(url.scheme, url.netloc, path, '', '')
 
-    mirrors = common.get_mirrors(url, 'entry.jar')
+    url = urllib.parse.SplitResult(url.scheme, url.netloc, path + '/entry.jar', '', '')
-    f = net.download_using_mirrors(mirrors)
+    download, new_etag = net.http_get(url.geturl(), etag, timeout)
-    entry, public_key, fingerprint = get_index_from_jar(f, fingerprint)
 
+    if download is None:
+        return None, new_etag
+
+    # jarsigner is used to verify the JAR, it requires a file for input
+    with tempfile.TemporaryDirectory() as dirname:
+        with (Path(dirname) / 'entry.jar').open('wb') as fp:
+            fp.write(download)
+            fp.flush()
+            entry, public_key, fingerprint = get_index_from_jar(fp.name, fingerprint)
+
+    name = entry['index']['name']
     sha256 = entry['index']['sha256']
-    mirrors = common.get_mirrors(url, entry['index']['name'][1:])
+    url = urllib.parse.SplitResult(url.scheme, url.netloc, path + name, '', '')
-    f = net.download_using_mirrors(mirrors)
+    index, _ignored = net.http_get(url.geturl(), None, timeout)
-    with open(f, 'rb') as fp:
-        index = fp.read()
     if sha256 != hashlib.sha256(index).hexdigest():
         raise VerificationException(
             _("SHA-256 of {url} does not match entry!").format(url=url)
         )
-    return json.loads(index), None
+    return json.loads(index), new_etag
 
 
 def get_index_from_jar(jarfile, fingerprint=None, allow_deprecated=False):

fdroidserver/init.py

@@ -19,15 +19,16 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import glob
-import logging
 import os
 import re
 import shutil
 import socket
 import sys
 from argparse import ArgumentParser
+import logging
 
-from . import _, common
+from . import _
+from . import common
 from .exception import FDroidException
 
 config = {}
@@ -37,13 +38,13 @@ def disable_in_config(key, value):
     """Write a key/value to the local config.yml, then comment it out."""
     import yaml
 
-    with open(common.CONFIG_FILE) as fp:
+    with open('config.yml') as f:
-        data = fp.read()
+        data = f.read()
     pattern = r'\n[\s#]*' + key + r':.*'
     repl = '\n#' + yaml.dump({key: value}, default_flow_style=False)
     data = re.sub(pattern, repl, data)
-    with open(common.CONFIG_FILE, 'w') as fp:
+    with open('config.yml', 'w') as f:
-        fp.writelines(data)
+        f.writelines(data)
 
 
 def main():
@@ -81,7 +82,7 @@ def main():
     )
     options = common.parse_args(parser)
 
-    common.set_console_logging(options.verbose, options.color)
+    common.set_console_logging(options.verbose)
 
     fdroiddir = os.getcwd()
     test_config = dict()
@@ -137,24 +138,24 @@ def main():
             _("Android SDK not found at {path}!").format(path=test_config['sdk_path'])
         )
 
-    if not os.path.exists(common.CONFIG_FILE):
+    if not os.path.exists('config.yml') and not os.path.exists('config.py'):
         # 'metadata' and 'tmp' are created in fdroid
         if not os.path.exists('repo'):
             os.mkdir('repo')
-        example_config_yml = os.path.join(examplesdir, common.CONFIG_FILE)
+        example_config_yml = os.path.join(examplesdir, 'config.yml')
         if os.path.exists(example_config_yml):
-            shutil.copyfile(example_config_yml, common.CONFIG_FILE)
+            shutil.copyfile(example_config_yml, 'config.yml')
         else:
             from pkg_resources import get_distribution
 
             versionstr = get_distribution('fdroidserver').version
             if not versionstr:
                 versionstr = 'master'
-            with open(common.CONFIG_FILE, 'w') as fp:
+            with open('config.yml', 'w') as fp:
                 fp.write('# see https://gitlab.com/fdroid/fdroidserver/blob/')
                 fp.write(versionstr)
-                fp.write(f'/examples/{common.CONFIG_FILE}\n')
+                fp.write('/examples/config.yml\n')
-        os.chmod(common.CONFIG_FILE, 0o0600)
+        os.chmod('config.yml', 0o0600)
         # If android_home is None, test_config['sdk_path'] will be used and
         # "$ANDROID_HOME" may be used if the env var is set up correctly.
         # If android_home is not None, the path given from the command line

fdroidserver/install.py

@@ -17,298 +17,35 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import glob
-import locale
-import logging
-import os
 import sys
-import termios
+import os
-import tty
+import glob
-from argparse import ArgumentParser, BooleanOptionalAction
+from argparse import ArgumentParser
-from pathlib import Path
+import logging
-from urllib.parse import urlencode, urlparse, urlunparse
 
-import defusedxml.ElementTree as XMLElementTree
+from . import _
-
+from . import common
-from . import _, common, github, index, net
+from .common import SdkToolsPopen
 from .exception import FDroidException
 
-DEFAULT_IPFS_GATEWAYS = ("https://gateway.ipfs.io/ipfs/",)
+config = None
-MAVEN_CENTRAL_MIRRORS = [
-    {
-        "url": "https://repo1.maven.org/maven2/",
-        "dnsA": ["199.232.16.209"],
-        "worksWithoutSNI": True,
-    },
-    {
-        "url": "https://repo.maven.apache.org/maven2/",
-        "dnsA": ["199.232.16.215"],
-        "worksWithoutSNI": True,
-    },
-    {
-        "url": "https://maven-central-asia.storage-download.googleapis.com/maven2/",
-    },
-    {
-        "url": "https://maven-central-eu.storage-download.googleapis.com/maven2/",
-    },
-    {
-        "url": "https://maven-central.storage-download.googleapis.com/maven2/",
-    },
-]
-
-
-# pylint: disable=unused-argument
-def download_apk(appid='org.fdroid.fdroid', privacy_mode=False):
-    """Download an APK from F-Droid via the first mirror that works."""
-    url = urlunparse(
-        urlparse(common.FDROIDORG_MIRRORS[0]['url'])._replace(
-            query=urlencode({'fingerprint': common.FDROIDORG_FINGERPRINT})
-        )
-    )
-
-    data, _ignored = index.download_repo_index_v2(url)
-    app = data.get('packages', dict()).get(appid)
-    preferred_version = None
-    for version in app['versions'].values():
-        if not preferred_version:
-            # if all else fails, use the first one
-            preferred_version = version
-        if not version.get('releaseChannels'):
-            # prefer APK in default release channel
-            preferred_version = version
-            break
-
-    mirrors = common.append_filename_to_mirrors(
-        preferred_version['file']['name'][1:], common.FDROIDORG_MIRRORS
-    )
-    ipfsCIDv1 = preferred_version['file'].get('ipfsCIDv1')
-    if ipfsCIDv1:
-        for gateway in DEFAULT_IPFS_GATEWAYS:
-            mirrors.append({'url': os.path.join(gateway, ipfsCIDv1)})
-    f = net.download_using_mirrors(mirrors)
-    if f and os.path.exists(f):
-        versionCode = preferred_version['manifest']['versionCode']
-        f = Path(f)
-        return str(f.rename(f.with_stem(f'{appid}_{versionCode}')).resolve())
-
-
-def download_fdroid_apk(privacy_mode=False):  # pylint: disable=unused-argument
-    """Directly download the current F-Droid APK and verify it.
-
-    This downloads the "download button" link, which is the version
-    that is best tested for new installs.
-
-    """
-    mirror = common.FDROIDORG_MIRRORS[0]
-    mirror['url'] = urlunparse(urlparse(mirror['url'])._replace(path='F-Droid.apk'))
-    return net.download_using_mirrors([mirror])
-
-
-def download_fdroid_apk_from_github(privacy_mode=False):
-    """Download F-Droid.apk from F-Droid's GitHub Releases."""
-    if common.config and not privacy_mode:
-        token = common.config.get('github_token')
-    else:
-        token = None
-    gh = github.GithubApi(token, 'https://github.com/f-droid/fdroidclient')
-    latest_apk = gh.get_latest_apk()
-    filename = os.path.basename(latest_apk)
-    return net.download_file(latest_apk, os.path.join(common.get_cachedir(), filename))
-
-
-def download_fdroid_apk_from_ipns(privacy_mode=False):
-    """Download the F-Droid APK from an IPNS repo."""
-    cid = 'k51qzi5uqu5dl4hbcksbdmplanu9n4hivnqsupqe6vzve1pdbeh418ssptldd3'
-    mirrors = [
-        {"url": f"https://ipfs.io/ipns/{cid}/F-Droid.apk"},
-    ]
-    if not privacy_mode:
-        mirrors.append({"url": f"https://{cid}.ipns.dweb.link/F-Droid.apk"})
-    return net.download_using_mirrors(mirrors)
-
-
-def download_fdroid_apk_from_maven(privacy_mode=False):
-    """Download F-Droid.apk from Maven Central and official mirrors."""
-    path = 'org/fdroid/fdroid/F-Droid'
-    if privacy_mode:
-        mirrors = MAVEN_CENTRAL_MIRRORS[:2]  # skip the Google servers
-    else:
-        mirrors = MAVEN_CENTRAL_MIRRORS
-    metadata = net.download_using_mirrors(
-        common.append_filename_to_mirrors(
-            os.path.join(path, 'maven-metadata.xml'), mirrors
-        )
-    )
-    version = XMLElementTree.parse(metadata).getroot().findall('*.//latest')[0].text
-    mirrors = common.append_filename_to_mirrors(
-        os.path.join(path, version, f'F-Droid-{version}.apk'), mirrors
-    )
-    return net.download_using_mirrors(mirrors)
-
-
-def install_fdroid_apk(privacy_mode=False):
-    """Download and install F-Droid.apk using all tricks we can muster.
-
-    By default, this first tries to fetch the official install APK
-    which is offered when someone clicks the "download" button on
-    https://f-droid.org/.  Then it will try all the mirrors and
-    methods until it gets something successful, or runs out of
-    options.
-
-    There is privacy_mode which tries to download from mirrors first,
-    so that this downloads from a mirror that has many different kinds
-    of files available, thereby breaking the clear link to F-Droid.
-
-    Returns
-    -------
-    None for success or the error message.
-
-    """
-    country_code = locale.getlocale()[0].split('_')[-1]
-    if privacy_mode is None and country_code in ('CN', 'HK', 'IR', 'TM'):
-        logging.warning(
-            _('Privacy mode was enabled based on your locale ({country_code}).').format(
-                country_code=country_code
-            )
-        )
-        privacy_mode = True
-
-    if privacy_mode or not (common.config and common.config.get('jarsigner')):
-        download_methods = [
-            download_fdroid_apk_from_maven,
-            download_fdroid_apk_from_ipns,
-            download_fdroid_apk_from_github,
-        ]
-    else:
-        download_methods = [
-            download_apk,
-            download_fdroid_apk_from_maven,
-            download_fdroid_apk_from_github,
-            download_fdroid_apk_from_ipns,
-            download_fdroid_apk,
-        ]
-    for method in download_methods:
-        try:
-            f = method(privacy_mode=privacy_mode)
-            break
-        except Exception as e:
-            logging.info(e)
-    else:
-        return _('F-Droid.apk could not be downloaded from any known source!')
-
-    fingerprint = common.apk_signer_fingerprint(f)
-    if fingerprint.upper() != common.FDROIDORG_FINGERPRINT:
-        return _('{path} has the wrong fingerprint ({fingerprint})!').format(
-            path=f, fingerprint=fingerprint
-        )
-    install_apk(f)
-
-
-def install_apk(f):
-    if common.config and common.config.get('apksigner'):
-        # TODO this should always verify, but that requires APK sig verification in Python #94
-        logging.info(_('Verifying package {path} with apksigner.').format(path=f))
-        common.verify_apk_signature(f)
-    if common.config and common.config.get('adb'):
-        if devices():
-            install_apks_to_devices([f])
-            os.remove(f)
-        else:
-            os.remove(f)
-            return _('No devices found for `adb install`! Please plug one in.')
 
 
 def devices():
-    """Get the list of device serials for use with adb commands."""
+    p = SdkToolsPopen(['adb', "devices"])
-    p = common.SdkToolsPopen(['adb', "devices"])
     if p.returncode != 0:
         raise FDroidException("An error occured when finding devices: %s" % p.output)
-    serials = list()
+    lines = [line for line in p.output.splitlines() if not line.startswith('* ')]
-    for line in p.output.splitlines():
+    if len(lines) < 3:
-        columns = line.strip().split("\t", maxsplit=1)
+        return []
-        if len(columns) == 2:
+    lines = lines[1:-1]
-            serial, status = columns
+    return [line.split()[0] for line in lines]
-            if status == 'device':
-                serials.append(serial)
-            else:
-                d = {'serial': serial, 'status': status}
-                logging.warning(_('adb reports {serial} is "{status}"!'.format(**d)))
-    return serials
-
-
-def install_apks_to_devices(apks):
-    """Install the list of APKs to all Android devices reported by `adb devices`."""
-    for apk in apks:
-        # Get device list each time to avoid device not found errors
-        devs = devices()
-        if not devs:
-            raise FDroidException(_("No attached devices found"))
-        logging.info(_("Installing %s...") % apk)
-        for dev in devs:
-            logging.info(
-                _("Installing '{apkfilename}' on {dev}...").format(
-                    apkfilename=apk, dev=dev
-                )
-            )
-            p = common.SdkToolsPopen(['adb', "-s", dev, "install", apk])
-            fail = ""
-            for line in p.output.splitlines():
-                if line.startswith("Failure"):
-                    fail = line[9:-1]
-            if not fail:
-                continue
-
-            if fail == "INSTALL_FAILED_ALREADY_EXISTS":
-                logging.warning(
-                    _('"{apkfilename}" is already installed on {dev}.').format(
-                        apkfilename=apk, dev=dev
-                    )
-                )
-            else:
-                raise FDroidException(
-                    _("Failed to install '{apkfilename}' on {dev}: {error}").format(
-                        apkfilename=apk, dev=dev, error=fail
-                    )
-                )
-
-
-def read_char():
-    """Read input from the terminal prompt one char at a time."""
-    fd = sys.stdin.fileno()
-    old_settings = termios.tcgetattr(fd)
-    try:
-        tty.setraw(fd)
-        ch = sys.stdin.read(1)
-    finally:
-        termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
-    return ch
-
-
-def strtobool(val):
-    """Convert a localized string representation of truth to True or False."""
-    return val.lower() in ('', 'y', 'yes', _('yes'), _('true'))  # '' is pressing Enter
-
-
-def prompt_user(yes, msg):
-    """Prompt user for yes/no, supporting Enter and Esc as accepted answers."""
-    run_install = yes
-    if yes is None and sys.stdout.isatty():
-        print(msg, end=' ', flush=True)
-        answer = ''
-        while True:
-            in_char = read_char()
-            if in_char == '\r':  # Enter key
-                break
-            if not in_char.isprintable():
-                sys.exit(1)
-            print(in_char, end='', flush=True)
-            answer += in_char
-        run_install = strtobool(answer)
-        print()
-    return run_install
 
 
 def main():
+    global config
+
+    # Parse command line...
     parser = ArgumentParser(
         usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]"
     )
@@ -325,56 +62,22 @@ def main():
         default=False,
         help=_("Install all signed applications available"),
     )
-    parser.add_argument(
-        "-p",
-        "--privacy-mode",
-        action=BooleanOptionalAction,
-        default=None,
-        help=_("Download F-Droid.apk using mirrors that leak less to the network"),
-    )
-    parser.add_argument(
-        "-y",
-        "--yes",
-        action="store_true",
-        default=None,
-        help=_("Automatic yes to all prompts."),
-    )
-    parser.add_argument(
-        "-n",
-        "--no",
-        action="store_false",
-        dest='yes',
-        help=_("Automatic no to all prompts."),
-    )
     options = common.parse_args(parser)
 
-    common.set_console_logging(options.verbose, options.color)
+    common.set_console_logging(options.verbose)
-    logging.captureWarnings(True)  # for SNIMissingWarning
-
-    common.get_config()
 
     if not options.appid and not options.all:
-        run_install = prompt_user(
+        parser.error(
-            options.yes,
+            _("option %s: If you really want to install all the signed apps, use --all")
-            _('Would you like to download and install F-Droid.apk via adb? (YES/no)'),
+            % "all"
         )
-        if run_install:
+
-            sys.exit(install_fdroid_apk(options.privacy_mode))
+    config = common.read_config()
-        sys.exit(1)
 
     output_dir = 'repo'
-    if (options.appid or options.all) and not os.path.isdir(output_dir):
+    if not os.path.isdir(output_dir):
-        logging.error(_("No signed output directory - nothing to do"))
+        logging.info(_("No signed output directory - nothing to do"))
-        run_install = prompt_user(
+        sys.exit(0)
-            options.yes,
-            _('Would you like to download the app(s) from f-droid.org? (YES/no)'),
-        )
-        if run_install:
-            for appid in options.appid:
-                f = download_apk(appid)
-                install_apk(f)
-            sys.exit(install_fdroid_apk(options.privacy_mode))
-        sys.exit(1)
 
     if options.appid:
         vercodes = common.read_pkg_args(options.appid, True)
@@ -396,14 +99,45 @@ def main():
         for appid, apk in apks.items():
             if not apk:
                 raise FDroidException(_("No signed APK available for %s") % appid)
-        install_apks_to_devices(apks.values())
 
-    elif options.all:
+    else:
         apks = {
             common.publishednameinfo(apkfile)[0]: apkfile
             for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk')))
         }
-        install_apks_to_devices(apks.values())
+
+    for appid, apk in apks.items():
+        # Get device list each time to avoid device not found errors
+        devs = devices()
+        if not devs:
+            raise FDroidException(_("No attached devices found"))
+        logging.info(_("Installing %s...") % apk)
+        for dev in devs:
+            logging.info(
+                _("Installing '{apkfilename}' on {dev}...").format(
+                    apkfilename=apk, dev=dev
+                )
+            )
+            p = SdkToolsPopen(['adb', "-s", dev, "install", apk])
+            fail = ""
+            for line in p.output.splitlines():
+                if line.startswith("Failure"):
+                    fail = line[9:-1]
+            if not fail:
+                continue
+
+            if fail == "INSTALL_FAILED_ALREADY_EXISTS":
+                logging.warning(
+                    _('"{apkfilename}" is already installed on {dev}.').format(
+                        apkfilename=apk, dev=dev
+                    )
+                )
+            else:
+                raise FDroidException(
+                    _("Failed to install '{apkfilename}' on {dev}: {error}").format(
+                        apkfilename=apk, dev=dev, error=fail
+                    )
+                )
 
     logging.info('\n' + _('Finished'))
 

fdroidserver/lint.py

@@ -24,7 +24,7 @@ import urllib.parse
 from argparse import ArgumentParser
 from pathlib import Path
 
-from fdroidserver._yaml import yaml
+import ruamel.yaml
 
 from . import _, common, metadata, rewritemeta
 
@@ -159,6 +159,10 @@ regex_checks = {
     ],
     'Donate': http_checks
     + [
+        (
+            re.compile(r'.*flattr\.com'),
+            _("Flattr donation methods belong in the FlattrID: field"),
+        ),
         (
             re.compile(r'.*liberapay\.com'),
             _("Liberapay donation methods belong in the Liberapay: field"),
@@ -213,82 +217,6 @@ regex_checks = {
     ],
 }
 
-# config keys that are currently ignored by lint, but could be supported.
-ignore_config_keys = (
-    'github_releases',
-    'java_paths',
-)
-
-bool_keys = (
-    'allow_disabled_algorithms',
-    'androidobservatory',
-    'build_server_always',
-    'deploy_process_logs',
-    'keep_when_not_allowed',
-    'make_current_version_link',
-    'nonstandardwebroot',
-    'per_app_repos',
-    'refresh_scanner',
-    'scan_binary',
-    'sync_from_local_copy_dir',
-)
-
-check_config_keys = (
-    'ant',
-    'apk_signing_key_block_list',
-    'archive',
-    'archive_description',
-    'archive_icon',
-    'archive_name',
-    'archive_older',
-    'archive_url',
-    'archive_web_base_url',
-    'awsbucket',
-    'awsbucket_index_only',
-    'binary_transparency_remote',
-    'cachedir',
-    'char_limits',
-    'current_version_name_source',
-    'git_mirror_size_limit',
-    'github_token',
-    'gpghome',
-    'gpgkey',
-    'gradle',
-    'identity_file',
-    'install_list',
-    'java_paths',
-    'keyaliases',
-    'keydname',
-    'keypass',
-    'keystore',
-    'keystorepass',
-    'lint_licenses',
-    'local_copy_dir',
-    'mirrors',
-    'mvn3',
-    'ndk_paths',
-    'path_to_custom_rclone_config',
-    'rclone_config',
-    'repo',
-    'repo_description',
-    'repo_icon',
-    'repo_key_sha256',
-    'repo_keyalias',
-    'repo_maxage',
-    'repo_name',
-    'repo_pubkey',
-    'repo_url',
-    'repo_web_base_url',
-    'scanner_signature_sources',
-    'sdk_path',
-    'servergitmirrors',
-    'serverwebroot',
-    'smartcardoptions',
-    'sync_from_local_copy_dir',
-    'uninstall_list',
-    'virustotal_apikey',
-)
-
 locale_pattern = re.compile(r"[a-z]{2,3}(-([A-Z][a-zA-Z]+|\d+|[a-z]+))*")
 
 versioncode_check_pattern = re.compile(r"(\\d|\[(0-9|\\d)_?(a-fA-F)?])[+]")
@@ -369,7 +297,7 @@ def check_update_check_data_int(app):  # noqa: D403
         # codeex can be empty as well
         if codeex and not versioncode_check_pattern.search(codeex):
             yield _(
-                f'UpdateCheckData must match the versionCode as integer (\\d or [0-9]): {codeex}'
+                f'UpdateCheckData must match the version code as integer (\\d or [0-9]): {codeex}'
             )
 
 
@@ -577,20 +505,11 @@ def check_format(app):
 
 
 def check_license_tag(app):
-    """Ensure all license tags contain only valid/approved values.
+    """Ensure all license tags contain only valid/approved values."""
-
+    if config['lint_licenses'] is None:
-    It is possible to disable license checking by setting a null or empty value,
-    e.g. `lint_licenses: ` or `lint_licenses: []`
-
-    """
-    if 'lint_licenses' in config:
-        lint_licenses = config['lint_licenses']
-        if lint_licenses is None:
         return
-    else:
+    if app.License not in config['lint_licenses']:
-        lint_licenses = APPROVED_LICENSES
+        if config['lint_licenses'] == APPROVED_LICENSES:
-    if app.License not in lint_licenses:
-        if lint_licenses == APPROVED_LICENSES:
             yield _(
                 'Unexpected license tag "{}"! Only use FSF or OSI '
                 'approved tags from https://spdx.org/license-list'
@@ -611,20 +530,10 @@ def check_extlib_dir(apps):
 
     used = set()
     for app in apps:
-        if app.Disabled:
+        for build in app.get('Builds', []):
-            continue
-        archive_policy = common.calculate_archive_policy(
-            app, common.config['archive_older']
-        )
-        builds = [build for build in app.Builds if not build.disable]
-
-        for i in range(len(builds)):
-            build = builds[i]
             for path in build.extlibs:
                 path = Path(path)
                 if path not in extlib_files:
-                    # Don't show error on archived versions
-                    if i >= len(builds) - archive_policy:
                     yield _(
                         "{appid}: Unknown extlib {path} in build '{versionName}'"
                     ).format(appid=app.id, path=path, versionName=build.versionName)
@@ -852,7 +761,7 @@ def lint_config(arg):
         passed = False
 
     with path.open() as fp:
-        data = yaml.load(fp)
+        data = ruamel.yaml.YAML(typ='safe').load(fp)
     common.config_type_check(arg, data)
 
     if path.name == mirrors_name:
@@ -876,41 +785,6 @@ def lint_config(arg):
                     msg += ' '
                     msg += _('Did you mean {code}?').format(code=', '.join(sorted(m)))
                 print(msg)
-    elif path.name == config_name and path.parent.name != 'config':
-        valid_keys = set(tuple(common.default_config) + bool_keys + check_config_keys)
-        for key in ignore_config_keys:
-            if key in valid_keys:
-                valid_keys.remove(key)
-        for key in data:
-            if key not in valid_keys:
-                passed = False
-                msg = _("ERROR: {key} not a valid key!").format(key=key)
-                m = difflib.get_close_matches(key.lower(), valid_keys, 2, 0.5)
-                if m:
-                    msg += ' '
-                    msg += _('Did you mean {code}?').format(code=', '.join(sorted(m)))
-                print(msg)
-                continue
-
-            if key in bool_keys:
-                t = bool
-            else:
-                t = type(common.default_config.get(key, ""))
-
-            show_error = False
-            if t is str:
-                if type(data[key]) not in (str, list, dict):
-                    passed = False
-                    show_error = True
-            elif type(data[key]) != t:
-                passed = False
-                show_error = True
-            if show_error:
-                print(
-                    _("ERROR: {key}'s value should be of type {t}!").format(
-                        key=key, t=t.__name__
-                    )
-                )
     elif path.name in (config_name, categories_name, antifeatures_name):
         for key in data:
             if path.name == config_name and key not in ('archive', 'repo'):
@@ -984,7 +858,7 @@ def main():
     paths = list()
     for arg in options.appid:
         if (
-            arg == common.CONFIG_FILE
+            arg == 'config.yml'
             or Path(arg).parent.name == 'config'
             or Path(arg).parent.parent.name == 'config'  # localized
         ):
@@ -1008,7 +882,9 @@ def main():
 
 
 def lint_metadata(options):
-    apps = common.read_app_args(options.appid)
+    # Get all apps...
+    allapps = metadata.read_metadata(options.appid)
+    apps = common.read_app_args(options.appid, allapps, False)
 
     anywarns = check_for_unsupported_metadata_files()
 

fdroidserver/looseversion.py

@@ -115,7 +115,7 @@ __license__ = "Python License 2.0"
 # been done in the StrictVersion class above.  This works great as long
 # as everyone can go along with bondage and discipline.  Hopefully a
 # (large) subset of Python module programmers will agree that the
-# particular flavor of bondage and discipline provided by StrictVersion
+# particular flavour of bondage and discipline provided by StrictVersion
 # provides enough benefit to be worth using, and will submit their
 # version numbering scheme to its domination.  The free-thinking
 # anarchists in the lot will never give in, though, and something needs

fdroidserver/metadata.py

@@ -18,18 +18,18 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import logging
+import git
-import math
-import os
-import platform
-import re
-from collections import OrderedDict
 from pathlib import Path
-
+import math
+import platform
+import os
+import re
+import logging
 import ruamel.yaml
+from collections import OrderedDict
 
-from . import _, common
+from . import common
-from ._yaml import yaml
+from . import _
 from .exception import MetaDataException
 
 srclibs = None
@@ -67,6 +67,7 @@ yaml_app_field_order = [
     'Translation',
     'Changelog',
     'Donate',
+    'FlattrID',
     'Liberapay',
     'OpenCollective',
     'Bitcoin',
@@ -127,6 +128,7 @@ class App(dict):
         self.Translation = ''
         self.Changelog = ''
         self.Donate = None
+        self.FlattrID = None
         self.Liberapay = None
         self.OpenCollective = None
         self.Bitcoin = None
@@ -406,6 +408,10 @@ class FieldValidator:
 
 # Generic value types
 valuetypes = {
+    FieldValidator("Flattr ID",
+                   r'^[0-9a-z]+$',
+                   ['FlattrID']),
+
     FieldValidator("Liberapay",
                    VALID_USERNAME_REGEX,
                    ['Liberapay']),
@@ -472,6 +478,7 @@ def parse_yaml_srclib(metadatapath):
 
     with metadatapath.open("r", encoding="utf-8") as f:
         try:
+            yaml = ruamel.yaml.YAML(typ='safe')
             data = yaml.load(f)
             if type(data) is not dict:
                 if platform.system() == 'Windows':
@@ -537,14 +544,14 @@ def read_srclibs():
 
     srclibs = {}
 
-    srclibs_dir = Path('srclibs')
+    srcdir = Path('srclibs')
-    srclibs_dir.mkdir(exist_ok=True)
+    srcdir.mkdir(exist_ok=True)
 
-    for metadatapath in sorted(srclibs_dir.glob('*.yml')):
+    for metadatapath in sorted(srcdir.glob('*.yml')):
         srclibs[metadatapath.stem] = parse_yaml_srclib(metadatapath)
 
 
-def read_metadata(appid_to_vercode={}, sort_by_time=False):
+def read_metadata(appids={}, sort_by_time=False):
     """Return a list of App instances sorted newest first.
 
     This reads all of the metadata files in a 'data' repository, then
@@ -552,7 +559,7 @@ def read_metadata(appid_to_vercode={}, sort_by_time=False):
     sorted based on creation time, newest first.  Most of the time,
     the newer files are the most interesting.
 
-    appid_to_vercode is a dict with appids a keys and versionCodes as values.
+    appids is a dict with appids a keys and versionCodes as values.
 
     """
     # Always read the srclibs before the apps, since they can use a srlib as
@@ -564,8 +571,9 @@ def read_metadata(appid_to_vercode={}, sort_by_time=False):
     for basedir in ('metadata', 'tmp'):
         Path(basedir).mkdir(exist_ok=True)
 
-    if appid_to_vercode:
+    if appids:
-        metadatafiles = common.get_metadata_files(appid_to_vercode)
+        vercodes = common.read_pkg_args(appids)
+        metadatafiles = common.get_metadata_files(vercodes)
     else:
         metadatafiles = list(Path('metadata').glob('*.yml')) + list(
             Path('.').glob('.fdroid.yml')
@@ -657,12 +665,14 @@ def parse_metadata(metadatapath):
         build_dir = common.get_build_dir(app)
         metadata_in_repo = build_dir / '.fdroid.yml'
         if metadata_in_repo.is_file():
-            commit_id = common.get_head_commit_id(build_dir)
+            try:
-            if commit_id is not None:
+                commit_id = common.get_head_commit_id(git.Repo(build_dir))
                 logging.debug(
                     _('Including metadata from %s@%s') % (metadata_in_repo, commit_id)
                 )
-            else:
+            # See https://github.com/PyCQA/pylint/issues/2856 .
+            # pylint: disable-next=no-member
+            except git.exc.InvalidGitRepositoryError:
                 logging.debug(
                     _('Including metadata from {path}').format(path=metadata_in_repo)
                 )
@@ -706,7 +716,8 @@ def parse_yaml_metadata(mf):
 
     """
     try:
-        yamldata = common.yaml.load(mf)
+        yaml = ruamel.yaml.YAML(typ='safe')
+        yamldata = yaml.load(mf)
     except ruamel.yaml.YAMLError as e:
         _warn_or_exception(
             _("could not parse '{path}'").format(path=mf.name)
@@ -1245,24 +1256,19 @@ def _app_to_yaml(app):
 def write_yaml(mf, app):
     """Write metadata in yaml format.
 
-    This requires the 'rt' round trip dumper to maintain order and needs
-    custom indent settings, so it needs to instantiate its own YAML
-    instance.  Therefore, this function deliberately avoids using any of
-    the common YAML parser setups.
-
     Parameters
     ----------
     mf
       active file discriptor for writing
     app
-      app metadata to written to the YAML file
+      app metadata to written to the yaml file
 
     """
     _del_duplicated_NoSourceSince(app)
     yaml_app = _app_to_yaml(app)
-    yamlmf = ruamel.yaml.YAML(typ='rt')
+    yaml = ruamel.yaml.YAML()
-    yamlmf.indent(mapping=2, sequence=4, offset=2)
+    yaml.indent(mapping=2, sequence=4, offset=2)
-    yamlmf.dump(yaml_app, stream=mf)
+    yaml.dump(yaml_app, stream=mf)
 
 
 def write_metadata(metadatapath, app):

fdroidserver/mirror.py

@@ -7,10 +7,13 @@ import posixpath
 import socket
 import subprocess
 import sys
-import urllib.parse
 from argparse import ArgumentParser
+import urllib.parse
 
-from . import _, common, index, update
+from . import _
+from . import common
+from . import index
+from . import update
 
 
 def _run_wget(path, urls, verbose=False):
@@ -88,7 +91,7 @@ def main():
     )
     options = common.parse_args(parser)
 
-    common.set_console_logging(options.verbose, options.color)
+    common.set_console_logging(options.verbose)
 
     if options.all:
         options.archive = True
@@ -130,7 +133,6 @@ def main():
             import io
             import json
             import zipfile
-
             from . import net
 
             url = _append_to_url_path(section, 'index-v1.jar')

fdroidserver/net.py

@@ -17,21 +17,13 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import copy
 import logging
 import os
-import random
+import requests
-import tempfile
 import time
 import urllib
-
-import requests
-import urllib3
 from requests.adapters import HTTPAdapter, Retry
-
+from requests.exceptions import ChunkedEncodingError
-from . import _, common
-
-logger = logging.getLogger(__name__)
 
 HEADERS = {'User-Agent': 'F-Droid'}
 
@@ -72,88 +64,14 @@ def download_file(url, local_filename=None, dldir='tmp', retries=3, backoff_fact
                         f.write(chunk)
                         f.flush()
             return local_filename
-        except requests.exceptions.ChunkedEncodingError as err:
+        except ChunkedEncodingError as err:
             if i == retries:
                 raise err
-            logger.warning('Download interrupted, retrying...')
+            logging.warning('Download interrupted, retrying...')
             time.sleep(backoff_factor * 2**i)
     raise ValueError("retries must be >= 0")
 
 
-def download_using_mirrors(mirrors, local_filename=None):
-    """Try to download the file from any working mirror.
-
-    Download the file that all URLs in the mirrors list point to,
-    trying all the tricks, starting with the most private methods
-    first.  The list of mirrors is converted into a list of mirror
-    configurations to try, in order that the should be attempted.
-
-    This builds mirror_configs_to_try using all possible combos to
-    try.  If a mirror is marked with worksWithoutSNI: True, then this
-    logic will try it twice: first without SNI, then again with SNI.
-
-    """
-    mirrors = common.parse_list_of_dicts(mirrors)
-    mirror_configs_to_try = []
-    for mirror in mirrors:
-        mirror_configs_to_try.append(mirror)
-        if mirror.get('worksWithoutSNI'):
-            m = copy.deepcopy(mirror)
-            del m['worksWithoutSNI']
-            mirror_configs_to_try.append(m)
-
-    if not local_filename:
-        for mirror in mirrors:
-            filename = urllib.parse.urlparse(mirror['url']).path.split('/')[-1]
-            if filename:
-                break
-        if filename:
-            local_filename = os.path.join(common.get_cachedir(), filename)
-        else:
-            local_filename = tempfile.mkstemp(prefix='fdroid-')
-
-    timeouts = (2, 10, 100)
-    last_exception = None
-    for timeout in timeouts:
-        for mirror in mirror_configs_to_try:
-            last_exception = None
-            urllib3.util.ssl_.HAS_SNI = not mirror.get('worksWithoutSNI')
-            try:
-                # the stream=True parameter keeps memory usage low
-                r = requests.get(
-                    mirror['url'],
-                    stream=True,
-                    allow_redirects=False,
-                    headers=HEADERS,
-                    # add jitter to the timeout to be less predictable
-                    timeout=timeout + random.randint(0, timeout),  # nosec B311
-                )
-                if r.status_code != 200:
-                    raise requests.exceptions.HTTPError(r.status_code, response=r)
-                with open(local_filename, 'wb') as f:
-                    for chunk in r.iter_content(chunk_size=1024):
-                        if chunk:  # filter out keep-alive new chunks
-                            f.write(chunk)
-                            f.flush()
-                return local_filename
-            except (
-                ConnectionError,
-                requests.exceptions.ChunkedEncodingError,
-                requests.exceptions.ConnectionError,
-                requests.exceptions.ContentDecodingError,
-                requests.exceptions.HTTPError,
-                requests.exceptions.SSLError,
-                requests.exceptions.StreamConsumedError,
-                requests.exceptions.Timeout,
-                requests.exceptions.UnrewindableBodyError,
-            ) as e:
-                last_exception = e
-                logger.debug(_('Retrying failed download: %s') % str(e))
-    # if it hasn't succeeded by now, then give up and raise last exception
-    if last_exception:
-        raise last_exception
-
-
 def http_get(url, etag=None, timeout=600):
     """Download the content from the given URL by making a GET request.
 

fdroidserver/nightly.py

@@ -19,25 +19,24 @@
 
 import base64
 import datetime
+import git
 import hashlib
-import inspect
 import logging
 import os
+import paramiko
 import platform
 import shutil
 import ssl
 import subprocess
 import sys
 import tempfile
+import yaml
+from urllib.parse import urlparse
 from argparse import ArgumentParser
 from typing import Optional
-from urllib.parse import urlparse
 
-import git
+from . import _
-import paramiko
+from . import common
-import yaml
-
-from . import _, common
 from .exception import VCSException
 
 # hard coded defaults for Android ~/.android/debug.keystore files
@@ -177,9 +176,7 @@ def _ssh_key_from_debug_keystore(keystore: Optional[str] = None) -> str:
     return ssh_private_key_file
 
 
-def get_repo_base_url(
+def get_repo_base_url(clone_url: str, repo_git_base: str, force_type: Optional[str] = None) -> str:
-    clone_url: str, repo_git_base: str, force_type: Optional[str] = None
-) -> str:
     """Generate the base URL for the F-Droid repository.
 
     Parameters
@@ -206,41 +203,6 @@ def get_repo_base_url(
     sys.exit(1)
 
 
-def clone_git_repo(clone_url, git_mirror_path):
-    """Clone a git repo into the given path, failing if a password is required.
-
-    If GitPython's safe mode is present, this will use that.  Otherwise,
-    this includes a very limited version of the safe mode just to ensure
-    this won't hang on password prompts.
-
-    https://github.com/gitpython-developers/GitPython/pull/2029
-
-    """
-    logging.debug(_('cloning {url}').format(url=clone_url))
-    try:
-        sig = inspect.signature(git.Repo.clone_from)
-        if 'safe' in sig.parameters:
-            git.Repo.clone_from(clone_url, git_mirror_path, safe=True)
-        else:
-            git.Repo.clone_from(
-                clone_url,
-                git_mirror_path,
-                env={
-                    'GIT_ASKPASS': '/bin/true',
-                    'SSH_ASKPASS': '/bin/true',
-                    'GIT_USERNAME': 'u',
-                    'GIT_PASSWORD': 'p',
-                    'GIT_HTTP_USERNAME': 'u',
-                    'GIT_HTTP_PASSWORD': 'p',
-                    'GIT_SSH': '/bin/false',  # for git < 2.3
-                    'GIT_TERMINAL_PROMPT': '0',
-                },
-            )
-    except git.exc.GitCommandError as e:
-        logging.warning(_('WARNING: only public git repos are supported!'))
-        raise VCSException(f'git clone {clone_url} failed:', str(e)) from e
-
-
 def main():
     """Deploy to F-Droid repository or generate SSH private key from keystore.
 
@@ -326,27 +288,19 @@ def main():
             # we are in GitLab CI
             repo_git_base = os.getenv('CI_PROJECT_PATH') + NIGHTLY
             clone_url = os.getenv('CI_PROJECT_URL') + NIGHTLY
-            repo_base = get_repo_base_url(
+            repo_base = get_repo_base_url(clone_url, repo_git_base, force_type='gitlab.com')
-                clone_url, repo_git_base, force_type='gitlab.com'
-            )
             servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base
-            deploy_key_url = (
+            deploy_key_url = clone_url + '/-/settings/repository#js-deploy-keys-settings'
-                f'{clone_url}/-/settings/repository#js-deploy-keys-settings'
-            )
             git_user_name = os.getenv('GITLAB_USER_NAME')
             git_user_email = os.getenv('GITLAB_USER_EMAIL')
         elif 'TRAVIS_REPO_SLUG' in os.environ:
             # we are in Travis CI
             repo_git_base = os.getenv('TRAVIS_REPO_SLUG') + NIGHTLY
             clone_url = 'https://github.com/' + repo_git_base
-            repo_base = get_repo_base_url(
+            repo_base = get_repo_base_url(clone_url, repo_git_base, force_type='github.com')
-                clone_url, repo_git_base, force_type='github.com'
-            )
             servergitmirror = 'git@github.com:' + repo_git_base
-            deploy_key_url = (
+            deploy_key_url = ('https://github.com/' + repo_git_base + '/settings/keys'
-                f'https://github.com/{repo_git_base}/settings/keys'
+                              + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys')
-                + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
-            )
             git_user_name = repo_git_base
             git_user_email = os.getenv('USER') + '@' + platform.node()
         elif (
@@ -355,35 +309,23 @@ def main():
             and 'CIRCLE_PROJECT_REPONAME' in os.environ
         ):
             # we are in Circle CI
-            repo_git_base = (
+            repo_git_base = (os.getenv('CIRCLE_PROJECT_USERNAME')
-                os.getenv('CIRCLE_PROJECT_USERNAME')
+                             + '/' + os.getenv('CIRCLE_PROJECT_REPONAME') + NIGHTLY)
-                + '/'
-                + os.getenv('CIRCLE_PROJECT_REPONAME')
-                + NIGHTLY
-            )
             clone_url = os.getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY
-            repo_base = get_repo_base_url(
+            repo_base = get_repo_base_url(clone_url, repo_git_base, force_type='github.com')
-                clone_url, repo_git_base, force_type='github.com'
-            )
             servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base
-            deploy_key_url = (
+            deploy_key_url = ('https://github.com/' + repo_git_base + '/settings/keys'
-                f'https://github.com/{repo_git_base}/settings/keys'
+                              + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys')
-                + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
-            )
             git_user_name = os.getenv('CIRCLE_USERNAME')
             git_user_email = git_user_name + '@' + platform.node()
         elif 'GITHUB_ACTIONS' in os.environ:
             # we are in Github actions
-            repo_git_base = os.getenv('GITHUB_REPOSITORY') + NIGHTLY
+            repo_git_base = (os.getenv('GITHUB_REPOSITORY') + NIGHTLY)
-            clone_url = os.getenv('GITHUB_SERVER_URL') + '/' + repo_git_base
+            clone_url = (os.getenv('GITHUB_SERVER_URL') + '/' + repo_git_base)
-            repo_base = get_repo_base_url(
+            repo_base = get_repo_base_url(clone_url, repo_git_base, force_type='github.com')
-                clone_url, repo_git_base, force_type='github.com'
-            )
             servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base
-            deploy_key_url = (
+            deploy_key_url = ('https://github.com/' + repo_git_base + '/settings/keys'
-                f'https://github.com/{repo_git_base}/settings/keys'
+                              + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys')
-                + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
-            )
             git_user_name = os.getenv('GITHUB_ACTOR')
             git_user_email = git_user_name + '@' + platform.node()
         else:
@@ -395,12 +337,16 @@ def main():
         git_mirror_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
         git_mirror_repodir = os.path.join(git_mirror_fdroiddir, 'repo')
         git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata')
+        git_mirror_statsdir = os.path.join(git_mirror_fdroiddir, 'stats')
         if not os.path.isdir(git_mirror_repodir):
-            clone_git_repo(clone_url, git_mirror_path)
+            logging.debug(_('cloning {url}').format(url=clone_url))
+            vcs = common.getvcs('git', clone_url, git_mirror_path)
+            p = vcs.git(['clone', '--', vcs.remote, str(vcs.local)])
+            if p.returncode != 0:
+                print('WARNING: only public git repos are supported!')
+                raise VCSException('git clone %s failed:' % clone_url, p.output)
         if not os.path.isdir(git_mirror_repodir):
             os.makedirs(git_mirror_repodir, mode=0o755)
-        if os.path.exists('LICENSE'):
-            shutil.copy2('LICENSE', git_mirror_path)
 
         mirror_git_repo = git.Repo.init(git_mirror_path)
         writer = mirror_git_repo.config_writer()
@@ -419,13 +365,9 @@ You can use it with the [F-Droid](https://f-droid.org/) Android app.
 
 [![{repo_url}]({repo_url}/icons/icon.png)](https://fdroid.link/#{repo_url})
 
-Last updated: {date}'''.format(
+Last updated: {date}'''.format(repo_git_base=repo_git_base,
-            repo_git_base=repo_git_base,
                                repo_url=repo_url,
-            date=datetime.datetime.now(datetime.timezone.utc).strftime(
+                               date=datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC'))
-                '%Y-%m-%d %H:%M:%S UTC'
-            ),
-        )
         with open(readme_path, 'w') as fp:
             fp.write(readme)
         mirror_git_repo.git.add(all=True)
@@ -439,6 +381,8 @@ Last updated: {date}'''.format(
             common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/')
         if os.path.isdir(git_mirror_metadatadir):
             common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/')
+        if os.path.isdir(git_mirror_statsdir):
+            common.local_rsync(options, [git_mirror_statsdir + '/'], 'stats/')
 
         ssh_private_key_file = _ssh_key_from_debug_keystore()
         # this is needed for GitPython to find the SSH key
@@ -478,17 +422,19 @@ Last updated: {date}'''.format(
             'keypass': PASSWORD,
             'keydname': DISTINGUISHED_NAME,
             'make_current_version_link': False,
+            'update_stats': True,
         }
-        with open(common.CONFIG_FILE, 'w', encoding='utf-8') as fp:
+        with open('config.yml', 'w') as fp:
             yaml.dump(config, fp, default_flow_style=False)
-        os.chmod(common.CONFIG_FILE, 0o600)
+        os.chmod('config.yml', 0o600)
         config = common.read_config()
         common.assert_config_keystore(config)
 
         logging.debug(
-            _(
+            _('Run over {cibase} to find -debug.apk. and skip repo_basedir {repo_basedir}').format(
-                'Run over {cibase} to find -debug.apk. and skip repo_basedir {repo_basedir}'
+                cibase=cibase,
-            ).format(cibase=cibase, repo_basedir=repo_basedir)
+                repo_basedir=repo_basedir
+            )
         )
 
         for root, dirs, files in os.walk(cibase):
@@ -550,6 +496,7 @@ Last updated: {date}'''.format(
         common.local_rsync(
             options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
         )
+        common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/')
         mirror_git_repo.git.add(all=True)
         mirror_git_repo.index.commit("update app metadata")
 
@@ -576,16 +523,10 @@ Last updated: {date}'''.format(
             if not os.path.exists(androiddir):
                 os.mkdir(androiddir)
                 logging.info(_('created {path}').format(path=androiddir))
-            logging.error(
+            logging.error(_('{path} does not exist!  Create it by running:').format(path=options.keystore)
-                _('{path} does not exist!  Create it by running:').format(
+                          + '\n    keytool -genkey -v -keystore ' + options.keystore + ' -storepass android \\'
-                    path=options.keystore
-                )
-                + '\n    keytool -genkey -v -keystore '
-                + options.keystore
-                + ' -storepass android \\'
                           + '\n     -alias androiddebugkey -keypass android -keyalg RSA -keysize 2048 -validity 10000 \\'
-                + '\n     -dname "CN=Android Debug,O=Android,C=US"'
+                          + '\n     -dname "CN=Android Debug,O=Android,C=US"')
-            )
             sys.exit(1)
         ssh_dir = os.path.join(os.getenv('HOME'), '.ssh')
         privkey = _ssh_key_from_debug_keystore(options.keystore)

fdroidserver/publish.py

@@ -28,21 +28,23 @@ mostly reports success by moving an APK from unsigned/ to repo/
 
 """
 
-import glob
+import sys
-import hashlib
-import json
-import logging
 import os
 import re
 import shutil
-import sys
+import glob
-import time
+import hashlib
-import zipfile
 from argparse import ArgumentParser
 from collections import OrderedDict
+import logging
 from gettext import ngettext
+import json
+import time
+import zipfile
 
-from . import _, common, metadata
+from . import _
+from . import common
+from . import metadata
 from .common import FDroidPopen
 from .exception import BuildException, FDroidException
 
@@ -137,13 +139,13 @@ def sign_sig_key_fingerprint_list(jar_file):
         raise FDroidException("Failed to sign '{}'!".format(jar_file))
 
 
-def store_publish_signer_fingerprints(appids, indent=None):
+def store_stats_fdroid_signing_key_fingerprints(appids, indent=None):
     """Store list of all signing-key fingerprints for given appids to HD.
 
     This list will later on be needed by fdroid update.
     """
-    if not os.path.exists('repo'):
+    if not os.path.exists('stats'):
-        os.makedirs('repo')
+        os.makedirs('stats')
     data = OrderedDict()
     fps = read_fingerprints_from_keystore()
     for appid in sorted(appids):
@@ -151,12 +153,9 @@ def store_publish_signer_fingerprints(appids, indent=None):
         if alias in fps:
             data[appid] = {'signer': fps[key_alias(appid)]}
 
-    jar_file = os.path.join('repo', 'signer-index.jar')
+    jar_file = os.path.join('stats', 'publishsigkeys.jar')
-    output = json.dumps(data, indent=indent)
     with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar:
-        jar.writestr('signer-index.json', output)
+        jar.writestr('publishsigkeys.json', json.dumps(data, indent=indent))
-    with open(os.path.join('repo', 'signer-index.json'), 'w') as fp:
-        fp.write(output)
     sign_sig_key_fingerprint_list(jar_file)
 
 
@@ -345,6 +344,7 @@ def main():
         glob.glob(os.path.join(unsigned_dir, '*.apk'))
         + glob.glob(os.path.join(unsigned_dir, '*.zip'))
     ):
+
         appid, vercode = common.publishednameinfo(apkfile)
         apkfilename = os.path.basename(apkfile)
         if vercodes and appid not in vercodes:
@@ -368,6 +368,7 @@ def main():
             if b.get("versionCode") == vercode:
                 build = b
         if app.Binaries or (build and build.binary):
+
             # It's an app where we build from source, and verify the apk
             # contents against a developer's binary, and then publish their
             # version if everything checks out.
@@ -378,21 +379,15 @@ def main():
             srcapk = srcapk.replace(unsigned_dir, binaries_dir)
 
             if not os.path.isfile(srcapk):
-                logging.error(
+                logging.error("...reference binary missing - publish skipped: "
-                    "...reference binary missing - publish skipped: '{refpath}'".format(
+                              "'{refpath}'".format(refpath=srcapk))
-                        refpath=srcapk
-                    )
-                )
                 failed += 1
             else:
                 # Compare our unsigned one with the downloaded one...
                 compare_result = common.verify_apks(srcapk, apkfile, tmp_dir)
                 if compare_result:
-                    logging.error(
+                    logging.error("...verification failed - publish skipped : "
-                        "...verification failed - publish skipped : {result}".format(
+                                  "{result}".format(result=compare_result))
-                            result=compare_result
-                        )
-                    )
                     failed += 1
                 else:
                     # Success! So move the downloaded file to the repo, and remove
@@ -404,6 +399,7 @@ def main():
                     logging.info('Published ' + apkfilename)
 
         elif apkfile.endswith('.zip'):
+
             # OTA ZIPs built by fdroid do not need to be signed by jarsigner,
             # just to be moved into place in the repo
             shutil.move(apkfile, os.path.join(output_dir, apkfilename))
@@ -411,6 +407,7 @@ def main():
             logging.info('Published ' + apkfilename)
 
         else:
+
             # It's a 'normal' app, i.e. we sign and publish it...
             skipsigning = False
 
@@ -449,11 +446,10 @@ def main():
 
                 signed_apk_path = os.path.join(output_dir, apkfilename)
                 if os.path.exists(signed_apk_path):
-                    raise BuildException(
+                    raise BuildException("Refusing to sign '{0}' file exists in both "
-                        _(
+                                         "{1} and {2} folder.".format(apkfilename,
-                            "Refusing to sign '{path}', file exists in both {dir1} and {dir2} folder."
+                                                                      unsigned_dir,
-                        ).format(path=apkfilename, dir1=unsigned_dir, dir2=output_dir)
+                                                                      output_dir))
-                    )
 
                 # Sign the application...
                 common.sign_apk(apkfile, signed_apk_path, keyalias)
@@ -464,7 +460,7 @@ def main():
                 publish_source_tarball(apkfilename, unsigned_dir, output_dir)
                 logging.info('Published ' + apkfilename)
 
-    store_publish_signer_fingerprints(allapps.keys())
+    store_stats_fdroid_signing_key_fingerprints(allapps.keys())
     status_update_json(generated_keys, signed_apks)
     logging.info('published list signing-key fingerprints')
 

fdroidserver/readmeta.py

@@ -17,8 +17,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from argparse import ArgumentParser
-
+from . import common
-from . import common, metadata
+from . import metadata
 
 
 def main():

fdroidserver/rewritemeta.py

@@ -17,14 +17,16 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import io
-import logging
-import shutil
-import tempfile
 from argparse import ArgumentParser
+import logging
+import io
+import tempfile
+import shutil
 from pathlib import Path
 
-from . import _, common, metadata
+from . import _
+from . import common
+from . import metadata
 
 config = None
 
@@ -79,7 +81,9 @@ def main():
 
     config = common.read_config()
 
-    apps = common.read_app_args(options.appid)
+    # Get all apps...
+    allapps = metadata.read_metadata(options.appid)
+    apps = common.read_app_args(options.appid, allapps, False)
 
     for appid, app in apps.items():
         path = Path(app.metadatapath)

fdroidserver/scanner.py

@@ -16,6 +16,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import imghdr
 import itertools
 import json
 import logging
@@ -28,21 +29,10 @@ import urllib.request
 import zipfile
 from argparse import ArgumentParser
 from dataclasses import dataclass, field, fields
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 from enum import IntEnum
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Union
-
-try:
-    import magic
-except ImportError:
-    import puremagic as magic
-
-if sys.version_info >= (3, 11):
-    import tomllib
-else:
-    import tomli as tomllib
 
 from . import _, common, metadata, scanner
 from .exception import BuildException, ConfigurationException, VCSException
@@ -63,198 +53,18 @@ MAVEN_URL_REGEX = re.compile(
 DEPFILE = {
     "Cargo.toml": ["Cargo.lock"],
     "pubspec.yaml": ["pubspec.lock"],
-    "package.json": ["package-lock.json", "yarn.lock", "pnpm-lock.yaml", "bun.lock"],
+    "package.json": ["package-lock.json", "yarn.lock", "pnpm-lock.yaml"],
 }
 
 SCANNER_CACHE_VERSION = 1
 
-DEFAULT_CATALOG_PREFIX_REGEX = re.compile(
-    r'''defaultLibrariesExtensionName\s*=\s*['"](\w+)['"]'''
-)
-GRADLE_CATALOG_FILE_REGEX = re.compile(
-    r'''(?:create\()?['"]?(\w+)['"]?\)?\s*\{[^}]*from\(files\(['"]([^"]+)['"]\)\)'''
-)
-VERSION_CATALOG_REGEX = re.compile(r'versionCatalogs\s*\{')
-
-APK_SIGNING_BLOCK_IDS = {
-    # https://source.android.com/docs/security/features/apksigning/v2#apk-signing-block
-    # 0x7109871a: 'APK signature scheme v2',
-    # https://source.android.com/docs/security/features/apksigning/v3#apk-signing-block
-    # 0xf05368c0: 'APK signature scheme v3',
-    # See "Security metadata in early 2018"
-    # https://android-developers.googleblog.com/2017/12/improving-app-security-and-performance.html
-    0x2146444E: 'Google Play Signature aka "Frosting"',
-    # 0x42726577: 'Verity padding',
-    # 0x6DFF800D: 'Source stamp V2 X509 cert',
-    # JSON with some metadata, used by Chinese company Meituan
-    0x71777777: 'Meituan payload',
-    # Dependencies metadata generated by Gradle and encrypted by Google Play.
-    # '...The data is compressed, encrypted by a Google Play signing key...'
-    # https://developer.android.com/studio/releases/gradle-plugin#dependency-metadata
-    0x504B4453: 'Dependency metadata',
-}
-
 
 class ExitCode(IntEnum):
     NONFREE_CODE = 1
 
 
-class GradleVersionCatalog:
-    """Parse catalog from libs.versions.toml.
-
-    https://docs.gradle.org/current/userguide/platforms.html
-    """
-
-    def __init__(self, catalog):
-        self.version = {
-            alias: self.get_version(version)
-            for alias, version in catalog.get("versions", {}).items()
-        }
-        self.libraries = {
-            self.alias_to_accessor(alias): self.library_to_coordinate(library)
-            for alias, library in catalog.get("libraries", {}).items()
-        }
-        self.plugins = {
-            self.alias_to_accessor(alias): self.plugin_to_coordinate(plugin)
-            for alias, plugin in catalog.get("plugins", {}).items()
-        }
-        self.bundles = {
-            self.alias_to_accessor(alias): self.bundle_to_coordinates(bundle)
-            for alias, bundle in catalog.get("bundles", {}).items()
-        }
-
-    @staticmethod
-    def alias_to_accessor(alias: str) -> str:
-        """Covert alias to accessor.
-
-        https://docs.gradle.org/current/userguide/platforms.html#sub:mapping-aliases-to-accessors
-        Alias is used to define a lib in catalog. Accessor is used to access it.
-        """
-        return alias.replace("-", ".").replace("_", ".")
-
-    def get_version(self, version: Union[dict, str]) -> str:
-        if isinstance(version, str):
-            return version
-        ref = version.get("ref")
-        if ref:
-            return self.version.get(ref, "")
-        return (
-            version.get("prefer", "")
-            or version.get("require", "")
-            or version.get("strictly", "")
-        )
-
-    def library_to_coordinate(self, library: Union[dict, str]) -> str:
-        """Generate the Gradle dependency coordinate from catalog."""
-        if isinstance(library, str):
-            return library
-        module = library.get("module")
-        if not module:
-            group = library.get("group")
-            name = library.get("name")
-            if group and name:
-                module = f"{group}:{name}"
-            else:
-                return ""
-
-        version = library.get("version")
-        if version:
-            return f"{module}:{self.get_version(version)}"
-        else:
-            return module
-
-    def plugin_to_coordinate(self, plugin: Union[dict, str]) -> str:
-        """Generate the Gradle plugin coordinate from catalog."""
-        if isinstance(plugin, str):
-            return plugin
-        id = plugin.get("id")
-        if not id:
-            return ""
-
-        version = plugin.get("version")
-        if version:
-            return f"{id}:{self.get_version(version)}"
-        else:
-            return id
-
-    def bundle_to_coordinates(self, bundle: list[str]) -> list[str]:
-        """Generate the Gradle dependency bundle coordinate from catalog."""
-        coordinates = []
-        for alias in bundle:
-            library = self.libraries.get(self.alias_to_accessor(alias))
-            if library:
-                coordinates.append(library)
-        return coordinates
-
-    def get_coordinate(self, accessor: str) -> list[str]:
-        """Get the Gradle coordinate from the catalog with an accessor."""
-        if accessor.startswith("plugins."):
-            return [
-                self.plugins.get(accessor[8:].removesuffix(".asLibraryDependency"), "")
-            ]
-        if accessor.startswith("bundles."):
-            return self.bundles.get(accessor[8:], [])
-        return [self.libraries.get(accessor, "")]
-
-
-def get_catalogs(root: str) -> dict[str, GradleVersionCatalog]:
-    """Get all Gradle dependency catalogs from settings.gradle[.kts].
-
-    Returns a dict with the extension and the corresponding catalog.
-    The extension is used as the prefix of the accessor to access libs in the catalog.
-    """
-    root = Path(root)
-    catalogs = {}
-    default_prefix = "libs"
-    catalog_files_m = []
-
-    def find_block_end(s, start):
-        pat = re.compile("[{}]")
-        depth = 1
-        for m in pat.finditer(s, pos=start):
-            if m.group() == "{":
-                depth += 1
-            else:
-                depth -= 1
-            if depth == 0:
-                return m.start()
-        else:
-            return -1
-
-    groovy_file = root / "settings.gradle"
-    kotlin_file = root / "settings.gradle.kts"
-    if groovy_file.is_file():
-        gradle_file = groovy_file
-    elif kotlin_file.is_file():
-        gradle_file = kotlin_file
-    else:
-        return {}
-
-    s = gradle_file.read_text(encoding="utf-8")
-    version_catalogs_m = VERSION_CATALOG_REGEX.search(s)
-    if version_catalogs_m:
-        start = version_catalogs_m.end()
-        end = find_block_end(s, start)
-        catalog_files_m = GRADLE_CATALOG_FILE_REGEX.finditer(s, start, end)
-
-    m_default = DEFAULT_CATALOG_PREFIX_REGEX.search(s)
-    if m_default:
-        default_prefix = m_default.group(1)
-    default_catalog_file = Path(root) / "gradle/libs.versions.toml"
-    if default_catalog_file.is_file():
-        with default_catalog_file.open("rb") as f:
-            catalogs[default_prefix] = GradleVersionCatalog(tomllib.load(f))
-    for m in catalog_files_m:
-        catalog_file = Path(root) / m.group(2).replace("$rootDir/", "")
-        if catalog_file.is_file():
-            with catalog_file.open("rb") as f:
-                catalogs[m.group(1)] = GradleVersionCatalog(tomllib.load(f))
-    return catalogs
-
-
 def get_gradle_compile_commands(build):
     compileCommands = [
-        'alias',
         'api',
         'apk',
         'classpath',
@@ -266,30 +76,19 @@ def get_gradle_compile_commands(build):
         'runtimeOnly',
     ]
     buildTypes = ['', 'release']
-    if build.gradle and build.gradle != ['yes']:
-        flavors = common.calculate_gradle_flavor_combination(build.gradle)
-    else:
     flavors = ['']
+    if build.gradle and build.gradle != ['yes']:
+        flavors += build.gradle
 
-    return [''.join(c) for c in itertools.product(flavors, buildTypes, compileCommands)]
+    commands = [
-
+        ''.join(c) for c in itertools.product(flavors, buildTypes, compileCommands)
-
-def get_gradle_compile_commands_without_catalog(build):
-    return [
-        re.compile(rf'''\s*{c}.*\s*\(?['"].*['"]''', re.IGNORECASE)
-        for c in get_gradle_compile_commands(build)
-    ]
-
-
-def get_gradle_compile_commands_with_catalog(build, prefix):
-    return [
-        re.compile(rf'\s*{c}.*\s*\(?{prefix}\.([a-z0-9.]+)', re.IGNORECASE)
-        for c in get_gradle_compile_commands(build)
     ]
+    return [re.compile(r'\s*' + c, re.IGNORECASE) for c in commands]
 
 
 def get_embedded_classes(apkfile, depth=0):
-    """Get the list of Java classes embedded into all DEX files.
+    """
+    Get the list of Java classes embedded into all DEX files.
 
     :return: set of Java classes names as string
     """
@@ -335,7 +134,7 @@ def get_embedded_classes(apkfile, depth=0):
 
 def _datetime_now():
     """Get datetime.now(), using this funciton allows mocking it for testing."""
-    return datetime.now(timezone.utc)
+    return datetime.utcnow()
 
 
 def _scanner_cachedir():
@@ -384,7 +183,8 @@ class SignatureDataController:
             raise SignatureDataVersionMismatchException()
 
     def check_last_updated(self):
-        """Check if the last_updated value is ok and raise an exception if expired or inaccessible.
+        """
+        Check if the last_updated value is ok and raise an exception if expired or inaccessible.
 
         :raises SignatureDataMalformedException: when timestamp value is
                                                  inaccessible or not parse-able
@@ -394,7 +194,7 @@ class SignatureDataController:
         last_updated = self.data.get("last_updated", None)
         if last_updated:
             try:
-                last_updated = datetime.fromtimestamp(last_updated, timezone.utc)
+                last_updated = datetime.fromtimestamp(last_updated)
             except ValueError as e:
                 raise SignatureDataMalformedException() from e
             except TypeError as e:
@@ -459,7 +259,8 @@ class SignatureDataController:
         logging.debug("write '{}' to cache".format(self.filename))
 
     def verify_data(self):
-        """Clean and validate `self.data`.
+        """
+        Clean and validate `self.data`.
 
         Right now this function does just a basic key sanitation.
         """
@@ -553,8 +354,6 @@ class SUSSDataController(SignatureDataController):
 
 
 class ScannerTool:
-    refresh_allowed = True
-
     def __init__(self):
         # we could add support for loading additional signature source
         # definitions from config.yml here
@@ -563,9 +362,7 @@ class ScannerTool:
 
         options = common.get_options()
         options_refresh_scanner = (
-            hasattr(options, "refresh_scanner")
+            hasattr(options, "refresh_scanner") and options.refresh_scanner
-            and options.refresh_scanner
-            and ScannerTool.refresh_allowed
         )
         if options_refresh_scanner or common.get_config().get('refresh_scanner'):
             self.refresh()
@@ -650,7 +447,8 @@ _SCANNER_TOOL = None
 
 
 def _get_tool():
-    """Lazy loading function for getting a ScannerTool instance.
+    """
+    Lazy loading function for getting a ScannerTool instance.
 
     ScannerTool initialization need to access `common.config` values. Those are only available after initialization through `common.read_config()`. So this factory assumes config was called at an erlier point in time.
     """
@@ -673,17 +471,6 @@ def scan_binary(apkfile):
             if regexp.match(classname):
                 logging.debug("Problem: found class '%s'" % classname)
                 problems += 1
-
-    logging.info(_('Scanning APK for extra signing blocks.'))
-    a = common.get_androguard_APK(str(apkfile))
-    a.parse_v2_v3_signature()
-    for b in a._v2_blocks:
-        if b in APK_SIGNING_BLOCK_IDS:
-            logging.debug(
-                f"Problem: found extra signing block '{APK_SIGNING_BLOCK_IDS[b]}'"
-            )
-            problems += 1
-
     if warnings:
         logging.warning(
             _("Found {count} warnings in {filename}").format(
@@ -705,7 +492,6 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
     Returns
     -------
     the number of fatal problems encountered.
-
     """
     count = 0
 
@@ -721,11 +507,11 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
         re.compile(r'^https://' + re.escape(repo) + r'/*')
         for repo in [
             'repo1.maven.org/maven2',  # mavenCentral()
+            'jcenter.bintray.com',  # jcenter()
             'jitpack.io',
             'www.jitpack.io',
             'repo.maven.apache.org/maven2',
             'oss.jfrog.org/artifactory/oss-snapshot-local',
-            'central.sonatype.com/repository/maven-snapshots',
             'oss.sonatype.org/content/repositories/snapshots',
             'oss.sonatype.org/content/repositories/releases',
             'oss.sonatype.org/content/groups/public',
@@ -747,12 +533,8 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
         ]
     ]
 
-    scanignore, scanignore_not_found_paths = common.getpaths_map(
+    scanignore = common.getpaths_map(build_dir, build.scanignore)
-        build_dir, build.scanignore
+    scandelete = common.getpaths_map(build_dir, build.scandelete)
-    )
-    scandelete, scandelete_not_found_paths = common.getpaths_map(
-        build_dir, build.scandelete
-    )
 
     scanignore_worked = set()
     scandelete_worked = set()
@@ -786,7 +568,6 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
         Returns
         -------
         0 as we explicitly ignore the file, so don't count an error
-
         """
         msg = 'Ignoring %s at %s' % (what, path_in_build_dir)
         logging.info(msg)
@@ -809,7 +590,6 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
         Returns
         -------
         0 as we deleted the offending file
-
         """
         msg = 'Removing %s at %s' % (what, path_in_build_dir)
         logging.info(msg)
@@ -837,7 +617,6 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
         Returns
         -------
         0, as warnings don't count as errors
-
         """
         if toignore(path_in_build_dir):
             return 0
@@ -863,7 +642,6 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
         Returns
         -------
         0 if the problem was ignored/deleted/is only a warning, 1 otherwise
-
         """
         options = common.get_options()
         if toignore(path_in_build_dir):
@@ -901,12 +679,8 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
     ]
 
     def is_image_file(path):
-        try:
+        if imghdr.what(path) is not None:
-            mimetype = magic.from_file(path, mime=True)
-            if mimetype and mimetype.startswith('image/'):
             return True
-        except Exception as e:
-            logging.info(e)
 
     def safe_path(path_in_build_dir):
         for sp in safe_paths:
@@ -914,21 +688,11 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
                 return True
         return False
 
-    def is_used_by_gradle_without_catalog(line):
+    gradle_compile_commands = get_gradle_compile_commands(build)
-        return any(
-            command.match(line)
-            for command in get_gradle_compile_commands_without_catalog(build)
-        )
 
-    def is_used_by_gradle_with_catalog(line, prefix):
+    def is_used_by_gradle(line):
-        for m in (
+        return any(command.match(line) for command in gradle_compile_commands)
-            command.match(line)
-            for command in get_gradle_compile_commands_with_catalog(build, prefix)
-        ):
-            if m:
-                return m
 
-    all_catalogs = {}
     # Iterate through all files in the source code
     for root, dirs, files in os.walk(build_dir, topdown=True):
         # It's topdown, so checking the basename is enough
@@ -936,9 +700,6 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
             if ignoredir in dirs:
                 dirs.remove(ignoredir)
 
-        if "settings.gradle" in files or "settings.gradle.kts" in files:
-            all_catalogs[str(root)] = get_catalogs(root)
-
         for curfile in files:
             if curfile in ['.DS_Store']:
                 continue
@@ -1004,13 +765,6 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
                 count += handleproblem(
                     _('Java JAR file'), path_in_build_dir, filepath, json_per_build
                 )
-            elif curfile.endswith('.wasm'):
-                count += handleproblem(
-                    _('WebAssembly binary file'),
-                    path_in_build_dir,
-                    filepath,
-                    json_per_build,
-                )
 
             elif curfile.endswith('.java'):
                 if not os.path.isfile(filepath):
@@ -1027,36 +781,15 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
                             break
 
             elif curfile.endswith('.gradle') or curfile.endswith('.gradle.kts'):
-                catalog_path = str(build_dir)
-                # Find the longest path of dir that the curfile is in
-                for p in all_catalogs:
-                    if os.path.commonpath([root, p]) == p:
-                        catalog_path = p
-                catalogs = all_catalogs.get(catalog_path, {})
-
                 if not os.path.isfile(filepath):
                     continue
                 with open(filepath, 'r', errors='replace') as f:
                     lines = f.readlines()
                 for i, line in enumerate(lines):
-                    if is_used_by_gradle_without_catalog(line):
+                    if is_used_by_gradle(line):
                         for name in suspects_found(line):
                             count += handleproblem(
-                                f"usual suspect '{name}'",
+                                "usual suspect '%s'" % (name),
-                                path_in_build_dir,
-                                filepath,
-                                json_per_build,
-                            )
-                    for prefix, catalog in catalogs.items():
-                        m = is_used_by_gradle_with_catalog(line, prefix)
-                        if not m:
-                            continue
-                        accessor = m[1]
-                        coordinates = catalog.get_coordinate(accessor)
-                        for coordinate in coordinates:
-                            for name in suspects_found(coordinate):
-                                count += handleproblem(
-                                    f"usual suspect '{prefix}.{accessor}: {name}'",
                                 path_in_build_dir,
                                 filepath,
                                 json_per_build,
@@ -1110,19 +843,11 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None):
                         json_per_build,
                     )
 
-    for p in scanignore_not_found_paths:
-        logging.error(_("Non-exist scanignore path: %s") % p)
-        count += 1
-
     for p in scanignore:
         if p not in scanignore_worked:
             logging.error(_('Unused scanignore path: %s') % p)
             count += 1
 
-    for p in scandelete_not_found_paths:
-        logging.error(_("Non-exist scandelete path: %s") % p)
-        count += 1
-
     for p in scandelete:
         if p not in scandelete_worked:
             logging.error(_('Unused scandelete path: %s') % p)
@@ -1203,7 +928,10 @@ def main():
             _get_tool()
         return
 
-    apps = common.read_app_args(appids, allow_version_codes=True)
+    # Read all app and srclib metadata
+
+    allapps = metadata.read_metadata()
+    apps = common.read_app_args(appids, allapps, True)
 
     build_dir = 'build'
     if not os.path.isdir(build_dir):
@@ -1304,7 +1032,7 @@ def main():
     logging.info(_("Finished"))
     if options.json:
         print(json.dumps(json_output))
-    elif probcount or options.verbose:
+    else:
         print(_("%d problems found") % probcount)
 
 
@@ -2227,13 +1955,13 @@ SUSS_DEFAULT = r'''{
         "https://www.android.com/gms/"
       ],
       "gradle_signatures": [
-        "com.google.android.gms(?!.(oss-licenses-plugin|strict-version-matcher-plugin))",
+        "com.google.android.gms(?!.oss-licenses-plugin)",
         "com.google.android.ump",
         "androidx.core:core-google-shortcuts",
+        "androidx.credentials:credentials",
         "androidx.credentials:credentials-play-services-auth",
         "androidx.media3:media3-cast",
         "androidx.media3:media3-datasource-cronet",
-        "androidx.wear:wear-remote-interactions",
         "androidx.work:work-gcm",
         "com.google.android.exoplayer:extension-cast",
         "com.google.android.exoplayer:extension-cronet",
@@ -2241,21 +1969,9 @@ SUSS_DEFAULT = r'''{
         "com.cloudinary:cloudinary-android.*:2\\.[12]\\.",
         "com.pierfrancescosoffritti.androidyoutubeplayer:chromecast-sender",
         "com.yayandroid:locationmanager",
-        "(?<!org.microg.gms:)play-services",
+        "play-services",
         "xyz.belvi.mobilevision:barcodescanner",
-        "com.google.api-client:google-api-client-android",
+        "com.google.api-client:google-api-client-android"
-        "com.google.maps.android:android-maps-utils",
-        "com.github.budowski:android-maps-utils",
-        "com.microsoft.identity:common",
-        "com.microsoft.identity.client:msal"
-      ],
-      "gradle_signatures_negative_examples": [
-        "com.google.android.gms.oss-licenses-plugin",
-        "com.google.android.gms.strict-version-matcher-plugin"
-      ],
-      "gradle_signatures_positive_examples": [
-        "com.google.android.gms:play-services-base",
-        "com.google.android.gms:play-services-oss-licenses"
       ],
       "license": "NonFree",
       "name": "Google Mobile Services"
@@ -2273,10 +1989,7 @@ SUSS_DEFAULT = r'''{
     },
     "com.google.android.libraries": {
       "code_signatures": [
-        "com/google/android/libraries(?!/accessibility)"
+        "com/google/android/libraries"
-      ],
-      "code_signatures_negative_examples": [
-        "com/google/android/libraries/accessibility"
       ],
       "gradle_signatures": [
         "com.google.android.libraries(?!.mapsplatform.secrets-gradle-plugin)"
@@ -2397,8 +2110,7 @@ SUSS_DEFAULT = r'''{
         "https://developers.google.com/ml-kit"
       ],
       "gradle_signatures": [
-        "com.google.mlkit",
+        "com.google.mlkit"
-        "io.github.g00fy2.quickie"
       ],
       "license": "NonFree",
       "name": "ML Kit"
@@ -2623,19 +2335,6 @@ SUSS_DEFAULT = r'''{
       "description": "processes real-time data at the intersection of commerce and culture, providing useful, actionable insights for brands and publishers. See <a rel='nofollow' href='https://www.crunchbase.com/organization/quantcast'>Crunchbase</a> and <a href='https://reports.exodus-privacy.eu.org/en/trackers/133/'>Exodus Privacy</a>.",
       "license": "NonFree"
     },
-    "com.revenuecat.purchases": {
-      "code_signatures": [
-        "com/revenuecat/purchases"
-      ],
-      "documentation": [
-        "https://www.revenuecat.com/"
-      ],
-      "gradle_signatures": [
-        "com.revenuecat.purchases"
-      ],
-      "license": "NonFree",
-      "name": "RevenueCat Purchases"
-    },
     "com.samsung.accessory": {
       "anti_features": [
         "NonFreeComp"
@@ -2769,9 +2468,6 @@ SUSS_DEFAULT = r'''{
         "com.wei.android.lib:fingerprintidentify",
         "com.github.uccmawei:FingerprintIdentify"
       ],
-      "gradle_signatures_positive_examples": [
-        "implementation \"com.github.uccmawei:fingerprintidentify:${safeExtGet(\"fingerprintidentify\", \"1.2.6\")}\""
-      ],
       "license": "NonFree",
       "name": "FingerprintIdentify"
     },
@@ -2955,16 +2651,6 @@ SUSS_DEFAULT = r'''{
       "license": "NonFree",
       "name": "Pushy"
     },
-    "org.gradle.toolchains.foojay-resolver-convention": {
-      "documentation": [
-        "https://github.com/gradle/foojay-toolchains"
-      ],
-      "gradle_signatures": [
-        "org.gradle.toolchains.foojay-resolver"
-      ],
-      "license": "Apache-2.0",
-      "name": "Foojay Toolchains Plugin"
-    },
     "org.mariuszgromada.math": {
       "code_signatures": [
         "org/mariuszgromada/math/mxparser/parsertokens/SyntaxStringBuilder",
@@ -2997,7 +2683,7 @@ SUSS_DEFAULT = r'''{
       "license": "NonFree"
     }
   },
-  "timestamp": 1747829076.702502,
+  "timestamp": 1725205987.66681,
   "version": 1,
-  "last_updated": 1750710966.431471
+  "last_updated": 1725950235.569432
 }'''

fdroidserver/signatures.py

@@ -15,13 +15,16 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-import logging
-import os
-import re
-import sys
 from argparse import ArgumentParser
 
-from . import _, common
+import re
+import os
+import sys
+import logging
+
+from . import _
+from . import common
+from . import net
 from .exception import FDroidException
 
 
@@ -65,8 +68,6 @@ def extract(options):
             elif httpre.match(apk):
                 if apk.startswith('https') or options.no_check_https:
                     try:
-                        from . import net
-
                         tmp_apk = os.path.join(tmp_dir, 'signed.apk')
                         net.download_file(apk, tmp_apk)
                         sigdir = extract_signature(tmp_apk)
@@ -103,7 +104,7 @@ def main():
     )
     parser.add_argument("--no-check-https", action="store_true", default=False)
     options = common.parse_args(parser)
-    common.set_console_logging(options.verbose, options.color)
+    common.set_console_logging(options.verbose)
     common.read_config()
 
     extract(options)

fdroidserver/signindex.py

@@ -17,13 +17,15 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
-import logging
 import os
 import time
 import zipfile
 from argparse import ArgumentParser
+import logging
 
-from . import _, common, metadata
+from . import _
+from . import common
+from . import metadata
 from .exception import FDroidException
 
 config = None

fdroidserver/tail.py

@@ -28,8 +28,8 @@ Example
 
 import os
 import sys
-import threading
 import time
+import threading
 
 
 class Tail(object):

fdroidserver/update.py

@@ -20,27 +20,25 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import argparse
-import copy
-import filecmp
-import glob
-import hashlib
-import json
-import logging
-import os
-import re
-import shutil
-import socket
 import sys
-import time
+import os
+import shutil
+import glob
+import logging
+import re
+import socket
 import warnings
 import zipfile
-from argparse import ArgumentParser
+import hashlib
-from datetime import datetime, timezone
+import json
-from pathlib import Path
+import time
-
+import yaml
+import copy
 import asn1crypto.cms
 import defusedxml.ElementTree as ElementTree
-import yaml
+from datetime import datetime, timezone
+from argparse import ArgumentParser
+from pathlib import Path
 
 try:
     from yaml import CSafeLoader as SafeLoader
@@ -50,13 +48,14 @@ except ImportError:
 import collections
 from binascii import hexlify
 
-from PIL import Image, PngImagePlugin
+from . import _
-
+from . import common
+from . import metadata
+from .common import DEFAULT_LOCALE
+from .exception import BuildException, FDroidException, VerificationException
 import fdroidserver.index
 
-from . import _, common, metadata
+from PIL import Image, PngImagePlugin
-from .common import DEFAULT_LOCALE
-from .exception import BuildException, FDroidException, NoVersionCodeException, VerificationException
 
 if hasattr(Image, 'DecompressionBombWarning'):
     warnings.simplefilter('error', Image.DecompressionBombWarning)
@@ -134,7 +133,7 @@ def disabled_algorithms_allowed():
             or common.default_config['allow_disabled_algorithms'])
 
 
-def status_update_json(output, apps, apks):
+def status_update_json(apps, apks):
     """Output a JSON file with metadata about this `fdroid update` run.
 
     Parameters
@@ -146,6 +145,7 @@ def status_update_json(output, apps, apks):
 
     """
     logging.debug(_('Outputting JSON'))
+    output = common.setup_status_output(start_timestamp)
     output['antiFeatures'] = dict()
     output['disabled'] = []
     output['archivePolicy0'] = []
@@ -202,13 +202,6 @@ def status_update_json(output, apps, apks):
     common.write_status_json(output, options.pretty)
 
 
-def output_status_stage(output, stage):
-    if 'stages' not in output:
-        output['stages'] = dict()
-    output['stages'][stage] = common.epoch_millis_now()
-    common.write_running_status_json(output)
-
-
 def delete_disabled_builds(apps, apkcache, repodirs):
     """Delete disabled build outputs.
 
@@ -323,13 +316,6 @@ def get_cache_file():
     return os.path.join('tmp', 'apkcache.json')
 
 
-def get_cache_mtime():
-    apkcachefile = get_cache_file()
-    if os.path.exists(apkcachefile):
-        return os.stat(apkcachefile).st_mtime
-    return 0
-
-
 def get_cache():
     """Get the cached dict of the APK index.
 
@@ -369,13 +355,7 @@ def get_cache():
             if not isinstance(v['antiFeatures'], dict):
                 v['antiFeatures'] = {k: {} for k in sorted(v['antiFeatures'])}
         if 'added' in v:
-            v['added'] = datetime.fromtimestamp(v['added'], tz=timezone.utc)
+            v['added'] = datetime.fromtimestamp(v['added'])
-        if v.get('srcname') and not v.get('srcnameSha256'):
-            f = f'archive/{v["srcname"]}'
-            if not os.path.exists(f):
-                f = f'repo/{v["srcname"]}'
-            if os.path.exists(f):
-                v['srcnameSha256'] = common.sha256sum(f)
 
     return apkcache
 
@@ -511,7 +491,7 @@ def insert_obbs(repodir, apps, apks):
             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
             continue
         if not re.match(r'^-?[0-9]+$', chunks[1]):
-            obbWarnDelete(f, _('The OBB versionCode must come after "{name}.":')
+            obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
                           .format(name=chunks[0]))
             continue
         versionCode = int(chunks[1])
@@ -550,7 +530,7 @@ VERSION_STRING_RE = re.compile(r'^([0-9]+)\.([0-9]+)\.([0-9]+)$')
 
 def version_string_to_int(version):
     """
-    Convert semantic version designation to versionCode.
+    Convert sermver version designation to version code.
 
     Approximately convert a [Major].[Minor].[Patch] version string
     consisting of numeric characters (0-9) and periods to a number. The
@@ -703,7 +683,7 @@ def scan_repo_for_ipas(apkcache, repodir, knownapks):
             apkcache[ipa_name] = ipa
             cachechanged = True
 
-        added = knownapks.recordapk(ipa_name)
+        added = knownapks.recordapk(ipa_name, ipa['packageName'])
         if added:
             ipa['added'] = added
 
@@ -807,20 +787,16 @@ def _strip_and_copy_image(in_file, outpath):
     It is not used at all in the F-Droid ecosystem, so its much safer
     just to remove it entirely.
 
-    This only uses ctime/mtime to check for a new file since this
+    This uses size+mtime to check for a new file since this process
-    process actually modifies the resulting file to strip out the EXIF.
+    actually modifies the resulting file to strip out the EXIF.
-    Therefore, whenever the file needs to be stripped, it will have a
-    newer ctime and most likely a different size.  The mtime is copied
-    from the source to the destination, so it can be the same.
 
     outpath can be path to either a file or dir.  The dir that outpath
     refers to must exist before calling this.
 
     Potential source of Python code to strip JPEGs without dependencies:
     http://www.fetidcascade.com/public/minimal_exif_writer.py
-
     """
-    logging.debug('copying %s %s', in_file, outpath)
+    logging.debug('copying ' + in_file + ' ' + outpath)
 
     if not os.path.exists(in_file):
         if os.path.islink(in_file):
@@ -834,10 +810,11 @@ def _strip_and_copy_image(in_file, outpath):
     else:
         out_file = outpath
 
-    if os.path.exists(out_file) and (
+    if os.path.exists(out_file):
-        os.path.getmtime(in_file) <= os.path.getmtime(out_file)
+        in_stat = os.stat(in_file)
-        and os.path.getctime(in_file) <= os.path.getctime(out_file)
+        out_stat = os.stat(out_file)
-    ):
+        if in_stat.st_size == out_stat.st_size \
+           and in_stat.st_mtime == out_stat.st_mtime:
             return
 
     extension = common.get_extension(in_file)[1]
@@ -1042,20 +1019,16 @@ def copy_triple_t_store_metadata(apps):
     for packageName, app in apps.items():
         builds = app.get('Builds', [])
         gradle_subdirs = set()
-        if builds:
+        if builds and builds[-1].subdir:
-            subdir = builds[-1].subdir or ''
             for flavor in builds[-1].gradle:
                 if flavor not in ('yes', 'no', True, False):
-                    p = os.path.join('build', packageName, subdir, 'src', flavor, 'play')
+                    p = os.path.join('build', packageName, builds[-1].subdir, 'src', flavor, 'play')
                     if os.path.exists(p):
                         gradle_subdirs.add(p)
             if not gradle_subdirs:
-                gradle_subdirs.update(glob.glob(os.path.join('build', packageName, subdir, 'src', '*', 'play')))
+                gradle_subdirs.update(glob.glob(os.path.join('build', packageName, builds[-1].subdir, 'src', '*', 'play')))
             if not gradle_subdirs:
-                gradle_subdirs.update(glob.glob(os.path.join('build', packageName, subdir, '*', 'src', '*', 'play')))
+                gradle_subdirs.update(glob.glob(os.path.join('build', packageName, builds[-1].subdir, '*', 'src', '*', 'play')))
-            if not gradle_subdirs:
-                # Flutter-style android subdir
-                gradle_subdirs.update(glob.glob(os.path.join('build', packageName, subdir, 'android', 'app', 'src', '*', 'play')))
         if not gradle_subdirs:
             sg_list = sorted(glob.glob(os.path.join('build', packageName, 'settings.gradle*')))
             if sg_list:
@@ -1124,9 +1097,6 @@ def copy_triple_t_store_metadata(apps):
                             repofilename = os.path.basename(f)
                             if segments[-2] == 'listing':
                                 locale = segments[-3]
-                                if dirname in GRAPHIC_NAMES:
-                                    repofilename = dirname + '.' + extension
-                                    dirname = ''
                             elif segments[-4] == 'listings':  # v2.x
                                 locale = segments[-3]
                                 if dirname in tt_graphic_names:
@@ -1154,7 +1124,6 @@ def insert_localized_app_metadata(apps):
 
     metadata/<locale>/
     fastlane/metadata/android/<locale>/
-    <subdir>/fastlane/metadata/android/<locale>/
     src/<buildFlavor>/fastlane/metadata/android/<locale>/
 
     ...as well as the /metadata/<packageName>/<locale> directory.
@@ -1174,7 +1143,7 @@ def insert_localized_app_metadata(apps):
     https://f-droid.org/en/docs/All_About_Descriptions_Graphics_and_Screenshots/#in-the-apps-build-metadata-in-an-fdroiddata-collection
     """
     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
-    sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', '**', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'), recursive=True)
+    sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 
@@ -1190,40 +1159,17 @@ def insert_localized_app_metadata(apps):
             locale = segments[-1]
             destdir = os.path.join('repo', packageName, locale)
 
-            builds = apps.get(packageName, {}).get('Builds', [])
+            # flavours specified in build receipt
-            found_in_subdir = (
+            build_flavours = ""
-                builds
-                and len(segments) > 6
-                and segments[-4] == "fastlane"
-                and segments[-3] == "metadata"
-                and segments[-2] == "android"
-                and '/'.join(segments[2:-4]) == builds[-1].get('subdir')
-            )
-
-            # flavors specified in build receipt
-            build_flavors = []
-            if builds and 'gradle' in builds[-1] and builds[-1]['gradle'] != ['yes']:
-                build_flavors = common.calculate_gradle_flavor_combination(
-                    builds[-1]['gradle']
-                )
-            found_in_flavor = (
-                len(segments) > 7
-                and segments[2] == 'src'
-                and segments[4] == "fastlane"
-                and segments[3] in build_flavors
-            )
-
             if (
-                not found_in_subdir
+                apps[packageName]
-                and not found_in_flavor
+                and len(apps[packageName].get('Builds', [])) > 0
-                and segments[0] == 'build'
+                and 'gradle' in apps[packageName]['Builds'][-1]
-                and segments[2] not in ('metadata', 'fastlane')
             ):
-                logging.debug(
+                build_flavours = apps[packageName]['Builds'][-1]['gradle']
-                    'Not scanning "{dir}" with unknown subdir or gradle flavor "{value}"'.format(
+
-                        dir=os.path.relpath(root), value=segments[3]
+            if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
-                    )
+                logging.debug("ignoring due to wrong flavour")
-                )
                 continue
 
             for f in files:
@@ -1261,7 +1207,9 @@ def insert_localized_app_metadata(apps):
                     try:
                         versionCode = int(base)
                         locale = segments[-2]
-                        if versionCode in [b["versionCode"] for b in builds]:
+                        if versionCode in [
+                            a["versionCode"] for a in apps[packageName]["Builds"]
+                        ]:
                             _set_localized_text_entry(
                                 apps[packageName],
                                 locale,
@@ -1486,18 +1434,19 @@ def insert_localized_ios_app_metadata(apps_with_packages):
         fdroidserver.update.copy_ios_screenshots_to_repo(screenshots, package_name)
 
         # lookup icons, copy them and put them into app
-        icon_src = _get_ipa_icon(Path('build') / package_name)
+        icon_path = _get_ipa_icon(Path('build') / package_name)
         icon_dest = Path('repo') / package_name / 'icon.png'  # for now just assume png
+        icon_stat = os.stat(icon_path)
         app['iconv2'] = {
             DEFAULT_LOCALE: {
                 'name': str(icon_dest).lstrip('repo'),
                 'sha256': common.sha256sum(icon_dest),
-                'size': os.path.getsize(icon_src),
+                'size': icon_stat.st_size,
             }
         }
-        if not icon_dest.exists() or not filecmp.cmp(icon_src, icon_dest):
+        if not icon_dest.exists():
             icon_dest.parent.mkdir(parents=True, exist_ok=True)
-            shutil.copy2(icon_src, icon_dest)
+            shutil.copy(icon_path, icon_dest)
 
 
 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
@@ -1576,10 +1525,8 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
                 repo_file['packageName'] = m.group(1)
                 repo_file['versionCode'] = int(m.group(2))
             srcfilename = name + b'_src.tar.gz'
-            srcpath = os.path.join(repodir, srcfilename)
+            if os.path.exists(os.path.join(repodir, srcfilename)):
-            if os.path.exists(srcpath):
                 repo_file['srcname'] = srcfilename.decode()
-                repo_file['srcnameSha256'] = common.sha256sum(srcpath.decode())
             repo_file['size'] = stat.st_size
 
             apkcache[name_utf8] = repo_file
@@ -1592,9 +1539,8 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
             default_date_param = None
 
         # Record in knownapks, getting the added date at the same time..
-        added = knownapks.recordapk(
+        added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
-            repo_file['apkName'], default_date=default_date_param
+                                    default_date=default_date_param)
-        )
         if added:
             repo_file['added'] = added
 
@@ -1801,7 +1747,6 @@ def scan_apk_androguard(apk, apkfile):
 
     xml = apkobject.get_android_manifest_xml()
     androidmanifest_xml = apkobject.xml['AndroidManifest.xml']
-
     if len(xml.nsmap) > 0:
         # one of them surely will be the Android one, or its corrupt
         xmlns = common.XMLNS_ANDROID
@@ -1811,12 +1756,8 @@ def scan_apk_androguard(apk, apkfile):
         xmlns = '{}'
 
     vcstr = androidmanifest_xml.get(xmlns + 'versionCode')
-    logging.debug("Version Code: %r (%s)" % (vcstr, apkfile))
 
-    if not vcstr:
+    if vcstr.startswith('0x'):
-        raise NoVersionCodeException(_("APK file {path} does not have a version code "
-                                       "in its manifest").format(path=apkfile))
-    elif vcstr.startswith('0x'):
         apk['versionCode'] = int(vcstr, 16)
     else:
         apk['versionCode'] = int(vcstr)
@@ -1909,7 +1850,7 @@ def scan_apk_androguard(apk, apkfile):
 
 
 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
-                allow_disabled_algorithms=False, archive_bad_sig=False, apps=None, cache_timestamp=0):
+                allow_disabled_algorithms=False, archive_bad_sig=False, apps=None):
     """Process the apk with the given filename in the given repo directory.
 
     This also extracts the icons.
@@ -1931,8 +1872,6 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
       disabled algorithms in the signature (e.g. MD5)
     archive_bad_sig
       move APKs with a bad signature to the archive
-    cache_timestamp
-      the timestamp of the cache file
 
     Returns
     -------
@@ -1946,8 +1885,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
     usecache = False
     if apkfilename in apkcache:
         apk = apkcache[apkfilename]
-        stat = os.stat(apkfile)
+        if apk.get('hash') == common.sha256sum(apkfile):
-        if apk.get('size') == stat.st_size and stat.st_mtime < cache_timestamp:
             logging.debug(_("Reading {apkfilename} from cache")
                           .format(apkfilename=apkfilename))
             usecache = True
@@ -1964,10 +1902,6 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
                             .format(apkfilename=apkfilename))
             return True, None, False
-        except NoVersionCodeException:
-            logging.warning(_("Skipping '{apkfilename}' without versionCode!")
-                            .format(apkfilename=apkfilename))
-            return True, None, False
 
         if apps:
             if apk['packageName'] in apps:
@@ -2006,10 +1940,8 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
 
         apk['apkName'] = apkfilename
         srcfilename = apkfilename[:-4] + "_src.tar.gz"
-        srcpath = os.path.join(repodir, srcfilename)
+        if os.path.exists(os.path.join(repodir, srcfilename)):
-        if os.path.exists(srcpath):
             apk['srcname'] = srcfilename
-            apk['srcnameSha256'] = common.sha256sum(srcpath)
 
         # verify the jar signature is correct, allow deprecated
         # algorithms only if the APK is in the archive.
@@ -2056,12 +1988,13 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
 
         if use_date_from_apk:
-            default_date_param = datetime.fromtimestamp(os.stat(apkfile).st_mtime, tz=timezone.utc)
+            default_date_param = datetime.fromtimestamp(os.stat(apkfile).st_mtime)
         else:
             default_date_param = None
 
         # Record in known apks, getting the added date at the same time..
-        added = knownapks.recordapk(apk['apkName'], default_date=default_date_param)
+        added = knownapks.recordapk(apk['apkName'], apk['packageName'],
+                                    default_date=default_date_param)
         if added:
             apk['added'] = added
 
@@ -2071,7 +2004,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
     return False, apk, cachechanged
 
 
-def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False, apps=None, cache_timestamp=0):
+def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False, apps=None):
     """Process the apks in the given repo directory.
 
     This also extracts the icons.
@@ -2086,8 +2019,6 @@ def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False, apps=Non
      b known apks info
     use_date_from_apk
       use date from APK (instead of current date) for newly added APKs
-    cache_timestamp
-      the timestamp of the cache file
 
     Returns
     -------
@@ -2109,7 +2040,7 @@ def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False, apps=Non
         apkfilename = apkfile[len(repodir) + 1:]
         ada = disabled_algorithms_allowed()
         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
-                                             use_date_from_apk, ada, True, apps, cache_timestamp)
+                                             use_date_from_apk, ada, True, apps)
         if skip:
             continue
         apks.append(apk)
@@ -2322,7 +2253,7 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi
                         continue
                 apkList.append(apk)
 
-        # Sort the apk list by versionCode. First is highest/newest.
+        # Sort the apk list by version code. First is highest/newest.
         sorted_list = sorted(apkList, key=lambda apk: apk['versionCode'], reverse=True)
         if currentVersionApk:
             # Insert apk which corresponds to currentVersion at the front
@@ -2330,7 +2261,13 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi
         return sorted_list
 
     for appid, app in apps.items():
-        keepversions = common.calculate_archive_policy(app, defaultkeepversions)
+
+        if app.get('ArchivePolicy') is not None:
+            keepversions = app['ArchivePolicy']
+        else:
+            keepversions = defaultkeepversions
+            if app.get('VercodeOperation'):
+                keepversions *= len(app['VercodeOperation'])
             if common.metadata_find_developer_signing_files(appid, app['CurrentVersionCode']):
                 keepversions *= 2
 
@@ -2468,12 +2405,12 @@ def create_metadata_from_template(apk):
 def read_added_date_from_all_apks(apps, apks):
     """No summary.
 
-    Added dates come from the repo/index-v2.json file but are
+    Added dates come from the stats/known_apks.txt file but are
     read when scanning apks and thus need to be applied form apk
     level to app level for _all_ apps and not only from non-archived
     ones
 
-    TODO: read the added dates directly from index-v2.json instead of
+    TODO: read the added dates directly from known_apks.txt instead of
           going through apks that way it also works for for repos that
           don't keep an archive of apks.
     """
@@ -2641,7 +2578,7 @@ def main():
     metadata.warnings_action = options.W
 
     config = common.read_config()
-    status_output = common.setup_status_output(start_timestamp)
+    common.setup_status_output(start_timestamp)
 
     if not (('jarsigner' in config or 'apksigner' in config)
             and 'keytool' in config):
@@ -2701,18 +2638,14 @@ def main():
 
     # Get APK cache
     apkcache = get_cache()
-    cache_timestamp = get_cache_mtime()
 
     # Delete builds for disabled apps
-    output_status_stage(status_output, 'delete_disabled_builds')
     delete_disabled_builds(apps, apkcache, repodirs)
 
     # Scan all apks in the main repo
-    output_status_stage(status_output, 'process_apks')
     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks,
-                                      options.use_date_from_apk, apps, cache_timestamp)
+                                      options.use_date_from_apk, apps)
 
-    output_status_stage(status_output, 'scan_repo_files')
     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
                                            options.use_date_from_apk)
     cachechanged = cachechanged or fcachechanged
@@ -2722,23 +2655,10 @@ def main():
     cachechanged = cachechanged or icachechanged
     apks += ipas
 
-    output_status_stage(status_output, 'remove_apks')
     appid_has_apks = set()
     appid_has_repo_files = set()
-    sha256_has_files = collections.defaultdict(list)
-    errors = 0
     remove_apks = []
     for apk in apks:
-        sha256 = apk['hash']
-        if sha256 in sha256_has_files:
-            errors += 1
-            for path2 in sha256_has_files[sha256]:
-                logging.error(
-                    _('{path1} is a duplicate of {path2}, remove one!').format(
-                        path1=apk["apkName"], path2=path2
-                    )
-                )
-        sha256_has_files[sha256].append(apk['apkName'])
         to_remove = get_apks_without_allowed_signatures(apps.get(apk['packageName']), apk)
         if to_remove:
             remove_apks.append(apk)
@@ -2781,22 +2701,19 @@ def main():
     for apk in remove_apks:
         apks.remove(apk)
 
+    mismatch_errors = ''
     for appid in appid_has_apks:
         if appid in appid_has_repo_files:
             appid_files = ', '.join(glob.glob(os.path.join('repo', appid + '_[0-9]*.*')))
-            errors += 1
+            mismatch_errors += (_('{appid} has both APKs and files: {files}')
-            logging.error(
+                                .format(appid=appid, files=appid_files)) + '\n'
-                _('{appid} has both APKs and files: {files}').format(
+    if mismatch_errors:
-                    appid=appid, files=appid_files
+        raise FDroidException(mismatch_errors)
-                )
-            )
-    if errors:
-        sys.exit(errors)
 
     # Scan the archive repo for apks as well
     if len(repodirs) > 1:
         archapks, cc = process_apks(apkcache, repodirs[1], knownapks,
-                                    options.use_date_from_apk, apps, cache_timestamp)
+                                    options.use_date_from_apk, apps)
         if cc:
             cachechanged = True
     else:
@@ -2805,25 +2722,18 @@ def main():
     if cachechanged:
         write_cache(apkcache)
 
-    output_status_stage(status_output, 'read_added_date_from_all_apks')
     # The added date currently comes from the oldest apk which might be in the archive.
     # So we need this populated at app level before continuing with only processing /repo
     # or /archive
     read_added_date_from_all_apks(apps, apks + archapks)
 
     if len(repodirs) > 1:
-        output_status_stage(status_output, 'archive_old_apks archive')
         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
-        output_status_stage(status_output, 'prepare_apps archive')
         archived_apps = prepare_apps(apps, archapks, repodirs[1])
-        output_status_stage(status_output, 'index.make archive')
         fdroidserver.index.make(archived_apps, archapks, repodirs[1], True)
 
-    output_status_stage(status_output, 'prepare_apps repo')
     repoapps = prepare_apps(apps, apks, repodirs[0])
 
-    output_status_stage(status_output, 'index.make repo')
-
     # APKs are placed into multiple repos based on the app package, providing
     # per-app subscription feeds for nightly builds and things like it
     if config['per_app_repos']:
@@ -2844,10 +2754,13 @@ def main():
     git_remote = config.get('binary_transparency_remote')
     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
         from . import btlog
-        output_status_stage(status_output, 'make_binary_transparency_log')
         btlog.make_binary_transparency_log(repodirs)
 
-    status_update_json(status_output, apps, apks + archapks)
+    if config['update_stats']:
+        # Update known apks info...
+        knownapks.writeifchanged()
+
+    status_update_json(apps, apks + archapks)
 
     logging.info(_("Finished"))
 

fdroidserver/verify.py

@@ -16,17 +16,18 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import sys
+import os
 import glob
 import json
 import logging
-import os
+import requests
-import sys
 from argparse import ArgumentParser
 from collections import OrderedDict
 
-import requests
+from . import _
-
+from . import common
-from . import _, common, net
+from . import net
 from .exception import FDroidException
 
 config = None
@@ -57,8 +58,8 @@ def _add_diffoscope_info(d):
         ]
         d['diffoscope']['External-Tools-Required'] = external_tools
 
-        from diffoscope.external_tools import EXTERNAL_TOOLS
         from diffoscope.tools import OS_NAMES, get_current_os
+        from diffoscope.external_tools import EXTERNAL_TOOLS
 
         current_os = get_current_os()
         os_list = [current_os] if (current_os in OS_NAMES) else iter(OS_NAMES)
@@ -79,30 +80,6 @@ def _add_diffoscope_info(d):
         pass
 
 
-def get_verified_json(path):
-    """Get the full collection of reports that is written out to verified.json."""
-    if os.path.exists(path):
-        try:
-            with open(path) as fp:
-                return json.load(fp)
-        except Exception as e:
-            logging.info(f'{path}: {e}')
-
-    data = OrderedDict()
-    data['packages'] = OrderedDict()
-
-    for f in glob.glob(os.path.join(os.path.dirname(path), '*.apk.json')):
-        with open(f) as fp:
-            reports = json.load(fp)
-        for report in reports.values():
-            packageName = report['local']['packageName']
-            if packageName not in data['packages']:
-                data['packages'][packageName] = []
-            data['packages'][packageName].append(report)
-
-    return data
-
-
 def write_json_report(url, remote_apk, unsigned_apk, compare_result):
     """Write out the results of the verify run to JSON.
 
@@ -143,30 +120,14 @@ def write_json_report(url, remote_apk, unsigned_apk, compare_result):
     with open(jsonfile, 'w') as fp:
         json.dump(data, fp, sort_keys=True)
 
-    appid, version_code = os.path.basename(unsigned_apk[:-4]).rsplit('_', 1)
-    appid_base = unsigned_apk.rsplit('_', 1)[0]
-    apkReports = sorted(
-        glob.glob(f'{appid_base}_[0-9]*.json'),  # don't include <appid>.json
-        key=lambda s: int(s[:-9].rsplit('_', 1)[1]),  # numeric sort by versionCode
-    )
-    with open(apkReports[-1]) as fp:
-        reports = json.load(fp)
-    appid_output = {'apkReports': apkReports}
-    most_recent = 0
-    for report_time, run in reports.items():
-        if float(report_time) > most_recent:
-            most_recent = float(report_time)
-            appid_output['lastRunVerified'] = run['verified']
-    with open(f'{appid_base}.json', 'w') as fp:
-        json.dump(appid_output, fp, cls=common.Encoder, sort_keys=True)
-
     if output['verified']:
-        write_verified_json(output)
-
-
-def write_verified_json(output):
         jsonfile = 'unsigned/verified.json'
-    data = get_verified_json(jsonfile)
+        if os.path.exists(jsonfile):
+            with open(jsonfile) as fp:
+                data = json.load(fp)
+        else:
+            data = OrderedDict()
+            data['packages'] = OrderedDict()
         packageName = output['local']['packageName']
 
         if packageName not in data['packages']:
@@ -196,12 +157,6 @@ def main():
         nargs='*',
         help=_("application ID with optional versionCode in the form APPID[:VERCODE]"),
     )
-    parser.add_argument(
-        "--clean-up-verified",
-        action="store_true",
-        default=False,
-        help=_("Remove source tarball and any APKs if successfully verified."),
-    )
     parser.add_argument(
         "--reuse-remote-apk",
         action="store_true",
@@ -269,26 +224,12 @@ def main():
                         ) from e
 
             unsigned_apk = os.path.join(unsigned_dir, apkfilename)
-            compare_result = common.verify_apks(
+            compare_result = common.verify_apks(remote_apk, unsigned_apk, tmp_dir)
-                remote_apk,
-                unsigned_apk,
-                tmp_dir,
-                clean_up_verified=options.clean_up_verified,
-            )
             if options.output_json:
                 write_json_report(url, remote_apk, unsigned_apk, compare_result)
             if compare_result:
                 raise FDroidException(compare_result)
 
-            if options.clean_up_verified:
-                src_tarball = os.path.join(
-                    unsigned_dir, common.get_src_tarball_name(appid, vercode)
-                )
-                for f in (remote_apk, unsigned_apk, src_tarball):
-                    if os.path.exists(f):
-                        logging.info(f"...cleaned up {f} after successful verification")
-                        os.remove(f)
-
             logging.info("...successfully verified")
             verified += 1
 

fdroidserver/vmtools.py

@@ -16,17 +16,17 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import json
+from os.path import isdir, isfile, basename, abspath, expanduser
-import logging
 import os
+import json
 import shutil
 import subprocess
 import textwrap
-import threading
+import logging
-from os.path import abspath, basename, expanduser, isdir, isfile
-
 from .common import FDroidException
 
+import threading
+
 lock = threading.Lock()
 
 

gradlew-fdroid

@@ -0,0 +1,302 @@
+#!/bin/bash
+
+bindir="$(dirname $0)"
+basedir="$(dirname $bindir)"
+# Check if GRADLE_VERSION_DIR/CACHEDIR is set from environment
+if [ -z "$GRADLE_VERSION_DIR" ]; then
+    gradle_version_dir="${basedir}/versions"
+else
+    gradle_version_dir="$GRADLE_VERSION_DIR"
+fi
+BUILDSERVER_CACHEDIR=/vagrant/cache
+if [ -n "$CACHEDIR" ]; then
+    cachedir="$CACHEDIR"
+elif [ -d $BUILDSERVER_CACHEDIR ]; then
+    cachedir=$BUILDSERVER_CACHEDIR
+fi
+args=("$@")
+
+run_gradle() {
+    if [ ! -d "${gradle_version_dir}/${v_found}" ]; then
+        download_gradle ${v_found}
+    fi
+    # shellcheck disable=SC2145
+    echo "Running ${gradle_version_dir}/${v_found}/bin/gradle ${args[@]}"
+    "${gradle_version_dir}/${v_found}/bin/gradle" "${args[@]}"
+    exit $?
+}
+
+download_gradle() {
+    URL="https://downloads.gradle.org/distributions/gradle-${1}-bin.zip"
+    shasum=$(get_sha $1)
+    if [ $? != 0 ]; then
+        echo "No hash for gradle version $1! Exiting..."
+        exit 1
+    fi
+    if [ -n "${cachedir}" ] && [ -e "${cachedir}/gradle-$1-bin.zip" ]; then
+        echo "Using cached ${cachedir}/gradle-$1-bin.zip ..."
+        gradle_zip="${cachedir}/gradle-$1-bin.zip"
+    else
+        echo "Downloading missing gradle version $1"
+        echo cachedir $cachedir
+        if [[ -n "${cachedir}" && ! -d "${cachedir}" ]]; then
+            mkdir -p "${cachedir}"
+        fi
+        if [[ -n "${cachedir}" && -d "${cachedir}" && -w "${cachedir}" ]]; then
+            tmpdir="${cachedir}"
+        else
+            tmpdir=$(mktemp -d)
+        fi
+        curl -o "${tmpdir}/gradle-$1-bin.zip" --silent --fail --show-error --location --retry 3 --retry-all-errors "${URL}"
+        gradle_zip="${tmpdir}/gradle-$1-bin.zip"
+    fi
+    echo "${shasum} ${gradle_zip}" | sha256sum -c -
+    if [ $? != 0 ]; then
+        echo "gradle download checksum mismatch! Exiting..."
+        exit 1
+    fi
+    mkdir -p "${gradle_version_dir}/"
+    unzip -q -d "${gradle_version_dir}" "${gradle_zip}"
+    mv "${gradle_version_dir}/gradle-$1" "${gradle_version_dir}/${v_found}"
+}
+
+get_sha() {
+    case $1 in
+        '0.7')    echo '4e354fcb0d5c0b0e7789cd6ee900456edaf993f6dd890c4a1c217d90d2a6a6ad' ;;
+        '0.8')    echo '940e623ea98e40ea9ad398770a6ebb91a61c0869d394dda81aa86b0f4f0025e7' ;;
+        '0.9')    echo '994e46d4b467254a0f25ce92b602618331b9b3ac8b32a094fd84ff0e0ceec135' ;;
+        '0.9.1')  echo '5d48cba95db031ec109ae9ab60561e960b6507046036e8191aa78572ec27e2a5' ;;
+        '0.9.2')  echo 'f94d7642348c558fc05ab5fd6fb947fb1ed8fed5931ddb73dd04fb0de22d669b' ;;
+        '1.0')    echo '894bca0360a7e2040815096788f118a2dd106ff6694221b274efb9c32bce0384' ;;
+        '1.1')    echo '552c1fc9f3a1b9668b79cc447370f0263e664ffb6d5c6e1c21e77ce0c8a20d4c' ;;
+        '1.2')    echo 'eb53da3704d24cabb7565f34a3bf16bcd863c4b0c139917606fb15d4f27c7bdf' ;;
+        '1.3')    echo 'ada68561efbb9f1cae0f9063974cbde15c180351a2f92bc2f1106e39ddcae5ba' ;;
+        '1.4')    echo 'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e' ;;
+        '1.5')    echo 'a5511a0659caa47d9d74fd2844c9da43157d2f78e63a0223c6289d88f5aaecbe' ;;
+        '1.6')    echo 'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4' ;;
+        '1.7')    echo '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30' ;;
+        '1.8')    echo 'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703' ;;
+        '1.9')    echo '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9' ;;
+        '1.10')   echo '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe' ;;
+        '1.11')   echo '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091' ;;
+        '1.12')   echo '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea' ;;
+        '2.0')    echo 'a1eb880c8755333c4d33c4351b269bebe517002532d3142c0b6164c9e8c081c3' ;;
+        '2.1')    echo '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878' ;;
+        '2.2')    echo '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb' ;;
+        '2.2.1')  echo '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418' ;;
+        '2.3')    echo '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774' ;;
+        '2.4')    echo 'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f' ;;
+        '2.5')    echo '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555' ;;
+        '2.6')    echo '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83' ;;
+        '2.7')    echo 'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06' ;;
+        '2.8')    echo 'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb' ;;
+        '2.9')    echo 'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f' ;;
+        '2.10')   echo '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13' ;;
+        '2.11')   echo '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6' ;;
+        '2.12')   echo 'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c' ;;
+        '2.13')   echo '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664' ;;
+        '2.14')   echo '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5' ;;
+        '2.14.1') echo 'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39' ;;
+        '3.0')    echo '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80' ;;
+        '3.1')    echo 'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc' ;;
+        '3.2')    echo '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935' ;;
+        '3.2.1')  echo '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689' ;;
+        '3.3')    echo 'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292' ;;
+        '3.4')    echo '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205' ;;
+        '3.4.1')  echo 'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2' ;;
+        '3.5')    echo '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110' ;;
+        '3.5.1')  echo '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8' ;;
+        '4.0')    echo '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe' ;;
+        '4.0.1')  echo 'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4' ;;
+        '4.0.2')  echo '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22' ;;
+        '4.1')    echo 'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43' ;;
+        '4.2')    echo '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae' ;;
+        '4.2.1')  echo 'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0' ;;
+        '4.3')    echo '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c' ;;
+        '4.3.1')  echo '15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7' ;;
+        '4.4')    echo 'fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f' ;;
+        '4.4.1')  echo 'e7cf7d1853dfc30c1c44f571d3919eeeedef002823b66b6a988d27e919686389' ;;
+        '4.5')    echo '03f2a43a314ff0fb843a85ef68078e06d181c4549c1e5fb983f289382b59b5e3' ;;
+        '4.5.1')  echo '3e2ea0d8b96605b7c528768f646e0975bd9822f06df1f04a64fd279b1a17805e' ;;
+        '4.6')    echo '98bd5fd2b30e070517e03c51cbb32beee3e2ee1a84003a5a5d748996d4b1b915' ;;
+        '4.7')    echo 'fca5087dc8b50c64655c000989635664a73b11b9bd3703c7d6cabd31b7dcdb04' ;;
+        '4.8')    echo 'f3e29692a8faa94eb0b02ebf36fa263a642b3ae8694ef806c45c345b8683f1ba' ;;
+        '4.8.1')  echo 'af334d994b5e69e439ab55b5d2b7d086da5ea6763d78054f49f147b06370ed71' ;;
+        '4.9')    echo 'e66e69dce8173dd2004b39ba93586a184628bc6c28461bc771d6835f7f9b0d28' ;;
+        '4.10')   echo '248cfd92104ce12c5431ddb8309cf713fe58de8e330c63176543320022f59f18' ;;
+        '4.10.1') echo 'e53ce3a01cf016b5d294eef20977ad4e3c13e761ac1e475f1ffad4c6141a92bd' ;;
+        '4.10.2') echo 'b49c6da1b2cb67a0caf6c7480630b51c70a11ca2016ff2f555eaeda863143a29' ;;
+        '4.10.3') echo '8626cbf206b4e201ade7b87779090690447054bc93f052954c78480fa6ed186e' ;;
+        '5.0')    echo '6157ac9f3410bc63644625b3b3e9e96c963afd7910ae0697792db57813ee79a6' ;;
+        '5.1')    echo '7506638a380092a0406364c79d6c87d03d23017fc25a5770379d1ce23c3fcd4d' ;;
+        '5.1.1')  echo '4953323605c5d7b89e97d0dc7779e275bccedefcdac090aec123375eae0cc798' ;;
+        '5.2')    echo 'ff322863250159595e93b5a4d17a6f0d21c59a1a0497c1e1cf1d53826485503f' ;;
+        '5.2.1')  echo '748c33ff8d216736723be4037085b8dc342c6a0f309081acf682c9803e407357' ;;
+        '5.3')    echo 'bed2bdd3955be5a09ca7e0201e9d131f194f7f6c466e1795a733733ccfb09f25' ;;
+        '5.3.1')  echo '1c59a17a054e9c82f0dd881871c9646e943ec4c71dd52ebc6137d17f82337436' ;;
+        '5.4')    echo 'c8c17574245ecee9ed7fe4f6b593b696d1692d1adbfef425bef9b333e3a0e8de' ;;
+        '5.4.1')  echo '7bdbad1e4f54f13c8a78abc00c26d44dd8709d4aedb704d913fb1bb78ac025dc' ;;
+        '5.5')    echo '8d78b2ed63e7f07ad169c1186d119761c4773e681f332cfe1901045b1b0141bc' ;;
+        '5.5.1')  echo '222a03fcf2fcaf3691767ce9549f78ebd4a77e73f9e23a396899fb70b420cd00' ;;
+        '5.6')    echo '15c02ef5dd3631ec02ac52e8725703e0285d9a7eecbf4e5939aa9e924604d01d' ;;
+        '5.6.1')  echo '0986244820e4a35d32d91df2ec4b768b5ba5d6c8246753794f85159f9963ec12' ;;
+        '5.6.2')  echo '32fce6628848f799b0ad3205ae8db67d0d828c10ffe62b748a7c0d9f4a5d9ee0' ;;
+        '5.6.3')  echo '60a6d8f687e3e7a4bc901cc6bc3db190efae0f02f0cc697e323e0f9336f224a3' ;;
+        '5.6.4')  echo '1f3067073041bc44554d0efe5d402a33bc3d3c93cc39ab684f308586d732a80d' ;;
+        '6.0')    echo '5a3578b9f0bb162f5e08cf119f447dfb8fa950cedebb4d2a977e912a11a74b91' ;;
+        '6.0.1')  echo 'd364b7098b9f2e58579a3603dc0a12a1991353ac58ed339316e6762b21efba44' ;;
+        '6.1')    echo 'd0c43d14e1c70a48b82442f435d06186351a2d290d72afd5b8866f15e6d7038a' ;;
+        '6.1.1')  echo '9d94e6e4a28ad328072ef6e56bce79a810494ae756751fdcedffdeaf27c093b1' ;;
+        '6.2')    echo 'b93a5f30d01195ec201e240f029c8b42d59c24086b8d1864112c83558e23cf8a' ;;
+        '6.2.1')  echo 'a68ca7ba57f3404c3f6fc1f70a02d3a7d78652e6b46bbfaff83fc9a17168c279' ;;
+        '6.2.2')  echo '0f6ba231b986276d8221d7a870b4d98e0df76e6daf1f42e7c0baec5032fb7d17' ;;
+        '6.3')    echo '038794feef1f4745c6347107b6726279d1c824f3fc634b60f86ace1e9fbd1768' ;;
+        '6.4')    echo 'b888659f637887e759749f6226ddfcb1cb04f828c58c41279de73c463fdbacc9' ;;
+        '6.4.1')  echo 'e58cdff0cee6d9b422dcd08ebeb3177bc44eaa09bd9a2e838ff74c408fe1cbcd' ;;
+        '6.5')    echo '23e7d37e9bb4f8dabb8a3ea7fdee9dd0428b9b1a71d298aefd65b11dccea220f' ;;
+        '6.5.1')  echo '50a7d30529fa939721fe9268a0205142f3f2302bcac5fb45b27a3902e58db54a' ;;
+        '6.6')    echo 'e6f83508f0970452f56197f610d13c5f593baaf43c0e3c6a571e5967be754025' ;;
+        '6.6.1')  echo '7873ed5287f47ca03549ab8dcb6dc877ac7f0e3d7b1eb12685161d10080910ac' ;;
+        '6.7')    echo '8ad57759019a9233dc7dc4d1a530cefe109dc122000d57f7e623f8cf4ba9dfc4' ;;
+        '6.7.1')  echo '3239b5ed86c3838a37d983ac100573f64c1f3fd8e1eb6c89fa5f9529b5ec091d' ;;
+        '6.8')    echo 'e2774e6fb77c43657decde25542dea710aafd78c4022d19b196e7e78d79d8c6c' ;;
+        '6.8.1')  echo 'fd591a34af7385730970399f473afabdb8b28d57fd97d6625c388d090039d6fd' ;;
+        '6.8.2')  echo '8de6efc274ab52332a9c820366dd5cf5fc9d35ec7078fd70c8ec6913431ee610' ;;
+        '6.8.3')  echo '7faa7198769f872826c8ef4f1450f839ec27f0b4d5d1e51bade63667cbccd205' ;;
+        '6.9')    echo '765442b8069c6bee2ea70713861c027587591c6b1df2c857a23361512560894e' ;;
+        '6.9.1')  echo '8c12154228a502b784f451179846e518733cf856efc7d45b2e6691012977b2fe' ;;
+        '6.9.2')  echo '8b356fd8702d5ffa2e066ed0be45a023a779bba4dd1a68fd11bc2a6bdc981e8f' ;;
+        '6.9.3')  echo 'dcf350b8ae1aa192fc299aed6efc77b43825d4fedb224c94118ae7faf5fb035d' ;;
+        '6.9.4')  echo '3e240228538de9f18772a574e99a0ba959e83d6ef351014381acd9631781389a' ;;
+        '7.0')    echo 'eb8b89184261025b0430f5b2233701ff1377f96da1ef5e278af6ae8bac5cc305' ;;
+        '7.0.1')  echo 'dccda8aa069563c8ba2f6cdfd0777df0e34a5b4d15138ca8b9757e94f4e8a8cb' ;;
+        '7.0.2')  echo '0e46229820205440b48a5501122002842b82886e76af35f0f3a069243dca4b3c' ;;
+        '7.1')    echo '2debee19271e1b82c6e41137d78e44e6e841035230a1a169ca47fd3fb09ed87b' ;;
+        '7.1.1')  echo 'bf8b869948901d422e9bb7d1fa61da6a6e19411baa7ad6ee929073df85d6365d' ;;
+        '7.2')    echo 'f581709a9c35e9cb92e16f585d2c4bc99b2b1a5f85d2badbd3dc6bff59e1e6dd' ;;
+        '7.3')    echo 'de8f52ad49bdc759164f72439a3bf56ddb1589c4cde802d3cec7d6ad0e0ee410' ;;
+        '7.3.1')  echo '9afb3ca688fc12c761a0e9e4321e4d24e977a4a8916c8a768b1fe05ddb4d6b66' ;;
+        '7.3.2')  echo '23b89f8eac363f5f4b8336e0530c7295c55b728a9caa5268fdd4a532610d5392' ;;
+        '7.3.3')  echo 'b586e04868a22fd817c8971330fec37e298f3242eb85c374181b12d637f80302' ;;
+        '7.4')    echo '8cc27038d5dbd815759851ba53e70cf62e481b87494cc97cfd97982ada5ba634' ;;
+        '7.4.1')  echo 'e5444a57cda4a95f90b0c9446a9e1b47d3d7f69057765bfb54bd4f482542d548' ;;
+        '7.4.2')  echo '29e49b10984e585d8118b7d0bc452f944e386458df27371b49b4ac1dec4b7fda' ;;
+        '7.5')    echo 'cb87f222c5585bd46838ad4db78463a5c5f3d336e5e2b98dc7c0c586527351c2' ;;
+        '7.5.1')  echo 'f6b8596b10cce501591e92f229816aa4046424f3b24d771751b06779d58c8ec4' ;;
+        '7.6')    echo '7ba68c54029790ab444b39d7e293d3236b2632631fb5f2e012bb28b4ff669e4b' ;;
+        '7.6.1')  echo '6147605a23b4eff6c334927a86ff3508cb5d6722cd624c97ded4c2e8640f1f87' ;;
+        '7.6.2')  echo 'a01b6587e15fe7ed120a0ee299c25982a1eee045abd6a9dd5e216b2f628ef9ac' ;;
+        '7.6.3')  echo '740c2e472ee4326c33bf75a5c9f5cd1e69ecf3f9b580f6e236c86d1f3d98cfac' ;;
+        '7.6.4')  echo 'bed1da33cca0f557ab13691c77f38bb67388119e4794d113e051039b80af9bb1' ;;
+        '8.0')    echo '4159b938ec734a8388ce03f52aa8f3c7ed0d31f5438622545de4f83a89b79788' ;;
+        '8.0.1')  echo '1b6b558be93f29438d3df94b7dfee02e794b94d9aca4611a92cdb79b6b88e909' ;;
+        '8.0.2')  echo 'ff7bf6a86f09b9b2c40bb8f48b25fc19cf2b2664fd1d220cd7ab833ec758d0d7' ;;
+        '8.1')    echo 'a62c5f99585dd9e1f95dab7b9415a0e698fa9dd1e6c38537faa81ac078f4d23e' ;;
+        '8.1.1')  echo 'e111cb9948407e26351227dabce49822fb88c37ee72f1d1582a69c68af2e702f' ;;
+        '8.2')    echo '38f66cd6eef217b4c35855bb11ea4e9fbc53594ccccb5fb82dfd317ef8c2c5a3' ;;
+        '8.2.1')  echo '03ec176d388f2aa99defcadc3ac6adf8dd2bce5145a129659537c0874dea5ad1' ;;
+        '8.3')    echo '591855b517fc635b9e04de1d05d5e76ada3f89f5fc76f87978d1b245b4f69225' ;;
+        '8.4')    echo '3e1af3ae886920c3ac87f7a91f816c0c7c436f276a6eefdb3da152100fef72ae' ;;
+        '8.5')    echo '9d926787066a081739e8200858338b4a69e837c3a821a33aca9db09dd4a41026' ;;
+        '8.6')    echo '9631d53cf3e74bfa726893aee1f8994fee4e060c401335946dba2156f440f24c' ;;
+        '8.7')    echo '544c35d6bd849ae8a5ed0bcea39ba677dc40f49df7d1835561582da2009b961d' ;;
+        '8.8')    echo 'a4b4158601f8636cdeeab09bd76afb640030bb5b144aafe261a5e8af027dc612' ;;
+        '8.9')    echo 'd725d707bfabd4dfdc958c624003b3c80accc03f7037b5122c4b1d0ef15cecab' ;;
+        '8.10')   echo '5b9c5eb3f9fc2c94abaea57d90bd78747ca117ddbbf96c859d3741181a12bf2a' ;;
+        '8.10.1') echo '1541fa36599e12857140465f3c91a97409b4512501c26f9631fb113e392c5bd1' ;;
+        *) exit 1
+    esac
+}
+
+contains() {
+    local e
+    for e in $2; do
+        [[ $e == $1 ]] && return 0;
+    done
+    return 1
+}
+
+# key-value pairs of what gradle version (value) each gradle plugin version
+# (key) should accept. plugin versions are actually prefixes and catch sub-
+# versions as well. Pairs are taken from:
+# https://developer.android.com/studio/releases/gradle-plugin#updating-gradle
+d_gradle_plugin_ver_k=(8.4 8.3 8.2 8.1 8.0 7.4 7.3 7.2.0 7.1 7.0 4.2 4.1 4.0 3.6   3.5   3.4   3.3   3.2    3.1 3.0 2.3 2.2 2.1.3  2.1    2.0)
+d_plugin_min_gradle_v=(8.6 8.4 8.2 8.0 8.0 7.5 7.4 7.3.3 7.2 7.0.2 6.7.1 6.5 6.1.1 5.6.4 5.4.1 5.1.1 4.10.1 4.6 4.4 4.1 3.3 2.14.1 2.14.1 2.12 2.12 2.4 2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4)
+
+# All gradle versions we know about
+plugin_v=(8.10.1 8.10 8.9 8.8 8.7 8.6 8.5 8.4 8.3 8.2.1 8.2 8.1.1 8.1 8.0.2 8.0.1 8.0 7.6.4 7.6.3 7.6.2 7.6.1 7.6 7.5.1 7.5 7.4.2 7.4.1 7.4 7.3.3 7.3.2 7.3.1 7.3 7.2 7.1.1 7.1 7.0.2 7.0.1 7.0 6.9.4 6.9.3 6.9.2 6.9.1 6.9 6.8.3 6.8.2 6.8.1 6.8 6.7.1 6.7 6.6.1 6.6 6.5.1 6.5 6.4.1 6.4 6.3 6.2.2 6.2.1 6.2 6.1.1 6.1 6.0.1 6.0 5.6.4 5.6.3 5.6.2 5.6.1 5.6 5.5.1 5.5 5.4.1 5.4 5.3.1 5.3 5.2.1 5.2 5.1.1 5.1 5.0 4.10.3 4.10.2 4.10.1 4.10 4.9 4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 2.0 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.5 1.4 1.3 1.2 1.1 1.0 0.9.2 0.9.1 0.9 0.8 0.7)
+
+v_all=${plugin_v[@]}
+
+# Earliest file takes priority
+# Last key takes priority if there are duplicates (matching java.util.Properties)
+for f in {.,..}/gradle/wrapper/gradle-wrapper.properties; do
+    [[ -f $f ]] || continue
+    while IFS='' read -r line || [ -n "$line" ]; do
+        line=$(printf "$line" | tr -d '\r') # strip Windows linefeeds
+        if [[ $line == 'distributionUrl='* ]]; then
+            wrapper_ver=${line#*/gradle-}
+            wrapper_ver=${wrapper_ver%-*.zip}
+        fi
+    done < $f
+    [[ -n $wrapper_ver ]] && break
+done
+
+if [[ -n $wrapper_ver ]]; then
+    v_found=$wrapper_ver
+    echo "Found $v_found via distributionUrl"
+    run_gradle
+fi
+
+# Earliest takes priority
+for f in {.,..}/build.gradle{,.kts}; do
+    [[ -f $f ]] || continue
+    while IFS='' read -r line || [ -n "$line" ]; do
+        line=$(printf "$line" | tr -d '\r') # strip Windows linefeeds
+        if [[ -z "$plugin_pver" && $line == *'com.android.tools.build:gradle:'* ]]; then
+            plugin_pver=${line#*[\'\"]com.android.tools.build:gradle:}
+            plugin_pver=${plugin_pver%[\'\"]*}
+        elif [[ -z "$wrapper_ver" && $line == *'gradleVersion = '* ]]; then
+            wrapper_ver=${line#*gradleVersion*=*[\'\"]}
+            wrapper_ver=${wrapper_ver%[\'\"]*}
+        fi
+    done < $f
+done
+
+if [[ -n $wrapper_ver ]]; then
+    v_found=$wrapper_ver
+    echo "Found $v_found via gradleVersion"
+    run_gradle
+fi
+
+if [[ -n $plugin_pver ]]; then
+    i=0
+    match=false
+    for k in "${d_gradle_plugin_ver_k[@]}"; do
+        if [[ $plugin_pver == ${k}* ]]; then
+            plugin_ver=${d_plugin_min_gradle_v[$i]}
+            match=true
+            break
+        fi
+        let i++
+    done
+    if $match; then
+        v_found=$plugin_ver
+        echo "Found $v_found via gradle plugin version $k"
+    fi
+fi
+
+# Find the highest version available
+for v in ${plugin_v[*]}; do
+    if contains $v "${v_all[*]}"; then
+        v_def=$v
+        break
+    fi
+done
+
+if [[ -z $v_found ]]; then
+    echo "No suitable gradle version found - defaulting to $v_def"
+    v_found=$v_def
+fi
+
+run_gradle

hooks/install-hooks.sh

@@ -2,7 +2,7 @@
 #
 # Install all the client hooks
 
-BASE_DIR="$(cd $(dirname $0) || exit; pwd -P)"
+BASE_DIR="$(cd $(dirname $0); pwd -P)"
 HOOK_NAMES="applypatch-msg pre-applypatch post-applypatch pre-commit prepare-commit-msg commit-msg post-commit pre-rebase post-checkout post-merge pre-receive update post-receive post-update pre-auto-gc"
 HOOK_DIR="$(git rev-parse --show-toplevel)/.git/hooks"
 

hooks/pre-commit

@@ -9,9 +9,9 @@ exec 1>&2
 files=`git diff-index --cached HEAD 2>&1 | sed 's/^:.*     //' | uniq | cut -b100-500`
 if [ -z "$files" ]; then
     PY_FILES="fdroid makebuildserver setup.py fdroidserver/*.py examples/*.py tests/*-release-checksums.py"
-    PY_TEST_FILES="tests/test_*.py"
+    PY_TEST_FILES="tests/*.TestCase"
     SH_FILES="hooks/pre-commit"
-    BASH_FILES="jenkins-build-all jenkins-setup-build-environment jenkins-test completion/bash-completion buildserver/provision-*"
+    BASH_FILES="gradlew-fdroid jenkins-build-all jenkins-setup-build-environment jenkins-test completion/bash-completion buildserver/provision-*"
     RB_FILES="buildserver/Vagrantfile"
     YML_FILES=".*.yml .yamllint */*.yml */*.yaml"
 else
@@ -27,16 +27,16 @@ else
     for f in $files; do
         test -e $f || continue
         case $f in
-            test_*.py)
-                PY_TEST_FILES+=" $f"
-                ;;
             *.py)
                 PY_FILES+=" $f"
                 ;;
+            *.TestCase)
+                PY_TEST_FILES+=" $f"
+                ;;
             *.rb)
                 RB_FILES+=" $f"
                 ;;
-            *.yml|*.yaml|.yamllint)
+            *.yml|.*.yml|.yamllint)
                 YML_FILES+=" $f"
                 ;;
             *)
@@ -66,7 +66,7 @@ cmd_exists() {
 }
 
 find_command() {
-	for name in "$@"; do
+	for name in $@; do
 		for suff in "3" "-3" "-python3" ""; do
 			cmd=${name}${suff}
 			if cmd_exists $cmd; then
@@ -91,7 +91,7 @@ if [ "$PY_FILES $PY_TEST_FILES" != " " ]; then
 	err "pyflakes tests failed!"
     fi
     # ignore vendored files
-    if ! $PYDOCSTYLE --match='(?!apksigcopier|looseversion|setup|test_).*\.py' $PY_FILES $PY_TEST_FILES; then
+    if ! $PYDOCSTYLE --match='(?!apksigcopier|looseversion).*\.py' $PY_FILES $PY_TEST_FILES; then
 	err "pydocstyle tests failed!"
     fi
 fi

locale/Makefile

@@ -1,7 +1,8 @@
 
-FILES = $(wildcard ../fdroidserver/*.py) \
+FILES = ../fdroid $(wildcard ../fdroidserver/*.py) \
 	$(wildcard /usr/lib/python3.*/argparse.py) \
-	../fdroid
+	$(wildcard /usr/lib/python3.*/optparse.py) \
+	$(wildcard /usr/lib/python3.*/getopt.py)
 
 # these are the supported languages
 ALL_LINGUAS = $(shell sed -En 's,include locale/([^/]+)/.*,\1,p' ../MANIFEST.in)

locale/POTFILES.in

@@ -0,0 +1,17 @@
+fdroid
+fdroidserver/btlog.py
+fdroidserver/build.py
+fdroidserver/checkupdates.py
+fdroidserver/common.py
+fdroidserver/deploy.py
+fdroidserver/import.py
+fdroidserver/init.py
+fdroidserver/install.py
+fdroidserver/lint.py
+fdroidserver/metadata.py
+fdroidserver/publish.py
+fdroidserver/rewritemeta.py
+fdroidserver/scanner.py
+fdroidserver/stats.py
+fdroidserver/update.py
+fdroidserver/verify.py

locale/pick-complete-translations.py

@@ -5,10 +5,9 @@
 import json
 import os
 import re
+import requests
 import subprocess
 
-import git
-import requests
 
 projectbasedir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 localedir = os.path.join(projectbasedir, 'locale')
@@ -19,79 +18,37 @@ if os.path.exists(cached_file):
     with open(cached_file) as fp:
         data = json.load(fp)
 else:
-    url = 'https://hosted.weblate.org/api/components/f-droid/fdroidserver/statistics/?format=json'
+    url = 'https://hosted.weblate.org/exports/stats/f-droid/fdroidserver/?format=json'
     r = requests.get(url)
     r.raise_for_status()
-    data = r.json()['results']
+    data = r.json()
 
 active = set()
 print('name                               locale   translated approved error-free')
 for locale in sorted(data, key=lambda locale: locale['code']):
-    print(
+    print('%26s' % locale['name'],
-        '%26s' % locale['name'],
           '%8s' % locale['code'],
           '%0.1f%%' % locale['translated_percent'],
           '%0.1f%%' % locale['approved_percent'],
           '%0.1f%%' % (100 - locale['failing_percent']),
-        sep='\t',
+          sep='\t')
-    )
     if locale['translated_percent'] >= 90 and locale['failing'] < 5:
         active.add(locale['code'])
 
 manifest_file = os.path.join(projectbasedir, 'MANIFEST.in')
 with open(manifest_file) as fp:
-    manifest_in = fp.read()
+    for line in fp.readlines():
-for m in re.findall(r'include locale/([^/]+)/LC_MESSAGES/fdroidserver.po', manifest_in):
+        m = re.match(r'include locale/([^/]+)/.*', line)
-    active.add(m)
+        if m:
+            active.add(m.group(1))
 
-repo = git.Repo(projectbasedir)
-weblate = repo.remotes.weblate
-weblate.fetch()
-upstream = repo.remotes.upstream
-upstream.fetch()
-
-if 'merge_weblate' in repo.heads:
-    merge_weblate = repo.heads['merge_weblate']
-    repo.create_tag(
-        'previous_merge_weblate',
-        ref=merge_weblate,
-        message=('Automatically created by %s' % __file__),
-    )
-else:
-    merge_weblate = repo.create_head('merge_weblate')
-merge_weblate.set_commit(upstream.refs.master)
-merge_weblate.checkout()
-
-active = sorted(active)
 manifest_lines = set()
 for locale in active:
-    po_file = f'locale/{locale}/LC_MESSAGES/fdroidserver.po'
+    manifest_lines.add('include locale/%s/LC_MESSAGES/fdroidserver.po\n' % locale)
-    manifest_lines.add(f'include {po_file}\n')
-    for commit in repo.iter_commits(
-        str(weblate.refs.master) + '...' + str(upstream.refs.master),
-        paths=[po_file],
-        max_count=10,
-        reverse=True,
-    ):
-        print(f'{locale}: git cherry-pick', commit)
-        repo.git.cherry_pick(commit)
 
 with open(manifest_file, 'a') as fp:
     for line in manifest_lines:
         if line:
             fp.write(line)
 
-# first filter duplicates
 subprocess.run(['sort', '-u', '-o', manifest_file, manifest_file])
-# then use a stable sort order
-subprocess.run(
-    ['sort', '--ignore-case', '--stable', '-o', manifest_file, manifest_file],
-    env={'LC_ALL': 'C'},
-)
-
-print('\tIf all else fails, try:')
-print('\tgit checkout -B merge_weblate weblate/master')
-print('\tgit rebase -i upstream/master')
-print('\t# select all in editor and cut all commit lines')
-print('\twl-paste | grep -Eo ".* \((%s)\) .*" | wl-copy' % '|'.join(active))
-print('\t# paste into editor, and make rebase\n')