Merge branch 'wire--checksum-to-rclone' into 'master'

deploy: wire up --no-checksum option to rclone functionality

See merge request fdroid/fdroidserver!1705

.bandit

@@ -0,0 +1,3 @@
+[bandit]
+skips: B110,B404,B408,B603,B607,B322
+targets: .

.editorconfig

@@ -0,0 +1,15 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[**.py]
+indent_style = space
+indent_size = 4
+
+[.gitlab-ci.yml]
+indent_style = space
+indent_size = 2

.gitignore

@@ -1,8 +1,79 @@
-/config.py
-/makebs.config.py
 *~
 *.pyc
 *.class
 *.box
+TAGS
+.idea
+.ropeproject/
+
 # files generated by build
-FDroidServer.egg-info/
+/build/
+/dist/
+env/
+ENV/
+/fdroidserver.egg-info/
+pylint.parseable
+/.testfiles/
+README.rst
+/.eggs/
+
+# editor tmp files
+.*.swp
+.ropeproject/
+
+# files generated by tests
+tmp/
+/tests/repo/icons*
+/tests/repo/status
+
+# files used in manual testing
+/config.yml
+/tmp/
+/logs/
+/metadata/
+/makebs.config.py
+makebuildserver.config.py
+/tests/.fdroid.keypass.txt
+/tests/.fdroid.keystorepass.txt
+/tests/.java.security
+/tests/fdroid-icon.png
+/tests/OBBMainOldVersion.apk
+/tests/OBBMainPatchCurrent.apk
+/tests/OBBMainTwoVersions.apk
+/tests/archive/categories.txt
+/tests/archive/diff/[1-9]*.json
+/tests/archive/entry.jar
+/tests/archive/entry.json
+/tests/archive/icons*
+/tests/archive/index-v1.jar
+/tests/archive/index-v1.json
+/tests/archive/index-v2.json
+/tests/archive/index.css
+/tests/archive/index.html
+/tests/archive/index.jar
+/tests/archive/index.png
+/tests/archive/index.xml
+/tests/archive/index_unsigned.jar
+/tests/metadata/org.videolan.vlc/en-US/icon*.png
+/tests/repo/diff/[1-9]*.json
+/tests/repo/index.css
+/tests/repo/index.html
+/tests/repo/index.jar
+/tests/repo/index.png
+/tests/repo/index_unsigned.jar
+/tests/repo/index-v1.jar
+/tests/repo/info.guardianproject.urzip/
+/tests/repo/info.guardianproject.checkey/en-US/phoneScreenshots/checkey-phone.png
+/tests/repo/info.guardianproject.checkey/en-US/phoneScreenshots/checkey.png
+/tests/repo/obb.mainpatch.current/en-US/featureGraphic_ffhLaojxbGAfu9ROe1MJgK5ux8d0OVc6b65nmvOBaTk=.png
+/tests/repo/obb.mainpatch.current/en-US/icon_WI0pkO3LsklrsTAnRr-OQSxkkoMY41lYe2-fAvXLiLg=.png
+/tests/repo/org.videolan.vlc/en-US/icon_yAfSvPRJukZzMMfUzvbYqwaD1XmHXNtiPBtuPVHW-6s=.png
+/tests/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk
+/tests/virustotal/
+/unsigned/
+
+# generated by gettext
+locale/*/LC_MESSAGES/fdroidserver.mo
+
+# sphinx
+public/

.gitlab-ci.yml

@@ -0,0 +1,854 @@
+---
+
+# Use merge request pipelines when a merge request is open for the branch.
+# Use branch pipelines when a merge request is not open for the branch.
+# https://docs.gitlab.com/ci/yaml/workflow/#switch-between-branch-pipelines-and-merge-request-pipelines
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
+      when: never
+    - if: $CI_COMMIT_BRANCH
+
+
+stages:
+  - lint
+  - test  # default for jobs that do not specify stage:
+  - deploy
+
+
+variables:
+  pip: pip3 --timeout 100 --retries 10
+  # speed up git checkout phase
+  GIT_DEPTH: 1
+
+
+# Run the whole test suite in an environment that is like the
+# buildserver guest VM.  This installs python3-babel because that is
+# only used by the test suite, and not needed in the buildserver.
+#
+# Some extra packages are required for this test run that are not
+# provided by the buildserver since they are not needed there:
+# * python3-babel for compiling localization files
+# * gnupg-agent for the full signing setup
+# * python3-clint for fancy progress bars for users
+# * python3-pycountry for linting config/mirrors.yml
+buildserver run-tests:
+  image: registry.gitlab.com/fdroid/fdroidserver:buildserver
+  script:
+    - apt-get update
+    - apt-get install gnupg-agent python3-babel python3-biplist python3-clint python3-pycountry
+    - ./tests/run-tests
+    # make sure that translations do not cause stacktraces
+    - cd $CI_PROJECT_DIR/locale
+    - for locale in *; do
+          test -d $locale || continue;
+          for cmd in `sed -n 's/.*("\(.*\)", *_.*/\1/p' $CI_PROJECT_DIR/fdroid`; do
+              LANGUAGE=$locale $CI_PROJECT_DIR/fdroid $cmd --help > /dev/null;
+          done
+      done
+
+# Test that the parsing of the .yml metadata format didn't change from last
+# released version. This uses the commit ID of the release tags,
+# rather than the release tag itself so that contributor forks do not
+# need to include the tags in them for this test to work.
+#
+# The COMMIT_ID should be bumped after each release, so that the list
+# of sed hacks needed does not continuously grow.
+metadata_v0:
+  image: registry.gitlab.com/fdroid/fdroidserver:buildserver
+  variables:
+    GIT_DEPTH: 1000
+    RELEASE_COMMIT_ID: 50aa35772b058e76b950c01e16019c072c191b73  # after switching to `git rev-parse`
+  script:
+    - git fetch https://gitlab.com/fdroid/fdroidserver.git $RELEASE_COMMIT_ID
+    - cd tests
+    - export GITCOMMIT=$(git rev-parse HEAD)
+    - git checkout $RELEASE_COMMIT_ID
+    - cd ..
+    - git clone --depth 1 https://gitlab.com/fdroid/fdroiddata.git
+    - rm -f fdroiddata/config.yml  # ignore config for this test
+    - cd fdroiddata
+    - ../tests/dump_internal_metadata_format.py
+    - cd ..
+    - git reset --hard
+    - git checkout $GITCOMMIT
+    - cd fdroiddata
+    - ../tests/dump_internal_metadata_format.py
+    - sed -i
+          -e '/ArchivePolicy:/d'
+          -e '/FlattrID:/d'
+          -e '/RequiresRoot:/d'
+          metadata/dump_*/*.yaml
+    - diff -uw metadata/dump_*
+
+.apt-template: &apt-template
+  variables:
+    DEBIAN_FRONTEND: noninteractive
+    LANG: C.UTF-8
+  before_script:
+    - echo Etc/UTC > /etc/timezone
+    - echo 'APT::Install-Recommends "0";'
+           'APT::Install-Suggests "0";'
+           'APT::Get::Assume-Yes "true";'
+           'Acquire::Retries "20";'
+           'Dpkg::Use-Pty "0";'
+           'quiet "1";'
+        >> /etc/apt/apt.conf.d/99gitlab
+    # Ubuntu and other distros often lack https:// support
+    - grep Debian /etc/issue.net
+        && { find /etc/apt/sources.list* -type f | xargs sed -i s,http:,https:, ; }
+    # The official Debian docker images ship without ca-certificates,
+    # TLS certificates cannot be verified until that is installed. The
+    # following code turns off TLS verification, and enables HTTPS, so
+    # at least unverified TLS is used for apt-get instead of plain
+    # HTTP.  Once ca-certificates is installed, the CA verification is
+    # enabled by removing this config.  This set up makes the initial
+    # `apt-get update` and `apt-get install` look the same as verified
+    # TLS to the network observer and hides the metadata.
+    - echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/99nocacertificates
+    - apt-get update
+    - apt-get install ca-certificates
+    - rm /etc/apt/apt.conf.d/99nocacertificates
+    - apt-get dist-upgrade
+
+# For jobs that only need to run when there are changes to Python files.
+.python-rules-changes: &python-rules-changes
+  rules:
+    - changes:
+        - .gitlab-ci.yml
+        - fdroid
+        - makebuildserver
+        - setup.py
+        - fdroidserver/*.py
+        - tests/*.py
+
+
+# Since F-Droid uses Debian as its default platform, from production
+# servers to CI to contributor machines, it is important to know when
+# changes in Debian break our stuff.  This tests against the latest
+# dependencies as they are included in Debian.
+debian_testing:
+  image: debian:testing
+  <<: *apt-template
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver"
+  script:
+    - apt-get install
+        aapt
+        androguard
+        apksigner
+        dexdump
+        fdroidserver
+        git
+        gnupg
+        ipfs-cid
+        python3-biplist
+        python3-defusedxml
+        python3-libcloud
+        python3-pycountry
+        python3-setuptools
+        sdkmanager
+    - python3 -c 'import fdroidserver'
+    - python3 -c 'import androguard'
+    - python3 -c 'import sdkmanager'
+    - cd tests
+    - ./run-tests
+
+
+# Test using latest LTS set up with the PPA, including Recommends.
+ubuntu_lts_ppa:
+  image: ubuntu:latest
+  <<: *apt-template
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver"
+  script:
+    - export ANDROID_HOME=/usr/lib/android-sdk
+    - apt-get install gnupg
+    - while ! apt-key adv --keyserver keyserver.ubuntu.com --recv-key 9AAC253193B65D4DF1D0A13EEC4632C79C5E0151; do sleep 15; done
+    - export RELEASE=$(sed -n 's,^Suites\x3a \([a-z]*\).*,\1,p' /etc/apt/sources.list.d/*.sources | head -1)
+    - echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list
+    - apt-get update
+    - apt-get dist-upgrade
+    - apt-get install --install-recommends
+        dexdump
+        fdroidserver
+        git
+        python3-biplist
+        python3-pycountry
+        python3-setuptools
+        sdkmanager
+
+    # Test things work with a default branch other than 'master'
+    - git config --global init.defaultBranch thisisnotmasterormain
+
+    - cd tests
+    - ./run-tests
+
+
+# Test to see how rclone works with S3
+test_deploy_to_s3_with_rclone:
+  image: debian:bookworm-slim
+  <<: *apt-template
+  tags:
+    - saas-linux-small-amd64  # the shared runners are known to support Docker.
+  services:
+    - name: docker:dind
+      command: ["--tls=false"]
+  variables:
+    DOCKER_HOST: "tcp://docker:2375"
+    DOCKER_DRIVER: overlay2
+    DOCKER_TLS_CERTDIR: ""
+  before_script:
+    # ensure minio is up before executing tests
+    - apt-get update
+    - apt-get install -y
+        androguard
+        apksigner
+        curl
+        docker.io
+        git
+        python3-venv
+        rclone
+    # This job requires working docker but will silently fail if docker is not available
+    - docker info
+    - python3 -m venv --system-site-packages test-venv
+    - . test-venv/bin/activate
+    - pip install testcontainers[minio]
+    - pip install .
+  script:
+    - python3 -m unittest -k test_update_remote_storage_with_rclone --verbose
+  rules:
+    - changes:
+        - .gitlab-ci.yml
+        - fdroidserver/deploy.py
+        - tests/test_deploy.py
+        - tests/test_integration.py
+
+
+# Test using Ubuntu/jammy LTS (supported til April, 2027) with depends
+# from pypi and sdkmanager.  The venv is used to isolate the dist
+# tarball generation environment from the clean install environment.
+ubuntu_jammy_pip:
+  image: ubuntu:jammy
+  <<: *apt-template
+  script:
+    - apt-get install git default-jdk-headless python3-pip python3-venv rsync
+
+    # setup venv to act as release build machine
+    - python3 -m venv sdist-env
+    - . sdist-env/bin/activate
+    - ./setup.py sdist
+    - deactivate
+    - tar tzf dist/fdroidserver-*.tar.gz
+
+    # back to bare machine to act as user's install machine
+    - export ANDROID_HOME=/opt/android-sdk
+    - $pip install sdkmanager
+    - sdkmanager 'build-tools;35.0.0'
+
+    # Install extras_require.optional from setup.py
+    - $pip install biplist pycountry
+
+    - $pip install dist/fdroidserver-*.tar.gz
+    - tar xzf dist/fdroidserver-*.tar.gz
+    - cd fdroidserver-*
+    - export PATH=$PATH:$ANDROID_HOME/build-tools/35.0.0
+    - fdroid=`which fdroid` ./tests/run-tests
+
+    # check localization was properly installed
+    - LANGUAGE='de' fdroid --help | grep 'Gültige Befehle sind'
+
+
+# Run all the various linters and static analysis tools.
+hooks/pre-commit:
+  stage: lint
+  image: debian:bookworm-slim
+  variables:
+    LANG: C.UTF-8
+  script:
+    - apt-get update
+    - apt-get -y install --no-install-recommends
+          bash
+          ca-certificates
+          dash
+          gcc
+          git
+          make
+          pycodestyle
+          pyflakes3
+          python3-dev
+          python3-git
+          python3-nose
+          python3-pip
+          python3-yaml
+    - ./hooks/pre-commit
+
+bandit:
+  image: debian:bookworm-slim
+  <<: *python-rules-changes
+  <<: *apt-template
+  script:
+    - apt-get install python3-pip
+    - $pip install --break-system-packages bandit
+    - bandit -r -ii --ini .bandit
+
+pylint:
+  stage: lint
+  image: debian:bookworm-slim
+  <<: *python-rules-changes
+  <<: *apt-template
+  script:
+    - apt-get install pylint python3-pip
+    - $pip install --break-system-packages pylint-gitlab
+    - pylint --output-format=colorized,pylint_gitlab.GitlabCodeClimateReporter:pylint-report.json
+            fdroid
+            makebuildserver
+            setup.py
+            fdroidserver/*.py
+            tests/*.py
+  artifacts:
+    reports:
+      codequality: pylint-report.json
+    when: always
+
+
+shellcheck:
+  stage: lint
+  image: debian:bookworm-slim
+  rules:
+    - changes:
+        - .gitlab-ci.yml
+        - hooks/install-hooks.sh
+        - hooks/pre-commit
+        - tests/run-tests
+  <<: *apt-template
+  script:
+    - apt-get install shellcheck
+    # TODO GitLab Code Quality report https://github.com/koalaman/shellcheck/issues/3155
+    - shellcheck --exclude SC2046,SC2090 --severity=warning --color
+        hooks/install-hooks.sh
+        hooks/pre-commit
+        tests/run-tests
+
+# Check all the dependencies in Debian to mirror production.  CVEs are
+# generally fixed in the latest versions in pip/pypi.org, so it isn't
+# so important to scan that kind of install in CI.
+# https://docs.safetycli.com/safety-docs/installation/gitlab
+safety:
+  image: debian:bookworm-slim
+  rules:
+    - if: $SAFETY_API_KEY
+      changes:
+        - .gitlab-ci.yml
+        - .safety-policy.yml
+        - pyproject.toml
+        - setup.py
+  <<: *apt-template
+  variables:
+    LANG: C.UTF-8
+  script:
+    - apt-get install
+        fdroidserver
+        python3-biplist
+        python3-pip
+        python3-pycountry
+    - $pip install --break-system-packages .
+
+    - $pip install --break-system-packages safety
+    - python3 -m safety --key "$SAFETY_API_KEY" --stage cicd scan
+
+
+# TODO tests/*/*/*.yaml are not covered
+yamllint:
+  stage: lint
+  image: debian:bookworm-slim
+  rules:
+    - changes:
+        - .gitlab-ci.yml
+        - .safety-policy.yml
+        - .yamllint
+        - tests/*.yml
+        - tests/*/*.yml
+        - tests/*/*/.*.yml
+  <<: *apt-template
+  variables:
+    LANG: C.UTF-8
+  script:
+    - apt-get install yamllint
+    - yamllint
+        .gitlab-ci.yml
+        .safety-policy.yml
+        .yamllint
+        tests/*.yml
+        tests/*/*.yml
+        tests/*/*/.*.yml
+
+
+locales:
+  stage: lint
+  image: debian:bookworm-slim
+  variables:
+    LANG: C.UTF-8
+  script:
+    - apt-get update
+    - apt-get -y install --no-install-recommends
+          gettext
+          make
+          python3-babel
+    - export EXITVALUE=0
+    - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
+    - make -C locale compile || set_error
+    - rm -f locale/*/*/*.mo
+    - pybabel compile --domain=fdroidserver --directory locale 2>&1 | { grep -F "error:" && exit 1; } || true
+    - exit $EXITVALUE
+
+
+black:
+  stage: lint
+  image: debian:bookworm-slim
+  <<: *apt-template
+  script:
+    - apt-get install black
+    - black --check --diff --color $CI_PROJECT_DIR
+
+fedora_latest:
+  image: fedora:39  # support ends on 2024-11-12
+  script:
+    # tricks to hopefully make runs more reliable
+    - echo "timeout=600" >> /etc/dnf/dnf.conf
+    - echo "retries=50" >> /etc/dnf/dnf.conf
+    - echo "keepcache=True" >> /etc/dnf/dnf.conf
+
+    - dnf -y update || dnf -y update
+    - dnf -y install @development-tools
+                     diffutils
+                     findutils
+                     git
+                     gnupg
+                     java-17-openjdk-devel
+                     openssl
+                     python3
+                     python3-babel
+                     python3-matplotlib
+                     python3-pip
+                     python3-pycountry
+                     rsync
+                     which
+    - $pip install sdkmanager
+    - ./setup.py sdist
+    - useradd -m -c "test account" --password "fakepassword"  testuser
+    - su testuser --login --command "cd `pwd`; $pip install --user dist/fdroidserver-*.tar.gz"
+    - test -e ~testuser/.local/share/locale/de/LC_MESSAGES/fdroidserver.mo
+    - export BUILD_TOOLS_VERSION=`sed -n "s,^MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py`
+    - export ANDROID_HOME=`pwd`/android-sdk
+    - mkdir -p $ANDROID_HOME/licenses/
+    - printf "\n8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e\n24333f8a63b6825ea9c5514f83c2829b004d1fee" > $ANDROID_HOME/licenses/android-sdk-license
+    - printf "\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license
+    - printf "\n79120722343a6f314e0719f863036c702b0e6b2a\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license-old
+    - mkdir ~/.android
+    - touch ~/.android/repositories.cfg
+    - sdkmanager "platform-tools" "build-tools;$BUILD_TOOLS_VERSION"
+    - chown -R testuser .
+    - cd tests
+    - su testuser --login --command
+        "cd `pwd`; export CI=$CI ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"
+
+
+macOS:
+  tags:
+    - saas-macos-medium-m1
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver"
+  script:
+    - export HOMEBREW_CURL_RETRIES=10
+    - brew update > /dev/null
+    - brew upgrade
+    - brew install fdroidserver
+
+    # Android SDK and Java JDK
+    - brew install --cask android-commandlinetools temurin  # temurin is a JDK
+
+    # test suite dependencies
+    - brew install bash coreutils gnu-sed
+    # TODO port tests/run-tests to POSIX and gsed, it has a couple GNU-isms like du --bytes
+    - export PATH="$(brew --prefix fdroidserver)/libexec/bin:$(brew --prefix coreutils)/libexec/gnubin:$PATH"
+
+    - brew autoremove
+    - brew info fdroidserver
+
+    - export BUILD_TOOLS_VERSION=`gsed -n "s,^MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py`
+    - export ANDROID_HOME="$(brew --prefix)/share/android-commandlinetools"
+    - mkdir -p "$ANDROID_HOME/licenses"
+    - echo -e "\n8933bad161af4178b1185d1a37fbf41ea5269c55" > "$ANDROID_HOME/licenses/android-sdk-license"
+    - echo -e "\nd56f5187479451eabf01fb78af6dfcb131a6481e" >> "$ANDROID_HOME/licenses/android-sdk-license"
+    - echo -e "\n24333f8a63b6825ea9c5514f83c2829b004d1fee" >> "$ANDROID_HOME/licenses/android-sdk-license"
+    - $(brew --prefix)/bin/sdkmanager "build-tools;$BUILD_TOOLS_VERSION"
+
+    - echo "macOS sticks with bash 3.x because of licenses, so avoid new bash syntax"
+    - /bin/bash --version
+    - /bin/bash -n tests/run-tests
+
+    # test fdroidserver from git with current package's dependencies
+    - fdroid="$(brew --prefix fdroidserver)/libexec/bin/python3 $PWD/fdroid" ./tests/run-tests
+
+
+gradle:
+  image: debian:trixie-slim
+  <<: *apt-template
+  rules:
+    - changes:
+        - .gitlab-ci.yml
+        - makebuildserver
+  script:
+    - apt-get install
+          ca-certificates
+          git
+          python3-colorama
+          python3-packaging
+          python3-requests
+    - ./tests/gradle-release-checksums.py
+
+
+# Run an actual build in a simple, faked version of the buildserver guest VM.
+fdroid build:
+  image: registry.gitlab.com/fdroid/fdroidserver:buildserver
+  rules:
+    - changes:
+        - .gitlab-ci.yml
+        - fdroidserver/build.py
+        - fdroidserver/common.py
+        - fdroidserver/exception.py
+        - fdroidserver/metadata.py
+        - fdroidserver/net.py
+        - fdroidserver/scanner.py
+        - fdroidserver/vmtools.py
+        # for the docker: job which depends on this one
+        - makebuildserver
+        - buildserver/*
+  cache:
+    key: "$CI_JOB_NAME"
+    paths:
+      - .gradle
+  script:
+    - apt-get update
+    - apt-get dist-upgrade
+    - apt-get clean
+
+    - test -n "$fdroidserver" || source /etc/profile.d/bsenv.sh
+
+    - ln -fsv "$CI_PROJECT_DIR" "$fdroidserver"
+
+    # TODO remove sdkmanager install once it is included in the buildserver image
+    - apt-get install sdkmanager
+    - rm -rf "$ANDROID_HOME/tools"  # TODO remove once sdkmanager can upgrade installed packages
+    - sdkmanager "tools" "platform-tools" "build-tools;31.0.0"
+
+    - git ls-remote https://gitlab.com/fdroid/fdroiddata.git master
+    - git clone --depth 1 https://gitlab.com/fdroid/fdroiddata.git
+    - cd fdroiddata
+    - for d in build logs repo tmp unsigned $home_vagrant/.android; do
+          test -d $d || mkdir $d;
+          chown -R vagrant $d;
+      done
+
+    - export GRADLE_USER_HOME=$home_vagrant/.gradle
+    - export fdroid="sudo --preserve-env --user vagrant
+          env PATH=$fdroidserver:$PATH
+          env PYTHONPATH=$fdroidserver:$fdroidserver/examples
+          env PYTHONUNBUFFERED=true
+          env TERM=$TERM
+          env HOME=$home_vagrant
+          fdroid"
+
+    - git -C $home_vagrant/gradlew-fdroid pull
+
+    - chown -R vagrant $home_vagrant
+    - chown -R vagrant $fdroidserver/.git
+    - chown vagrant $fdroidserver/
+    - chown -R vagrant .git
+    - chown vagrant .
+
+    # try user build
+    - $fdroid build --verbose --latest org.fdroid.fdroid.privileged
+
+    # try on-server build
+    - $fdroid build --verbose --on-server --no-tarball --latest org.fdroid.fdroid
+
+    # each `fdroid build --on-server` run expects sudo, then uninstalls it
+    - if dpkg --list sudo; then echo "sudo should not be still there"; exit 1; fi
+    - 'if [ ! -f repo/status/running.json ]; then echo "ERROR: running.json does not exist!"; exit 1; fi'
+    - 'if [ ! -f repo/status/build.json ]; then echo "ERROR: build.json does not exist!"; exit 1; fi'
+
+
+# test the plugin API and specifically the fetchsrclibs plugin, which
+# is used by the `fdroid build` job.  This uses a fixed commit from
+# fdroiddata because that one is known to work, and this is a CI job,
+# so it should be isolated from the normal churn of fdroiddata.
+plugin_fetchsrclibs:
+  image: debian:bookworm-slim
+  <<: *apt-template
+  rules:
+    - changes:
+        - .gitlab-ci.yml
+        - examples/fdroid_fetchsrclibs.py
+        - fdroidserver/__main__.py
+  script:
+    - apt-get install
+        curl
+        git
+        python3-cffi
+        python3-matplotlib
+        python3-nacl
+        python3-paramiko
+        python3-pil
+        python3-pip
+        python3-pycparser
+        python3-venv
+    - python3 -m venv --system-site-packages env
+    - . env/bin/activate
+    - export PATH="$CI_PROJECT_DIR:$PATH"
+    - export PYTHONPATH="$CI_PROJECT_DIR/examples"
+    # workaround https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1003252
+    - export SETUPTOOLS_USE_DISTUTILS=stdlib
+    - $pip install -e .
+    - fdroid | grep fetchsrclibs
+
+    - mkdir fdroiddata
+    - commitid=b9e9a077d720c86ff6fff4dbb341254cc4370b1a
+    - curl https://gitlab.com/fdroid/fdroiddata/-/archive/${commitid}/fdroiddata-${commitid}.tar.gz
+          | tar -xz --directory=fdroiddata --strip-components=1
+    - cd fdroiddata
+    - fdroid fetchsrclibs freemap.opentrail:4 --verbose
+    - test -d build/freemap.opentrail/.git
+    - test -d build/srclib/andromaps/.git
+    - test -d build/srclib/freemaplib/.git
+    - test -d build/srclib/freemaplibProj/.git
+    - test -d build/srclib/JCoord/.git
+    - test -d build/srclib/javaproj/.git
+
+
+# test a full update and deploy cycle to gitlab.com
+servergitmirrors:
+  image: debian:bookworm-slim
+  <<: *apt-template
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver"
+  script:
+    - apt-get install
+        default-jdk-headless
+        git
+        openssh-client
+        openssl
+        python3-cffi
+        python3-cryptography
+        python3-matplotlib
+        python3-nacl
+        python3-pil
+        python3-pip
+        python3-pycparser
+        python3-setuptools
+        python3-venv
+        rsync
+        wget
+    - apt-get install apksigner
+    - python3 -m venv --system-site-packages env
+    - . env/bin/activate
+    - export PYTHONPATH=`pwd`
+    - export SETUPTOOLS_USE_DISTUTILS=stdlib  # https://github.com/pypa/setuptools/issues/2956
+    - $pip install -e .
+    - mkdir /root/.ssh/
+    - ./tests/key-tricks.py
+    - ssh-keyscan gitlab.com >> /root/.ssh/known_hosts
+    - test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo
+    - cp tests/config.yml tests/keystore.jks /tmp/fdroid/
+    - cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/
+    - cd /tmp/fdroid
+    - touch fdroid-icon.png
+    - printf "\nservergitmirrors\x3a 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.yml
+    - $PYTHONPATH/fdroid update --verbose --create-metadata
+    - $PYTHONPATH/fdroid deploy --verbose
+    - export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json`
+    - echo $DLURL
+    - wget $DLURL/index-v1.jar
+    - diff repo/index-v1.jar index-v1.jar
+
+Build documentation:
+  image: debian:bookworm-slim
+  <<: *python-rules-changes
+  <<: *apt-template
+  script:
+    - apt-get install make python3-sphinx python3-numpydoc python3-pydata-sphinx-theme pydocstyle fdroidserver
+    - apt purge fdroidserver
+    # ignore vendored files
+    - pydocstyle --verbose --match='(?!apksigcopier|looseversion|setup|test_).*\.py' fdroidserver
+    - cd docs
+    - sphinx-apidoc -o ./source ../fdroidserver -M -e
+    - PYTHONPATH=.. sphinx-autogen -o generated source/*.rst
+    - PYTHONPATH=.. make html
+  artifacts:
+    paths:
+      - docs/build/html/
+
+
+# this job will only run in branches called "windows" until the Windows port is complete
+Windows:
+  tags:
+    - windows
+  rules:
+    - if: $CI_COMMIT_BRANCH == "windows"
+  script:
+    - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
+    - choco install --no-progress -y git --force --params "/GitAndUnixToolsOnPath"
+    - choco install --no-progress -y python3 --version=3.10
+    - choco install --no-progress -y jdk8
+    - choco install --no-progress -y rsync
+    - refreshenv
+    - python -m pip install --upgrade babel pip setuptools
+    - python -m pip install -e .
+
+    - $files = @(Get-ChildItem tests\test_*.py)
+    - foreach ($f in $files) {
+          write-output $f;
+          python -m unittest $f;
+          if( $LASTEXITCODE -eq 0 ) {
+              write-output "SUCCESS $f";
+          } else {
+              write-output "ERROR $f failed";
+          }
+      }
+
+    # these are the tests that must pass
+    - python -m unittest -k
+        checkupdates
+        exception
+        import_subcommand
+        test_lint
+        test_metadata
+        test_rewritemeta
+        test_vcs
+        tests.test_init
+        tests.test_main
+  after_script:
+    - Copy-Item C:\ProgramData\chocolatey\logs\chocolatey.log
+  artifacts:
+    when: always
+    paths:
+      - "*.log"
+  allow_failure:
+    exit_codes: 1
+
+
+pages:
+  image: alpine:latest
+  stage: deploy
+  script:
+    - cp docs/build/html public -r  # GL Pages needs the files in a directory named "public"
+  artifacts:
+    paths:
+      - public
+  needs:
+    - job: "Build documentation"
+      optional: true
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'  # only publish pages on default (master) branch
+
+
+# This job pushes the official CI docker image based on the master
+# branch, so in fdroid/fdroidserver, it should only run on the master
+# branch.  Otherwise, tags or other branches will overwrite the docker
+# image which is supposed to be what is in master.
+docker:
+  dependencies:
+    - fdroid build
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver"
+      changes:
+        - .gitlab-ci.yml
+        - makebuildserver
+        - buildserver/*
+  image: docker:dind
+  services:
+    - docker:dind
+  variables:
+    RELEASE_IMAGE: $CI_REGISTRY_IMAGE:buildserver
+  script:
+    # git ref names can contain many chars that are not allowed in docker tags
+    - export TEST_IMAGE=$CI_REGISTRY_IMAGE:$(printf $CI_COMMIT_REF_NAME | sed 's,[^a-zA-Z0-9_.-],_,g')
+    - cd buildserver
+    - docker build -t $TEST_IMAGE --build-arg GIT_REV_PARSE_HEAD=$(git rev-parse HEAD) .
+    - docker tag $TEST_IMAGE $RELEASE_IMAGE
+    - docker tag $TEST_IMAGE ${RELEASE_IMAGE}-bookworm
+    - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com
+    # This avoids filling up gitlab.com free tier accounts with unused docker images.
+    - if test -z "$FDROID_PUSH_DOCKER_IMAGE"; then
+          echo "Skipping docker push to save quota on your gitlab namespace.";
+          echo "If you want to enable the push, set FDROID_PUSH_DOCKER_IMAGE in";
+          echo "https://gitlab.com/$CI_PROJECT_NAMESPACE/fdroidserver/-/settings/ci_cd#js-cicd-variables-settings";
+          exit 0;
+      fi
+    - docker push $RELEASE_IMAGE
+    - docker push $RELEASE_IMAGE-bookworm
+
+
+# PUBLISH is the signing server.  It has a very minimal manual setup.
+PUBLISH:
+  image: debian:bookworm-backports
+  <<: *python-rules-changes
+  script:
+    - apt-get update
+    - apt-get -qy upgrade
+    - apt-get -qy install --no-install-recommends -t bookworm-backports
+        androguard
+        apksigner
+        curl
+        default-jdk-headless
+        git
+        gpg
+        gpg-agent
+        python3-asn1crypto
+        python3-defusedxml
+        python3-git
+        python3-ruamel.yaml
+        python3-yaml
+        rsync
+
+    # Run only relevant parts of the test suite, other parts will fail
+    # because of this minimal base setup.
+    - python3 -m unittest
+        tests/test_gpgsign.py
+        tests/test_metadata.py
+        tests/test_publish.py
+        tests/test_signatures.py
+        tests/test_signindex.py
+
+    - cd tests
+    - mkdir archive
+    - mkdir unsigned
+    - cp urzip-release-unsigned.apk unsigned/info.guardianproject.urzip_100.apk
+    - grep '^key.*pass' config.yml | sed 's,\x3a ,=,' > $CI_PROJECT_DIR/variables
+    - sed -Ei 's,^(key.*pass|keystore)\x3a.*,\1\x3a {env\x3a \1},' config.yml
+    - printf '\ngpghome\x3a {env\x3a gpghome}\n' >> config.yml
+    - |
+      tee --append $CI_PROJECT_DIR/variables <<EOF
+          gpghome=$CI_PROJECT_DIR/tests/gnupghome
+          keystore=$CI_PROJECT_DIR/tests/keystore.jks
+          serverwebroot=/tmp
+          export gpghome keypass keystorepass keystore serverwebroot
+      EOF
+    - source $CI_PROJECT_DIR/variables
+    # silence warnings
+    - chmod 0600 config.yml config/*.yml config/*/*.yml
+    - chmod 0700 $gpghome
+
+    - export PATH=$CI_PROJECT_DIR:$PATH
+
+    # run signpkg.sh
+    - fdroid publish --verbose
+    - fdroid gpgsign --verbose
+    - rsync --progress repo/* $serverwebroot/
+
+    # run signindex.sh
+    - fdroid gpgsign --verbose
+    - fdroid signindex --verbose
+    - rsync --stats repo/* $serverwebroot/

.mailmap

@@ -0,0 +1,2 @@
+Gregor Düster <git@gdstr.eu> FestplattenSchnitzel <festplatte.schnitzel@posteo.de>
+Hans-Christoph Steiner <hans@eds.org> <hans@guardianproject.info>

.safety-policy.yml

@@ -0,0 +1,55 @@
+---
+
+version: '3.0'
+
+scanning-settings:
+  max-depth: 6
+  exclude:
+
+report:
+  dependency-vulnerabilities:
+    enabled: true
+    auto-ignore-in-report:
+      vulnerabilities:
+        52495:
+          reason: setuptools comes from Debian
+          expires: '2025-01-31'
+        60350:
+          reason: GitPython comes from Debian https://security-tracker.debian.org/tracker/CVE-2023-40267
+          expires: '2025-01-31'
+        60789:
+          reason: GitPython comes from Debian https://security-tracker.debian.org/tracker/CVE-2023-40590
+          expires: '2025-01-31'
+        60841:
+          reason: GitPython comes from Debian https://security-tracker.debian.org/tracker/CVE-2023-41040
+          expires: '2025-01-31'
+        62044:
+          reason: "F-Droid doesn't fetch pip dependencies directly from hg/mercurial repositories: https://data.safetycli.com/v/62044/f17/"
+          expires: '2025-01-31'
+        63687:
+          reason: Only affects Windows https://security-tracker.debian.org/tracker/CVE-2024-22190
+          expires: '2026-01-31'
+        67599:
+          reason: Only affects pip when using --extra-index-url, which is never the case in fdroidserver CI.
+          expires: '2026-05-31'
+        70612:
+          reason: jinja2 is not used by fdroidserver, nor any dependencies I could find via debtree and pipdeptree.
+          expires: '2026-05-31'
+        72132:
+          reason: We get these packages from Debian, zipp is not used in production, and its only a DoS.
+          expires: '2026-08-31'
+        72236:
+          reason: setuptools is not used in production to download or install packages, they come from Debian.
+          expires: '2026-08-31'
+
+fail-scan-with-exit-code:
+  dependency-vulnerabilities:
+    enabled: true
+    fail-on-any-of:
+      cvss-severity:
+        - critical
+        - high
+        - medium
+
+security-updates:
+  dependency-vulnerabilities:

.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+    "recommendations": [
+        "ms-python.python",
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,21 @@
+{
+    "python.formatting.blackArgs": [
+        "--config=pyproject.toml"
+    ],
+    "python.formatting.provider": "black",
+    "python.linting.banditEnabled": true,
+    "python.linting.banditArgs": [
+        "-ii",
+        "--ini=.bandit",
+    ],
+    "python.linting.enabled": true,
+    "python.linting.mypyArgs": [
+        "--config-file=mypy.ini"
+    ],
+    "python.linting.mypyEnabled": true,
+    "python.linting.flake8Enabled": true,
+    "python.linting.pylintArgs": [
+        "--rcfile=.pylint-rcfile"
+    ],
+    "python.linting.pylintEnabled": true,
+}

.weblate

@@ -0,0 +1,3 @@
+[weblate]
+url = https://hosted.weblate.org/api/
+translation = f-droid/fdroidserver

.well-known/funding-manifest-urls

@@ -0,0 +1 @@
+https://f-droid.org/funding.json

.yamllint

@@ -0,0 +1,7 @@
+---
+
+extends: default
+rules:
+  document-start: disable
+  line-length: disable
+  truthy: disable

CHANGELOG.md

@@ -0,0 +1,458 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
+
+## [2.5.0] - NEXT
+
+### Removed
+
+* deploy: `awsaccesskeyid:` and `awssecretkey:` config items removed, use the
+  standard env vars: `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
+
+## [2.4.2] - 2025-06-24
+
+### Fixed
+
+* nightly: fix bug that clones nightly repo to wrong location
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1672
+* Sync translations for all supported languages: es pl ru
+
+## [2.4.1] - 2025-06-23
+
+### Added
+
+* build: Clearer error messages when working with Git.
+* verify: generate <appid>.json files that list all reports
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1632
+
+### Fixed
+
+* deploy: use master branch when working complete git-mirror repo
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1666
+* update: use ctime/mtime to control _strip_and_copy_image runs
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1665
+* update: If categories.yml only has icon:, then add name:
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1659
+* update: fix handling of Triple-T 1.0.0 graphics
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1652
+* update: never execute any VCS e.g. git
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1630
+* config: lazyload environment variables in config.yml
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1645
+* config: make localized name/description/icon optional
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1649
+* lint: add repo_key_sha256 to list of valid config keys
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1643
+* build: calculate all combinations of gradle flavors
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1638
+* build: set SOURCE_DATE_EPOCH from app's git otherwise fdroiddata metadata file
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1653
+* Sync translations for all supported languages: ca cs de fr ga ja pl pt pt_BR
+  pt_PT ru sq tr uk zh_Hans
+
+### Removed
+
+## [2.4.0] - 2025-03-25
+
+### Added
+
+* lint: support the base _config.yml_.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1606
+
+### Fixed
+
+* Expand {env: foo} config syntax to be allowed any place a string is.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1610
+* Only show "unsafe permissions on config.yml" when secrets are present.
+* Standardized config files on ruamel.yaml with a YAML 1.2 data format.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1611
+* Brought back error when a package has multiple package types (e.g. xapk and
+  apk).  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1602
+* Reworked test suite to be entirely based on Python unittest (thanks @mindston).
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1587
+* publish/signindex/gpgsign no longer load the _qrcode_ and _requests_ modules,
+  and can operate without them installed.
+* scanner: add bun.lock as lock file of package.json
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1615
+* index: fail if user sets mirrors:isPrimary wrong
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1617
+  https://gitlab.com/fdroid/fdroidserver/-/issues/1125
+* Sync translations for all supported languages: bo ca cs de es fr ga hu it ja
+  ko nb_NO pl pt pt_BR pt_PT ro ru sq sr sw tr uk zh_Hans zh_Hant
+
+### Removed
+
+* checkupdates: remove auto_author: config, it is no longer used.
+* Purge support for the long-deprecated _config.py_ config file.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1607
+
+
+## [2.3.5] - 2025-01-20
+
+### Fixed
+
+* Fix issue where APKs with v1-only signatures and targetSdkVersion < 30 could
+  be maliciously crafted to bypass AllowedAPKSigningKeys
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1588
+* Ignore apksigner v33.x, it has bugs verifying APKs with v3/v3.1 sigs.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1593
+* Sync translations for: ca cs de es fr ga ja pt_BR pt_PT ru sq sr uk zh_Hans
+
+## [2.3.4] - 2024-12-12
+
+### Fixed
+
+* Fix localhost network tests on systems with IPv6.
+* lint: only error out on missing extlib on versions not archived.
+
+## [2.3.3] - 2024-12-11
+
+### Added
+
+* verify: `--clean-up-verified` to delete files used when verifying an APK if
+  the verification was successful.
+
+### Fixed
+
+* Support Python 3.13 in the full test suite.
+* Sync translations for: ca de fr ja pl ro ru sr ta
+* update: only generate _index.png_ when making _index.html_, allowing the repo
+  operator to set a different repo icon, e.g. not the QR Code.
+
+## [2.3.2] - 2024-11-26
+
+### Fixed
+
+* install: fix downloading from GitHub Releases and Maven Central.
+* Sync translations for: ca fa fr pt ru sr ta zh_Hant
+
+## [2.3.1] - 2024-11-25
+
+### Fixed
+
+* Sync all translations for: cs de es fr ga pt_BR ru sq zh_Hans.
+* Drop use of deprecated imghdr library to support Python 3.13.
+* Install biplist and pycountry by default on macOS.
+* Fixed running test suite out of dist tarball.
+
+## [2.3.0] - 2024-11-21
+
+### Added
+
+* YAML 1.2 as native format for all _.yml_ files, including metadata and config.
+* install: will now fetch _F-Droid.apk_ and install it via `adb`.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1546
+* scanner: scan APK Signing Block for known block types like Google Play
+  Signature aka "Frosting".
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1555
+* Support Rclone for deploying to many different cloud services.
+* deploy: support deploying to GitHub Releases.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1471
+* scanner: support libs.versions.toml
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1526
+* Consider subdir for triple-t metadata discovery in Flutter apps.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1541
+* deploy: added `index_only:` mode for mirroring the index to small hosting
+  locations. https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1420
+* Support publishing repos in AltStore format.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1465
+* Support indexing iOS IPA app files.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1413
+* deploy: _config/mirrors.yml_ file with support for adding per-mirror metadata,
+  like `countryCode:`.
+* Repo's categories are now set in the config files.
+* lint: check syntax of config files.
+* publish: `--error-on-failed` to exit when signing/verifying fails.
+* scanner: `--refresh` and `refresh_config:` to control triggering a refresh of
+  the rule sets.
+* Terminal output colorization and `--color` argument to control it.
+* New languages: Catalan (ca), Irish (ga), Japanese (ja), Serbian (sr), and
+  Swahili (sw).
+* Support donation links from `community_bridge`, `buy_me_a_coffee`.
+
+### Fixed
+
+* Use last modified time and file size for caching data about scanned APKs
+  instead of SHA-256 checksum.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1542
+* `repo_web_base_url:` config for generating per-app URLs for viewing in
+  browsers.  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1178
+* `fdroid scanner` flags WebAssembly binary _.wasm_ files.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1562
+* Test suite as standard Python `unittest` setup (thanks @ghost.adh).
+* scanner: error on dependency files without lock file.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1504
+* nightly: finding APKs in the wrong directory. (thanks @WrenIX)
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1512
+* `AllowedAPKSigningKeys` works with all single-signer APK signatures.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1466
+* Sync all translations for: cs de it ko pl pt pt_BR pt_PT ro ru sq tr uk
+  zh_Hans zh_Hant.
+* Support Androguard 4.x.
+* Support Python 3.12.
+
+### Removed
+
+* Drop all uses of _stats/known_apks.txt_ and the `update_stats:` config key.
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1547
+* The `maven:` field is now always a string, with `yes` as a legacy special
+  value.  It is no longer treated like a boolean in any case.
+* scanner: jcenter is no longer an allowed Maven repo.
+* build: `--reset-server` removed (thanks @gotmi1k).
+
+## [2.2.2] - 2024-04-24
+
+### Added
+
+* Include sdkmanager as dep in setup.py for Homebrew package.
+  https://github.com/Homebrew/homebrew-core/pull/164510
+
+## [2.2.1] - 2023-03-09
+
+### Added
+
+* `download_repo_index_v2()` and `download_repo_index_v2()` API functions
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1323
+
+### Fixed
+
+* Fix OpenJDK detection on different CPU architectures
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1315
+
+### Removed
+
+* Purge all references to `zipalign`, that is delegated to other things
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1316
+* Remove obsolete, unused `buildozer` build type
+  https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1322
+
+## [2.2.0] - 2023-02-20
+
+### Added
+* Support index-v2 format, localizable Anti-Features, Categories
+* New entry point for repos, entry.jar, signed with modern algorithms
+* New config/ subdirectory for localizable configuration
+* Script entries in metadata files (init, prebuild, build, etc) now handled as
+  lists so they now support using && or ; in the script, and behave like
+  .gitlab-ci.yml and other CI YAML.
+* GPG signatures for index-v1.json and index-v2.json
+* Use default.txt as fallback changelog when inserting fastlane metadata
+* scanner: F-Droid signatures now maintained in fdroid/suss
+* scanner: maintain signature sources in config.yml, including Exodus Privacy
+* scanner: use dexdump for class names
+* scanner: directly scan APK files when given a path
+* scanner: recursively scan APKs for DEX and ZIP using file magic
+* signindex: validate index files before signing
+* update: set ArchivePolicy based on VercodeOperation/signature
+* Include IPFS CIDv1 in index-v2.json for hosting repos on IPFS
+* Per-repo beta channel configuration
+* Add Czech translation
+
+### Fixed
+
+* apksigner v30 or higher now required for verifying and signing APKs
+* 3.9 as minimum supported Python version
+* Lots of translation updates
+* Better pip packaging
+* nightly: big overhaul for reliable operation on all Debian/Ubuntu versions
+* Improved logging, fewer confusing verbose messages
+* scanner: fix detection of binary files without extension
+* import: more reliable operation, including Flutter apps
+* Support Java 20 and up
+
+### Removed
+* Remove obsolete `fdroid stats` command
+
+## [2.1.1] - 2022-09-06
+
+* gradlew-fdroid: Include latest versions and checksums
+* nightly: update Raw URLs to fix breakage and avoid redirects
+* signindex: gpg-sign index-v1.json and deploy it
+* update: fix --use-date-from-apk when used with files (#1012)
+
+## [2.1] - 2022-02-22
+
+For a more complete overview, see the [2.1
+milestone](https://gitlab.com/fdroid/fdroidserver/-/milestones/11)
+
+## [2.0.5] - 2022-09-06
+
+### Fixed
+
+* gradlew-fdroid: Include latest versions and checksums
+* nightly: add support for GitHub Actions
+* nightly: update Raw URLs to fix breakage and avoid redirects
+* update: fix --use-date-from-apk when used with files (#1012)
+* Fix GitLab CI
+
+## [2.0.4] - 2022-06-29
+
+### Fixed
+
+* deploy: ensure progress is instantiated before trying to use it
+* signindex: gpg-sign index-v1.json and deploy it
+  [1080](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1080)
+  [1124](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1124)
+
+## [2.0.3] - 2021-07-01
+
+### Fixed
+
+* Support AutoUpdateMode: Version without pattern
+  [931](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/931)
+
+## [2.0.2] - 2021-06-01
+
+### Fixed
+
+* fix "ruamel round_trip_dump will be removed"
+  [932](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/932)
+
+## [2.0.1] - 2021-03-09
+
+### Fixed
+
+* metadata: stop setting up source repo when running lint/rewritemeta
+* scanner: show error if scan_binary fails to run apkanalyzer
+* common: properly parse version from NDK's source.properties
+* update: stop extracting and storing XML icons, they're useless
+* index: raise error rather than crash on bad repo file
+* update: handle large, corrupt, or inaccessible fastlane/triple-t files
+* Update SPDX License List
+* checkupdates: set User-Agent to make gitlab.com happy
+* Run push_binary_transparency only once
+
+## [2.0] - 2021-01-31
+
+For a more complete overview, see the [2.0
+milestone](https://gitlab.com/fdroid/fdroidserver/-/milestones/10)
+
+### Added
+* `fdroid update` inserts donation links based on upstream's _FUNDING.yml_
+  ([!754](https://gitlab.com/fdroid/fdroidserver/merge_requests/754))
+* Stable, public API for most useful functions
+  ([!798](https://gitlab.com/fdroid/fdroidserver/merge_requests/798))
+* Load with any YAML lib and use with the API, no more custom parser needed
+  ([!826](https://gitlab.com/fdroid/fdroidserver/merge_requests/826))
+  ([!838](https://gitlab.com/fdroid/fdroidserver/merge_requests/838))
+* _config.yml_ for a safe, easy, standard configuration format
+  ([!663](https://gitlab.com/fdroid/fdroidserver/merge_requests/663))
+* Config options can be set from environment variables using this syntax:
+  `keystorepass: {env: keystorepass}`
+  ([!669](https://gitlab.com/fdroid/fdroidserver/merge_requests/669))
+* Add SHA256 to filename of repo graphics
+  ([!669](https://gitlab.com/fdroid/fdroidserver/merge_requests/669))
+* Support for srclibs metadata in YAML format
+  ([!700](https://gitlab.com/fdroid/fdroidserver/merge_requests/700))
+* Check srclibs and app-metadata files with yamllint
+  ([!721](https://gitlab.com/fdroid/fdroidserver/merge_requests/721))
+* Added plugin system for adding subcommands to `fdroid`
+  ([!709](https://gitlab.com/fdroid/fdroidserver/merge_requests/709))
+* `fdroid update`, `fdroid publish`, and `fdroid signindex` now work
+  with SmartCard HSMs, specifically the NitroKey HSM
+  ([!779](https://gitlab.com/fdroid/fdroidserver/merge_requests/779))
+  ([!782](https://gitlab.com/fdroid/fdroidserver/merge_requests/782))
+* `fdroid update` support for Triple-T Gradle Play Publisher v2.x
+  ([!683](https://gitlab.com/fdroid/fdroidserver/merge_requests/683))
+* Translated into: bo de es fr hu it ko nb_NO pl pt pt_BR pt_PT ru sq tr uk
+  zh_Hans zh_Hant
+
+### Fixed
+* Smoother process for signing APKs with `apksigner`
+  ([!736](https://gitlab.com/fdroid/fdroidserver/merge_requests/736))
+  ([!821](https://gitlab.com/fdroid/fdroidserver/merge_requests/821))
+* `apksigner` is used by default on new repos
+* All parts except _build_ and _publish_ work without the Android SDK
+  ([!821](https://gitlab.com/fdroid/fdroidserver/merge_requests/821))
+* Description: is now passed to clients unchanged, no HTML conversion
+  ([!828](https://gitlab.com/fdroid/fdroidserver/merge_requests/828))
+* Lots of improvements for scanning for proprietary code and trackers
+  ([!748](https://gitlab.com/fdroid/fdroidserver/merge_requests/748))
+  ([!REPLACE](https://gitlab.com/fdroid/fdroidserver/merge_requests/REPLACE))
+  ([!844](https://gitlab.com/fdroid/fdroidserver/merge_requests/844))
+* `fdroid mirror` now generates complete, working local mirror repos
+* fix build-logs dissapearing when deploying
+  ([!685](https://gitlab.com/fdroid/fdroidserver/merge_requests/685))
+* do not crash when system encoding can not be retrieved
+  ([!671](https://gitlab.com/fdroid/fdroidserver/merge_requests/671))
+* checkupdates: UpdateCheckIngore gets properly observed now
+  ([!659](https://gitlab.com/fdroid/fdroidserver/merge_requests/659),
+  [!660](https://gitlab.com/fdroid/fdroidserver/merge_requests/660))
+* keep yaml metadata when rewrite failed
+  ([!658](https://gitlab.com/fdroid/fdroidserver/merge_requests/658))
+* import: `template.yml` now supports omitting values
+  ([!657](https://gitlab.com/fdroid/fdroidserver/merge_requests/657))
+* build: deploying buildlogs with rsync
+  ([!651](https://gitlab.com/fdroid/fdroidserver/merge_requests/651))
+* `fdroid init` generates PKCS12 keystores, drop Java < 8 support
+  ([!801](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/801))
+* Parse Version Codes specified in hex
+  ([!692](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/692))
+* Major refactoring on core parts of code to be more Pythonic
+  ([!756](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/756))
+* `fdroid init` now works when installed with pip
+
+### Removed
+* Removed all support for _.txt_ and _.json_ metadata
+  ([!772](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/772))
+* dropped support for Debian 8 _jessie_ and 9 _stretch_
+* dropped support for Ubuntu releases older than bionic 18.04
+* dropped `fdroid server update` and `fdroid server init`,
+  use `fdroid deploy`
+* `fdroid dscanner` was removed.
+  ([!711](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/711))
+* `make_current_version_link` is now off by default
+* Dropped `force_build_tools` config option
+  ([!797](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/797))
+* Dropped `accepted_formats` config option, there is only _.yml_ now
+  ([!818](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/818))
+* `Provides:` was removed as a metadata field
+  ([!654](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/654))
+* Remove unused `latestapps.dat`
+  ([!794](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/794))
+
+
+## [1.1.4] - 2019-08-15
+### Fixed
+* include bitcoin validation regex required by fdroiddata
+* merged Debian patches to fix test suite there
+
+## [1.1.3] - 2019-07-03
+### Fixed
+* fixed test suite when run from source tarball
+* fixed test runs in Debian
+
+## [1.1.2] - 2019-03-29
+### Fixed
+* fix bug while downloading repo index
+  ([!636](https://gitlab.com/fdroid/fdroidserver/merge_requests/636))
+
+## [1.1.1] - 2019-02-03
+### Fixed
+* support APK Signature v2 and v3
+* all SDK Version values are output as integers in the index JSON
+* take graphics from Fastlane dirs using any valid RFC5646 locale
+* print warning if not running in UTF-8 encoding
+* fdroid build: hide --on-server cli flag
+
+## [1.1] - 2019-01-28
+### Fixed
+* a huge update with many fixes and new features:
+  https://gitlab.com/fdroid/fdroidserver/milestones/7
+* can run without and Android SDK installed
+* much more reliable operation with large binary APK collections
+* sync all translations, including newly added languages: hu it ko pl pt_PT ru
+* many security fixes, based on the security audit
+* NoSourceSince automatically adds SourceGone Anti-Feature
+* aapt scraping works with all known aapt versions
+* smoother mirror setups
+* much faster `fdroid update` when using androguard
+
+[Unreleased]: https://gitlab.com/fdroid/fdroidserver/compare/1.1.4...master
+[1.1.4]: https://gitlab.com/fdroid/fdroidserver/compare/1.1.3...1.1.4
+[1.1.3]: https://gitlab.com/fdroid/fdroidserver/compare/1.1.2...1.1.3
+[1.1.2]: https://gitlab.com/fdroid/fdroidserver/compare/1.1.1...1.1.2
+[1.1.1]: https://gitlab.com/fdroid/fdroidserver/compare/1.1...1.1.1
+[1.1]: https://gitlab.com/fdroid/fdroidserver/tags/1.1

CONTRIBUTING.md

@@ -0,0 +1,66 @@
+There are many ways to contribute, you can find out all the ways on our
+[Contribute](https://f-droid.org/contribute/) page. Find out how to get
+involved, including as a translator, data analyst, tester, helping others, and
+much more!
+
+## Contributing Code
+
+We want more contributors and want different points of view represented. Some
+parts of the code make contributing quick and easy. Other parts make it
+difficult and slow, so we ask that contributors have patience.
+
+To submit a patch, please open a merge request on GitLab. If you are thinking of
+making a large contribution, open an issue or merge request before starting
+work, to get comments from the community. Someone may be already working on the
+same thing, or there may be reasons why that feature isn't implemented. Once
+there is agreement, then the work might need to proceed asynchronously with the
+core team towards the solution.
+
+To make it easier to review and accept your merge request, please follow these
+guidelines:
+
+* When at all possible, include tests. These can either be added to an existing
+  test, or completely new. Practicing test-driven development will make it
+  easiest to get merged. That usually means starting your work by writing tests.
+
+* See [help-wanted](https://gitlab.com/fdroid/fdroidserver/-/issues/?sort=updated_desc&state=opened&label_name%5B%5D=help-wanted)
+  tags for things that maintainers have marked as things they want to see
+  merged.
+
+* The amount of technical debt varies widely in this code base. There are some
+  parts where the code is nicely isolated with good test coverage. There are
+  other parts that are tangled and complicated, full of technical debt, and
+  difficult to test.
+
+* The general approach is to treat the tangled and complicated parts as an
+  external API (albeit a bad one). That means it needs to stay unchanged as much
+  as possible. Changes to those parts of the code will trigger a migration,
+  which can require a lot of time and coordination. When there is time for large
+  development efforts, we refactor the code to get rid of those areas of
+  technical debt.
+
+* We use [_black_](https://black.readthedocs.io/) code format, run `black .` to
+  format the code. Whenever editing code in any file, the new code should be
+  formatted as _black_. Some files are not yet fully in _black_ format (see
+  _pyproject.toml_), our goal is to opportunistically convert the code whenever
+  possible. As of the time of this writing, forcing the code format on all files
+  would be too disruptive.  The officially supported _black_ version is the one
+  in Debian/stable.
+
+* Many of the tests run very fast and can be run interactively in isolation.
+  Some of the essential test cases run slowly because they do things like
+  signing files and generating signing keys.
+
+* Some parts of the code are difficult to test, and currently require a
+  relatively complete production setup in order to effectively test them. That
+  is mostly the code around building packages, managing the disposable VM, and
+  scheduling build jobs to run.
+
+* For user visible changes (API changes, behaviour changes, etc.), consider
+  adding a note in _CHANGELOG.md_. This could be a summarizing description of
+  the change, and could explain the grander details. Have a look through
+  existing entries for inspiration. Please note that this is NOT simply a copy
+  of git-log one-liners. Also note that security fixes get an entry in
+  _CHANGELOG.md_. This file helps users get more in-depth information of what
+  comes with a specific release without having to sift through the higher noise
+  ratio in git-log.

MANIFEST.in

@@ -0,0 +1,945 @@
+include buildserver/config.buildserver.yml
+include buildserver/provision-android-ndk
+include buildserver/provision-android-sdk
+include buildserver/provision-apt-get-install
+include buildserver/provision-apt-proxy
+include buildserver/provision-gradle
+include buildserver/setup-env-vars
+include buildserver/Vagrantfile
+include CHANGELOG.md
+include completion/bash-completion
+include examples/config.yml
+include examples/fdroid_exportkeystore.py
+include examples/fdroid_export_keystore_to_nitrokey.py
+include examples/fdroid_extract_repo_pubkey.py
+include examples/fdroid_fetchsrclibs.py
+include examples/fdroid_nitrokeyimport.py
+include examples/opensc-fdroid.cfg
+include examples/public-read-only-s3-bucket-policy.json
+include examples/template.yml
+include examples/Vagrantfile.yaml
+include gradlew-fdroid
+include LICENSE
+include locale/ba/LC_MESSAGES/fdroidserver.po
+include locale/bo/LC_MESSAGES/fdroidserver.po
+include locale/ca/LC_MESSAGES/fdroidserver.po
+include locale/cs/LC_MESSAGES/fdroidserver.po
+include locale/de/LC_MESSAGES/fdroidserver.po
+include locale/es/LC_MESSAGES/fdroidserver.po
+include locale/fr/LC_MESSAGES/fdroidserver.po
+include locale/ga/LC_MESSAGES/fdroidserver.po
+include locale/hu/LC_MESSAGES/fdroidserver.po
+include locale/it/LC_MESSAGES/fdroidserver.po
+include locale/ja/LC_MESSAGES/fdroidserver.po
+include locale/ko/LC_MESSAGES/fdroidserver.po
+include locale/nb_NO/LC_MESSAGES/fdroidserver.po
+include locale/pl/LC_MESSAGES/fdroidserver.po
+include locale/pt/LC_MESSAGES/fdroidserver.po
+include locale/pt_BR/LC_MESSAGES/fdroidserver.po
+include locale/pt_PT/LC_MESSAGES/fdroidserver.po
+include locale/ro/LC_MESSAGES/fdroidserver.po
+include locale/ru/LC_MESSAGES/fdroidserver.po
+include locale/sq/LC_MESSAGES/fdroidserver.po
+include locale/sr/LC_MESSAGES/fdroidserver.po
+include locale/sw/LC_MESSAGES/fdroidserver.po
+include locale/tr/LC_MESSAGES/fdroidserver.po
+include locale/uk/LC_MESSAGES/fdroidserver.po
+include locale/zh_Hans/LC_MESSAGES/fdroidserver.po
+include locale/zh_Hant/LC_MESSAGES/fdroidserver.po
+include makebuildserver
+include README.md
+include tests/aosp_testkey_debug.keystore
+include tests/apk.embedded_1.apk
+include tests/bad-unicode-*.apk
+include tests/build-tools/17.0.0/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/17.0.0/aapt-output-com.politedroid_3.txt
+include tests/build-tools/17.0.0/aapt-output-com.politedroid_4.txt
+include tests/build-tools/17.0.0/aapt-output-com.politedroid_5.txt
+include tests/build-tools/17.0.0/aapt-output-com.politedroid_6.txt
+include tests/build-tools/17.0.0/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/17.0.0/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/17.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/17.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/17.0.0/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/17.0.0/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/17.0.0/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/17.0.0/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/17.0.0/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/17.0.0/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/18.1.1/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/18.1.1/aapt-output-com.politedroid_3.txt
+include tests/build-tools/18.1.1/aapt-output-com.politedroid_4.txt
+include tests/build-tools/18.1.1/aapt-output-com.politedroid_5.txt
+include tests/build-tools/18.1.1/aapt-output-com.politedroid_6.txt
+include tests/build-tools/18.1.1/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/18.1.1/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/18.1.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/18.1.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/18.1.1/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/18.1.1/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/18.1.1/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/18.1.1/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/18.1.1/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/18.1.1/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/19.0.0/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/19.0.0/aapt-output-com.politedroid_3.txt
+include tests/build-tools/19.0.0/aapt-output-com.politedroid_4.txt
+include tests/build-tools/19.0.0/aapt-output-com.politedroid_5.txt
+include tests/build-tools/19.0.0/aapt-output-com.politedroid_6.txt
+include tests/build-tools/19.0.0/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/19.0.0/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/19.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/19.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/19.0.0/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/19.0.0/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/19.0.0/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/19.0.0/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/19.0.0/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/19.0.0/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/19.1.0/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/19.1.0/aapt-output-com.politedroid_3.txt
+include tests/build-tools/19.1.0/aapt-output-com.politedroid_4.txt
+include tests/build-tools/19.1.0/aapt-output-com.politedroid_5.txt
+include tests/build-tools/19.1.0/aapt-output-com.politedroid_6.txt
+include tests/build-tools/19.1.0/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/19.1.0/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/19.1.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/19.1.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/19.1.0/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/19.1.0/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/19.1.0/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/19.1.0/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/19.1.0/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/19.1.0/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/20.0.0/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/20.0.0/aapt-output-com.politedroid_3.txt
+include tests/build-tools/20.0.0/aapt-output-com.politedroid_4.txt
+include tests/build-tools/20.0.0/aapt-output-com.politedroid_5.txt
+include tests/build-tools/20.0.0/aapt-output-com.politedroid_6.txt
+include tests/build-tools/20.0.0/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/20.0.0/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/20.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/20.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/20.0.0/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/20.0.0/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/20.0.0/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/20.0.0/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/20.0.0/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/20.0.0/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/21.1.1/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/21.1.1/aapt-output-com.politedroid_3.txt
+include tests/build-tools/21.1.1/aapt-output-com.politedroid_4.txt
+include tests/build-tools/21.1.1/aapt-output-com.politedroid_5.txt
+include tests/build-tools/21.1.1/aapt-output-com.politedroid_6.txt
+include tests/build-tools/21.1.1/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/21.1.1/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/21.1.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/21.1.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/21.1.1/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/21.1.1/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/21.1.1/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/21.1.1/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/21.1.1/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/21.1.1/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/21.1.2/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/21.1.2/aapt-output-com.politedroid_3.txt
+include tests/build-tools/21.1.2/aapt-output-com.politedroid_4.txt
+include tests/build-tools/21.1.2/aapt-output-com.politedroid_5.txt
+include tests/build-tools/21.1.2/aapt-output-com.politedroid_6.txt
+include tests/build-tools/21.1.2/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/21.1.2/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/21.1.2/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/21.1.2/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/21.1.2/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/21.1.2/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/21.1.2/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/21.1.2/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/21.1.2/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/21.1.2/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/22.0.0/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/22.0.0/aapt-output-com.politedroid_3.txt
+include tests/build-tools/22.0.0/aapt-output-com.politedroid_4.txt
+include tests/build-tools/22.0.0/aapt-output-com.politedroid_5.txt
+include tests/build-tools/22.0.0/aapt-output-com.politedroid_6.txt
+include tests/build-tools/22.0.0/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/22.0.0/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/22.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/22.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/22.0.0/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/22.0.0/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/22.0.0/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/22.0.0/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/22.0.0/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/22.0.0/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/22.0.1/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/22.0.1/aapt-output-com.politedroid_3.txt
+include tests/build-tools/22.0.1/aapt-output-com.politedroid_4.txt
+include tests/build-tools/22.0.1/aapt-output-com.politedroid_5.txt
+include tests/build-tools/22.0.1/aapt-output-com.politedroid_6.txt
+include tests/build-tools/22.0.1/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/22.0.1/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/22.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/22.0.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/22.0.1/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/22.0.1/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/22.0.1/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/22.0.1/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/22.0.1/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/22.0.1/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/23.0.0/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/23.0.0/aapt-output-com.politedroid_3.txt
+include tests/build-tools/23.0.0/aapt-output-com.politedroid_4.txt
+include tests/build-tools/23.0.0/aapt-output-com.politedroid_5.txt
+include tests/build-tools/23.0.0/aapt-output-com.politedroid_6.txt
+include tests/build-tools/23.0.0/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/23.0.0/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/23.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/23.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/23.0.0/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/23.0.0/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/23.0.0/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/23.0.0/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/23.0.0/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/23.0.0/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/23.0.1/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/23.0.1/aapt-output-com.politedroid_3.txt
+include tests/build-tools/23.0.1/aapt-output-com.politedroid_4.txt
+include tests/build-tools/23.0.1/aapt-output-com.politedroid_5.txt
+include tests/build-tools/23.0.1/aapt-output-com.politedroid_6.txt
+include tests/build-tools/23.0.1/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/23.0.1/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/23.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/23.0.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/23.0.1/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/23.0.1/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/23.0.1/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/23.0.1/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/23.0.1/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/23.0.1/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/23.0.2/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/23.0.2/aapt-output-com.politedroid_3.txt
+include tests/build-tools/23.0.2/aapt-output-com.politedroid_4.txt
+include tests/build-tools/23.0.2/aapt-output-com.politedroid_5.txt
+include tests/build-tools/23.0.2/aapt-output-com.politedroid_6.txt
+include tests/build-tools/23.0.2/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/23.0.2/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/23.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/23.0.2/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/23.0.2/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/23.0.2/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/23.0.2/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/23.0.2/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/23.0.2/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/23.0.2/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/23.0.3/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/23.0.3/aapt-output-com.politedroid_3.txt
+include tests/build-tools/23.0.3/aapt-output-com.politedroid_4.txt
+include tests/build-tools/23.0.3/aapt-output-com.politedroid_5.txt
+include tests/build-tools/23.0.3/aapt-output-com.politedroid_6.txt
+include tests/build-tools/23.0.3/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/23.0.3/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/23.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/23.0.3/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/23.0.3/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/23.0.3/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/23.0.3/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/23.0.3/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/23.0.3/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/23.0.3/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/24.0.0/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/24.0.0/aapt-output-com.politedroid_3.txt
+include tests/build-tools/24.0.0/aapt-output-com.politedroid_4.txt
+include tests/build-tools/24.0.0/aapt-output-com.politedroid_5.txt
+include tests/build-tools/24.0.0/aapt-output-com.politedroid_6.txt
+include tests/build-tools/24.0.0/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/24.0.0/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/24.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/24.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/24.0.0/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/24.0.0/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/24.0.0/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/24.0.0/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/24.0.0/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/24.0.0/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/24.0.1/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/24.0.1/aapt-output-com.politedroid_3.txt
+include tests/build-tools/24.0.1/aapt-output-com.politedroid_4.txt
+include tests/build-tools/24.0.1/aapt-output-com.politedroid_5.txt
+include tests/build-tools/24.0.1/aapt-output-com.politedroid_6.txt
+include tests/build-tools/24.0.1/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/24.0.1/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/24.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/24.0.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/24.0.1/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/24.0.1/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/24.0.1/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/24.0.1/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/24.0.1/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/24.0.1/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/24.0.2/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/24.0.2/aapt-output-com.politedroid_3.txt
+include tests/build-tools/24.0.2/aapt-output-com.politedroid_4.txt
+include tests/build-tools/24.0.2/aapt-output-com.politedroid_5.txt
+include tests/build-tools/24.0.2/aapt-output-com.politedroid_6.txt
+include tests/build-tools/24.0.2/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/24.0.2/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/24.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/24.0.2/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/24.0.2/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/24.0.2/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/24.0.2/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/24.0.2/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/24.0.2/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/24.0.2/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/24.0.3/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/24.0.3/aapt-output-com.politedroid_3.txt
+include tests/build-tools/24.0.3/aapt-output-com.politedroid_4.txt
+include tests/build-tools/24.0.3/aapt-output-com.politedroid_5.txt
+include tests/build-tools/24.0.3/aapt-output-com.politedroid_6.txt
+include tests/build-tools/24.0.3/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/24.0.3/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/24.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/24.0.3/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/24.0.3/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/24.0.3/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/24.0.3/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/24.0.3/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/24.0.3/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/24.0.3/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/25.0.0/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/25.0.0/aapt-output-com.politedroid_3.txt
+include tests/build-tools/25.0.0/aapt-output-com.politedroid_4.txt
+include tests/build-tools/25.0.0/aapt-output-com.politedroid_5.txt
+include tests/build-tools/25.0.0/aapt-output-com.politedroid_6.txt
+include tests/build-tools/25.0.0/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/25.0.0/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/25.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/25.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/25.0.0/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/25.0.0/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/25.0.0/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/25.0.0/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/25.0.0/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/25.0.0/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/25.0.1/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/25.0.1/aapt-output-com.politedroid_3.txt
+include tests/build-tools/25.0.1/aapt-output-com.politedroid_4.txt
+include tests/build-tools/25.0.1/aapt-output-com.politedroid_5.txt
+include tests/build-tools/25.0.1/aapt-output-com.politedroid_6.txt
+include tests/build-tools/25.0.1/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/25.0.1/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/25.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/25.0.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/25.0.1/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/25.0.1/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/25.0.1/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/25.0.1/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/25.0.1/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/25.0.1/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/25.0.2/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/25.0.2/aapt-output-com.politedroid_3.txt
+include tests/build-tools/25.0.2/aapt-output-com.politedroid_4.txt
+include tests/build-tools/25.0.2/aapt-output-com.politedroid_5.txt
+include tests/build-tools/25.0.2/aapt-output-com.politedroid_6.txt
+include tests/build-tools/25.0.2/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/25.0.2/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/25.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/25.0.2/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/25.0.2/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/25.0.2/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/25.0.2/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/25.0.2/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/25.0.2/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/25.0.2/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/25.0.3/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/25.0.3/aapt-output-com.politedroid_3.txt
+include tests/build-tools/25.0.3/aapt-output-com.politedroid_4.txt
+include tests/build-tools/25.0.3/aapt-output-com.politedroid_5.txt
+include tests/build-tools/25.0.3/aapt-output-com.politedroid_6.txt
+include tests/build-tools/25.0.3/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/25.0.3/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/25.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/25.0.3/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/25.0.3/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/25.0.3/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/25.0.3/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/25.0.3/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/25.0.3/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/25.0.3/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/26.0.0/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/26.0.0/aapt-output-com.politedroid_3.txt
+include tests/build-tools/26.0.0/aapt-output-com.politedroid_4.txt
+include tests/build-tools/26.0.0/aapt-output-com.politedroid_5.txt
+include tests/build-tools/26.0.0/aapt-output-com.politedroid_6.txt
+include tests/build-tools/26.0.0/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/26.0.0/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/26.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/26.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/26.0.0/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/26.0.0/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/26.0.0/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/26.0.0/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/26.0.0/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/26.0.0/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/26.0.1/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/26.0.1/aapt-output-com.politedroid_3.txt
+include tests/build-tools/26.0.1/aapt-output-com.politedroid_4.txt
+include tests/build-tools/26.0.1/aapt-output-com.politedroid_5.txt
+include tests/build-tools/26.0.1/aapt-output-com.politedroid_6.txt
+include tests/build-tools/26.0.1/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/26.0.1/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/26.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/26.0.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/26.0.1/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/26.0.1/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/26.0.1/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/26.0.1/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/26.0.1/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/26.0.1/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/26.0.2/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/26.0.2/aapt-output-com.politedroid_3.txt
+include tests/build-tools/26.0.2/aapt-output-com.politedroid_4.txt
+include tests/build-tools/26.0.2/aapt-output-com.politedroid_5.txt
+include tests/build-tools/26.0.2/aapt-output-com.politedroid_6.txt
+include tests/build-tools/26.0.2/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/26.0.2/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/26.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/26.0.2/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/26.0.2/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/26.0.2/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/26.0.2/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/26.0.2/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/26.0.2/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/26.0.2/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/26.0.3/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/26.0.3/aapt-output-com.politedroid_3.txt
+include tests/build-tools/26.0.3/aapt-output-com.politedroid_4.txt
+include tests/build-tools/26.0.3/aapt-output-com.politedroid_5.txt
+include tests/build-tools/26.0.3/aapt-output-com.politedroid_6.txt
+include tests/build-tools/26.0.3/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/26.0.3/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/26.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/26.0.3/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/26.0.3/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/26.0.3/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/26.0.3/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/26.0.3/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/26.0.3/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/26.0.3/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/27.0.0/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/27.0.0/aapt-output-com.politedroid_3.txt
+include tests/build-tools/27.0.0/aapt-output-com.politedroid_4.txt
+include tests/build-tools/27.0.0/aapt-output-com.politedroid_5.txt
+include tests/build-tools/27.0.0/aapt-output-com.politedroid_6.txt
+include tests/build-tools/27.0.0/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/27.0.0/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/27.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/27.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/27.0.0/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/27.0.0/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/27.0.0/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/27.0.0/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/27.0.0/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/27.0.0/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/27.0.1/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/27.0.1/aapt-output-com.politedroid_3.txt
+include tests/build-tools/27.0.1/aapt-output-com.politedroid_4.txt
+include tests/build-tools/27.0.1/aapt-output-com.politedroid_5.txt
+include tests/build-tools/27.0.1/aapt-output-com.politedroid_6.txt
+include tests/build-tools/27.0.1/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/27.0.1/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/27.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/27.0.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/27.0.1/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/27.0.1/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/27.0.1/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/27.0.1/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/27.0.1/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/27.0.1/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/27.0.2/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/27.0.2/aapt-output-com.politedroid_3.txt
+include tests/build-tools/27.0.2/aapt-output-com.politedroid_4.txt
+include tests/build-tools/27.0.2/aapt-output-com.politedroid_5.txt
+include tests/build-tools/27.0.2/aapt-output-com.politedroid_6.txt
+include tests/build-tools/27.0.2/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/27.0.2/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/27.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/27.0.2/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/27.0.2/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/27.0.2/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/27.0.2/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/27.0.2/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/27.0.2/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/27.0.2/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/27.0.3/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/27.0.3/aapt-output-com.politedroid_3.txt
+include tests/build-tools/27.0.3/aapt-output-com.politedroid_4.txt
+include tests/build-tools/27.0.3/aapt-output-com.politedroid_5.txt
+include tests/build-tools/27.0.3/aapt-output-com.politedroid_6.txt
+include tests/build-tools/27.0.3/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/27.0.3/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/27.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/27.0.3/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/27.0.3/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/27.0.3/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/27.0.3/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/27.0.3/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/27.0.3/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/27.0.3/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/28.0.0/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/28.0.0/aapt-output-com.politedroid_3.txt
+include tests/build-tools/28.0.0/aapt-output-com.politedroid_4.txt
+include tests/build-tools/28.0.0/aapt-output-com.politedroid_5.txt
+include tests/build-tools/28.0.0/aapt-output-com.politedroid_6.txt
+include tests/build-tools/28.0.0/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/28.0.0/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/28.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/28.0.0/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/28.0.0/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/28.0.0/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/28.0.0/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/28.0.0/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/28.0.0/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/28.0.0/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/28.0.1/aapt-output-com.moez.QKSMS_182.txt
+include tests/build-tools/28.0.1/aapt-output-com.politedroid_3.txt
+include tests/build-tools/28.0.1/aapt-output-com.politedroid_4.txt
+include tests/build-tools/28.0.1/aapt-output-com.politedroid_5.txt
+include tests/build-tools/28.0.1/aapt-output-com.politedroid_6.txt
+include tests/build-tools/28.0.1/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/28.0.1/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/28.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/28.0.1/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/28.0.1/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/28.0.1/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/28.0.1/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/28.0.1/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/28.0.1/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/28.0.1/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/28.0.2/aapt-output-com.politedroid_3.txt
+include tests/build-tools/28.0.2/aapt-output-com.politedroid_4.txt
+include tests/build-tools/28.0.2/aapt-output-com.politedroid_5.txt
+include tests/build-tools/28.0.2/aapt-output-com.politedroid_6.txt
+include tests/build-tools/28.0.2/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/28.0.2/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/28.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/28.0.2/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/28.0.2/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/28.0.2/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/28.0.2/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/28.0.2/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/28.0.2/aapt-output-org.droidtr.keyboard_34.txt
+include tests/build-tools/28.0.2/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/28.0.3/aapt-output-com.example.test.helloworld_1.txt
+include tests/build-tools/28.0.3/aapt-output-com.politedroid_3.txt
+include tests/build-tools/28.0.3/aapt-output-com.politedroid_4.txt
+include tests/build-tools/28.0.3/aapt-output-com.politedroid_5.txt
+include tests/build-tools/28.0.3/aapt-output-com.politedroid_6.txt
+include tests/build-tools/28.0.3/aapt-output-duplicate.permisssions_9999999.txt
+include tests/build-tools/28.0.3/aapt-output-info.guardianproject.urzip_100.txt
+include tests/build-tools/28.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt
+include tests/build-tools/28.0.3/aapt-output-no.min.target.sdk_987.txt
+include tests/build-tools/28.0.3/aapt-output-obb.main.oldversion_1444412523.txt
+include tests/build-tools/28.0.3/aapt-output-obb.main.twoversions_1101613.txt
+include tests/build-tools/28.0.3/aapt-output-obb.main.twoversions_1101615.txt
+include tests/build-tools/28.0.3/aapt-output-obb.main.twoversions_1101617.txt
+include tests/build-tools/28.0.3/aapt-output-obb.mainpatch.current_1619.txt
+include tests/build-tools/28.0.3/aapt-output-souch.smsbypass_9.txt
+include tests/build-tools/generate.sh
+include tests/check-fdroid-apk
+include tests/com.fake.IpaApp_1000000000001.ipa
+include tests/config.yml
+include tests/config/antiFeatures.yml
+include tests/config/categories.yml
+include tests/config/de/antiFeatures.yml
+include tests/config/fa/antiFeatures.yml
+include tests/config/ic_antifeature_ads.xml
+include tests/config/ic_antifeature_disabledalgorithm.xml
+include tests/config/ic_antifeature_knownvuln.xml
+include tests/config/ic_antifeature_nonfreeadd.xml
+include tests/config/ic_antifeature_nonfreeassets.xml
+include tests/config/ic_antifeature_nonfreedep.xml
+include tests/config/ic_antifeature_nonfreenet.xml
+include tests/config/ic_antifeature_nosourcesince.xml
+include tests/config/ic_antifeature_nsfw.xml
+include tests/config/ic_antifeature_tracking.xml
+include tests/config/ic_antifeature_upstreamnonfree.xml
+include tests/config/ro/antiFeatures.yml
+include tests/config/zh-rCN/antiFeatures.yml
+include tests/corrupt-featureGraphic.png
+include tests/dummy-keystore.jks
+include tests/dump_internal_metadata_format.py
+include tests/extra/manual-vmtools-test.py
+include tests/funding-usernames.yaml
+include tests/get_android_tools_versions/android-ndk-r10e/RELEASE.TXT
+include tests/get_android_tools_versions/android-sdk/ndk-bundle/package.xml
+include tests/get_android_tools_versions/android-sdk/ndk-bundle/source.properties
+include tests/get_android_tools_versions/android-sdk/ndk/11.2.2725575/source.properties
+include tests/get_android_tools_versions/android-sdk/ndk/17.2.4988734/source.properties
+include tests/get_android_tools_versions/android-sdk/ndk/21.3.6528147/source.properties
+include tests/get_android_tools_versions/android-sdk/patcher/v4/source.properties
+include tests/get_android_tools_versions/android-sdk/platforms/android-30/source.properties
+include tests/get_android_tools_versions/android-sdk/skiaparser/1/source.properties
+include tests/get_android_tools_versions/android-sdk/tools/source.properties
+include tests/gnupghome/pubring.gpg
+include tests/gnupghome/random_seed
+include tests/gnupghome/secring.gpg
+include tests/gnupghome/trustdb.gpg
+include tests/gradle-maven-blocks.yaml
+include tests/gradle-release-checksums.py
+include tests/IsMD5Disabled.java
+include tests/issue-1128-min-sdk-30-poc.apk
+include tests/issue-1128-poc1.apk
+include tests/issue-1128-poc2.apk
+include tests/issue-1128-poc3a.apk
+include tests/issue-1128-poc3b.apk
+include tests/janus.apk
+include tests/key-tricks.py
+include tests/keystore.jks
+include tests/metadata-rewrite-yml/app.with.special.build.params.yml
+include tests/metadata-rewrite-yml/fake.ota.update.yml
+include tests/metadata-rewrite-yml/org.fdroid.fdroid.yml
+include tests/metadata/apk/info.guardianproject.urzip.yaml
+include tests/metadata/apk/org.dyndns.fules.ck.yaml
+include tests/metadata/app.with.special.build.params.yml
+include tests/metadata/app.with.special.build.params/en-US/antifeatures/50_Ads.txt
+include tests/metadata/app.with.special.build.params/en-US/antifeatures/50_Tracking.txt
+include tests/metadata/app.with.special.build.params/en-US/antifeatures/Ads.txt
+include tests/metadata/app.with.special.build.params/en-US/antifeatures/NoSourceSince.txt
+include tests/metadata/app.with.special.build.params/zh-CN/antifeatures/49_Tracking.txt
+include tests/metadata/app.with.special.build.params/zh-CN/antifeatures/50_Ads.txt
+include tests/metadata/com.politedroid.yml
+include tests/metadata/dump/app.with.special.build.params.yaml
+include tests/metadata/dump/com.politedroid.yaml
+include tests/metadata/dump/org.adaway.yaml
+include tests/metadata/dump/org.smssecure.smssecure.yaml
+include tests/metadata/dump/org.videolan.vlc.yaml
+include tests/metadata/duplicate.permisssions.yml
+include tests/metadata/fake.ota.update.yml
+include tests/metadata/info.guardianproject.checkey.yml
+include tests/metadata/info.guardianproject.checkey/en-US/description.txt
+include tests/metadata/info.guardianproject.checkey/en-US/name.txt
+include tests/metadata/info.guardianproject.checkey/en-US/phoneScreenshots/checkey-phone.png
+include tests/metadata/info.guardianproject.checkey/en-US/phoneScreenshots/checkey.png
+include tests/metadata/info.guardianproject.checkey/en-US/summary.txt
+include tests/metadata/info.guardianproject.checkey/ja-JP/name.txt
+include tests/metadata/info.guardianproject.urzip.yml
+include tests/metadata/info.guardianproject.urzip/en-US/changelogs/100.txt
+include tests/metadata/info.guardianproject.urzip/en-US/changelogs/default.txt
+include tests/metadata/info.guardianproject.urzip/en-US/full_description.txt
+include tests/metadata/info.guardianproject.urzip/en-US/images/featureGraphic.png
+include tests/metadata/info.guardianproject.urzip/en-US/images/icon.png
+include tests/metadata/info.guardianproject.urzip/en-US/short_description.txt
+include tests/metadata/info.guardianproject.urzip/en-US/title.txt
+include tests/metadata/info.guardianproject.urzip/en-US/video.txt
+include tests/metadata/info.zwanenburg.caffeinetile.yml
+include tests/metadata/no.min.target.sdk.yml
+include tests/metadata/obb.main.oldversion.yml
+include tests/metadata/obb.main.twoversions.yml
+include tests/metadata/obb.mainpatch.current.yml
+include tests/metadata/org.adaway.yml
+include tests/metadata/org.fdroid.ci.test.app.yml
+include tests/metadata/org.fdroid.fdroid.yml
+include tests/metadata/org.maxsdkversion.yml
+include tests/metadata/org.smssecure.smssecure.yml
+include tests/metadata/org.smssecure.smssecure/signatures/134/28969C09.RSA
+include tests/metadata/org.smssecure.smssecure/signatures/134/28969C09.SF
+include tests/metadata/org.smssecure.smssecure/signatures/134/MANIFEST.MF
+include tests/metadata/org.smssecure.smssecure/signatures/135/28969C09.RSA
+include tests/metadata/org.smssecure.smssecure/signatures/135/28969C09.SF
+include tests/metadata/org.smssecure.smssecure/signatures/135/MANIFEST.MF
+include tests/metadata/org.videolan.vlc.yml
+include tests/metadata/raw.template.yml
+include tests/metadata/souch.smsbypass.yml
+include tests/minimal_targetsdk_30_unsigned.apk
+include tests/Norway_bouvet_europe_2.obf.zip
+include tests/no_targetsdk_minsdk1_unsigned.apk
+include tests/no_targetsdk_minsdk30_unsigned.apk
+include tests/openssl-version-check-test.py
+include tests/org.bitbucket.tickytacky.mirrormirror_1.apk
+include tests/org.bitbucket.tickytacky.mirrormirror_2.apk
+include tests/org.bitbucket.tickytacky.mirrormirror_3.apk
+include tests/org.bitbucket.tickytacky.mirrormirror_4.apk
+include tests/org.dyndns.fules.ck_20.apk
+include tests/org.sajeg.fallingblocks_3.apk
+include tests/repo/com.example.test.helloworld_1.apk
+include tests/repo/com.politedroid_3.apk
+include tests/repo/com.politedroid_4.apk
+include tests/repo/com.politedroid_5.apk
+include tests/repo/com.politedroid_6.apk
+include tests/repo/duplicate.permisssions_9999999.apk
+include tests/repo/entry.json
+include tests/repo/fake.ota.update_1234.zip
+include tests/repo/index-v1.json
+include tests/repo/index-v2.json
+include tests/repo/index.xml
+include tests/repo/info.zwanenburg.caffeinetile_4.apk
+include tests/repo/main.1101613.obb.main.twoversions.obb
+include tests/repo/main.1101615.obb.main.twoversions.obb
+include tests/repo/main.1434483388.obb.main.oldversion.obb
+include tests/repo/main.1619.obb.mainpatch.current.obb
+include tests/repo/no.min.target.sdk_987.apk
+include tests/repo/obb.main.oldversion_1444412523.apk
+include tests/repo/obb.main.twoversions_1101613.apk
+include tests/repo/obb.main.twoversions_1101615.apk
+include tests/repo/obb.main.twoversions_1101617.apk
+include tests/repo/obb.main.twoversions_1101617_src.tar.gz
+include tests/repo/obb.mainpatch.current/en-US/featureGraphic.png
+include tests/repo/obb.mainpatch.current/en-US/icon.png
+include tests/repo/obb.mainpatch.current/en-US/phoneScreenshots/screenshot-main.png
+include tests/repo/obb.mainpatch.current/en-US/sevenInchScreenshots/screenshot-tablet-main.png
+include tests/repo/obb.mainpatch.current_1619.apk
+include tests/repo/obb.mainpatch.current_1619_another-release-key.apk
+include tests/repo/org.maxsdkversion_4.apk
+include tests/repo/org.videolan.vlc/en-US/icon.png
+include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot10.png
+include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot12.png
+include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot15.png
+include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot18.png
+include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot20.png
+include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot22.png
+include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot4.png
+include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot7.png
+include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot9.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot0.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot1.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot11.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot13.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot14.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot16.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot17.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot19.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot2.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot21.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot23.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot3.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot5.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot6.png
+include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot8.png
+include tests/repo/patch.1619.obb.mainpatch.current.obb
+include tests/repo/souch.smsbypass_9.apk
+include tests/repo/urzip-*.apk
+include tests/repo/v1.v2.sig_1020.apk
+include tests/run-tests
+include tests/SANAPPSI.RSA
+include tests/SANAPPSI.SF
+include tests/shared_test_code.py
+include tests/signindex/guardianproject-v1.jar
+include tests/signindex/guardianproject.jar
+include tests/signindex/testy.jar
+include tests/signindex/unsigned.jar
+include tests/source-files/at.bitfire.davdroid/build.gradle
+include tests/source-files/catalog.test/app/build.gradle
+include tests/source-files/catalog.test/build.gradle.kts
+include tests/source-files/catalog.test/buildSrc/build.gradle.kts
+include tests/source-files/catalog.test/buildSrc/settings.gradle.kts
+include tests/source-files/catalog.test/buildSrc2/build.gradle.kts
+include tests/source-files/catalog.test/buildSrc2/settings.gradle.kts
+include tests/source-files/catalog.test/core/build.gradle
+include tests/source-files/catalog.test/gradle/libs.versions.toml
+include tests/source-files/catalog.test/libs.versions.toml
+include tests/source-files/catalog.test/settings.gradle.kts
+include tests/source-files/cn.wildfirechat.chat/avenginekit/build.gradle
+include tests/source-files/cn.wildfirechat.chat/build.gradle
+include tests/source-files/cn.wildfirechat.chat/chat/build.gradle
+include tests/source-files/cn.wildfirechat.chat/client/build.gradle
+include tests/source-files/cn.wildfirechat.chat/client/src/main/AndroidManifest.xml
+include tests/source-files/cn.wildfirechat.chat/emojilibrary/build.gradle
+include tests/source-files/cn.wildfirechat.chat/gradle/build_libraries.gradle
+include tests/source-files/cn.wildfirechat.chat/imagepicker/build.gradle
+include tests/source-files/cn.wildfirechat.chat/mars-core-release/build.gradle
+include tests/source-files/cn.wildfirechat.chat/push/build.gradle
+include tests/source-files/cn.wildfirechat.chat/settings.gradle
+include tests/source-files/com.anpmech.launcher/app/build.gradle
+include tests/source-files/com.anpmech.launcher/app/src/main/AndroidManifest.xml
+include tests/source-files/com.anpmech.launcher/build.gradle
+include tests/source-files/com.anpmech.launcher/settings.gradle
+include tests/source-files/com.github.jameshnsears.quoteunquote/build.gradle
+include tests/source-files/com.github.shadowsocks/core/build.gradle.kts
+include tests/source-files/com.github.shadowsocks/mobile/build.gradle.kts
+include tests/source-files/com.infomaniak.mail/Core/gradle/core.versions.toml
+include tests/source-files/com.infomaniak.mail/gradle/libs.versions.toml
+include tests/source-files/com.infomaniak.mail/settings.gradle
+include tests/source-files/com.integreight.onesheeld/build.gradle
+include tests/source-files/com.integreight.onesheeld/gradle/wrapper/gradle-wrapper.properties
+include tests/source-files/com.integreight.onesheeld/localeapi/build.gradle
+include tests/source-files/com.integreight.onesheeld/localeapi/src/main/AndroidManifest.xml
+include tests/source-files/com.integreight.onesheeld/oneSheeld/build.gradle
+include tests/source-files/com.integreight.onesheeld/oneSheeld/src/main/AndroidManifest.xml
+include tests/source-files/com.integreight.onesheeld/pagerIndicator/build.gradle
+include tests/source-files/com.integreight.onesheeld/pagerIndicator/src/main/AndroidManifest.xml
+include tests/source-files/com.integreight.onesheeld/pullToRefreshlibrary/build.gradle
+include tests/source-files/com.integreight.onesheeld/pullToRefreshlibrary/src/main/AndroidManifest.xml
+include tests/source-files/com.integreight.onesheeld/quickReturnHeader/build.gradle
+include tests/source-files/com.integreight.onesheeld/quickReturnHeader/src/main/AndroidManifest.xml
+include tests/source-files/com.integreight.onesheeld/settings.gradle
+include tests/source-files/com.jens.automation2/app/build.gradle
+include tests/source-files/com.jens.automation2/build.gradle
+include tests/source-files/com.kunzisoft.testcase/build.gradle
+include tests/source-files/com.lolo.io.onelist/app/build.gradle.kts
+include tests/source-files/com.lolo.io.onelist/build.gradle.kts
+include tests/source-files/com.lolo.io.onelist/gradle/libs.versions.toml
+include tests/source-files/com.lolo.io.onelist/gradle/wrapper/gradle-wrapper.properties
+include tests/source-files/com.lolo.io.onelist/settings.gradle
+include tests/source-files/com.nextcloud.client.dev/src/generic/fastlane/metadata/android/en-US/full_description.txt
+include tests/source-files/com.nextcloud.client.dev/src/generic/fastlane/metadata/android/en-US/short_description.txt
+include tests/source-files/com.nextcloud.client.dev/src/generic/fastlane/metadata/android/en-US/title.txt
+include tests/source-files/com.nextcloud.client.dev/src/versionDev/fastlane/metadata/android/en-US/full_description.txt
+include tests/source-files/com.nextcloud.client.dev/src/versionDev/fastlane/metadata/android/en-US/short_description.txt
+include tests/source-files/com.nextcloud.client.dev/src/versionDev/fastlane/metadata/android/en-US/title.txt
+include tests/source-files/com.nextcloud.client/build.gradle
+include tests/source-files/com.nextcloud.client/src/generic/fastlane/metadata/android/en-US/full_description.txt
+include tests/source-files/com.nextcloud.client/src/generic/fastlane/metadata/android/en-US/short_description.txt
+include tests/source-files/com.nextcloud.client/src/generic/fastlane/metadata/android/en-US/title.txt
+include tests/source-files/com.nextcloud.client/src/versionDev/fastlane/metadata/android/en-US/full_description.txt
+include tests/source-files/com.nextcloud.client/src/versionDev/fastlane/metadata/android/en-US/short_description.txt
+include tests/source-files/com.nextcloud.client/src/versionDev/fastlane/metadata/android/en-US/title.txt
+include tests/source-files/com.seafile.seadroid2/app/build.gradle
+include tests/source-files/com.ubergeek42.WeechatAndroid/app/build.gradle.kts
+include tests/source-files/com.ubergeek42.WeechatAndroid/app/src/main/res/values/strings.xml
+include tests/source-files/de.varengold.activeTAN/build.gradle
+include tests/source-files/dev.patrickgold.florisboard/app/build.gradle.kts
+include tests/source-files/eu.siacs.conversations/build.gradle
+include tests/source-files/eu.siacs.conversations/metadata/en-US/name.txt
+include tests/source-files/fdroid/fdroidclient/AndroidManifest.xml
+include tests/source-files/fdroid/fdroidclient/build.gradle
+include tests/source-files/firebase-allowlisted/app/build.gradle
+include tests/source-files/firebase-allowlisted/build.gradle
+include tests/source-files/firebase-suspect/app/build.gradle
+include tests/source-files/firebase-suspect/build.gradle
+include tests/source-files/flavor.test/build.gradle
+include tests/source-files/info.guardianproject.ripple/build.gradle
+include tests/source-files/lockfile.test/flutter/.dart_tool/flutter_gen/pubspec.yaml
+include tests/source-files/lockfile.test/flutter/pubspec.lock
+include tests/source-files/lockfile.test/flutter/pubspec.yaml
+include tests/source-files/lockfile.test/javascript/package.json
+include tests/source-files/lockfile.test/javascript/yarn.lock
+include tests/source-files/lockfile.test/rust/subdir/Cargo.lock
+include tests/source-files/lockfile.test/rust/subdir/Cargo.toml
+include tests/source-files/lockfile.test/rust/subdir/subdir/subdir/Cargo.toml
+include tests/source-files/lockfile.test/rust/subdir2/Cargo.toml
+include tests/source-files/open-keychain/open-keychain/build.gradle
+include tests/source-files/open-keychain/open-keychain/OpenKeychain/build.gradle
+include tests/source-files/org.mozilla.rocket/app/build.gradle
+include tests/source-files/org.noise_planet.noisecapture/app/build.gradle
+include tests/source-files/org.noise_planet.noisecapture/settings.gradle
+include tests/source-files/org.noise_planet.noisecapture/sosfilter/build.gradle
+include tests/source-files/org.piepmeyer.gauguin/build.gradle.kts
+include tests/source-files/org.piepmeyer.gauguin/libs.versions.toml
+include tests/source-files/org.piepmeyer.gauguin/settings.gradle.kts
+include tests/source-files/org.tasks/app/build.gradle.kts
+include tests/source-files/org.tasks/build.gradle
+include tests/source-files/org.tasks/build.gradle.kts
+include tests/source-files/org.tasks/buildSrc/build.gradle.kts
+include tests/source-files/org.tasks/settings.gradle.kts
+include tests/source-files/osmandapp/osmand/build.gradle
+include tests/source-files/osmandapp/osmand/gradle/wrapper/gradle-wrapper.properties
+include tests/source-files/OtakuWorld/build.gradle
+include tests/source-files/realm/react-native/android/build.gradle
+include tests/source-files/se.manyver/android/app/build.gradle
+include tests/source-files/se.manyver/android/build.gradle
+include tests/source-files/se.manyver/android/gradle.properties
+include tests/source-files/se.manyver/android/gradle/wrapper/gradle-wrapper.properties
+include tests/source-files/se.manyver/android/settings.gradle
+include tests/source-files/se.manyver/app.json
+include tests/source-files/se.manyver/index.android.js
+include tests/source-files/se.manyver/package.json
+include tests/source-files/se.manyver/react-native.config.js
+include tests/source-files/ut.ewh.audiometrytest/app/build.gradle
+include tests/source-files/ut.ewh.audiometrytest/app/src/main/AndroidManifest.xml
+include tests/source-files/ut.ewh.audiometrytest/build.gradle
+include tests/source-files/ut.ewh.audiometrytest/settings.gradle
+include tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties
+include tests/source-files/Zillode/syncthing-silk/build.gradle
+include tests/SpeedoMeterApp.main_1.apk
+include tests/test_build.py
+include tests/test_checkupdates.py
+include tests/test_common.py
+include tests/test_deploy.py
+include tests/test_exception.py
+include tests/test_gradlew-fdroid
+include tests/test_import_subcommand.py
+include tests/test_index.py
+include tests/test_init.py
+include tests/test_install.py
+include tests/test_lint.py
+include tests/test_main.py
+include tests/test_metadata.py
+include tests/test_nightly.py
+include tests/test_publish.py
+include tests/test_rewritemeta.py
+include tests/test_scanner.py
+include tests/test_signatures.py
+include tests/test_signindex.py
+include tests/test_update.py
+include tests/test_vcs.py
+include tests/triple-t-1-graphics/build/de.wivewa.dialer/app/src/main/play/en-US/listing/featureGraphic/play_store_feature_graphic.png
+include tests/triple-t-1-graphics/build/de.wivewa.dialer/app/src/main/play/en-US/listing/icon/icon.png
+include tests/triple-t-1-graphics/build/de.wivewa.dialer/app/src/main/play/en-US/listing/phoneScreenshots/1.png
+include tests/triple-t-1-graphics/metadata/de.wivewa.dialer.yml
+include tests/triple-t-2/build/org.piwigo.android/app/.gitignore
+include tests/triple-t-2/build/org.piwigo.android/app/build.gradle
+include tests/triple-t-2/build/org.piwigo.android/app/src/debug/res/values/constants.xml
+include tests/triple-t-2/build/org.piwigo.android/app/src/debug/res/values/strings.xml
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/java/org/piwigo/PiwigoApplication.java
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/contact-email.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/contact-website.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/default-language.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/de-DE/full-description.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/de-DE/short-description.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/de-DE/title.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/en-US/full-description.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/en-US/graphics/feature-graphic/piwigo-full.png
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/en-US/graphics/icon/piwigo-icon.png
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/en-US/graphics/phone-screenshots/01_Login.jpg
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/en-US/graphics/phone-screenshots/02_Albums.jpg
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/en-US/graphics/phone-screenshots/03_Photos.jpg
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/en-US/graphics/phone-screenshots/04_Albums_horizontal.jpg
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/en-US/graphics/phone-screenshots/05_Menu.jpg
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/en-US/graphics/tablet-screenshots/01_Login.png
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/en-US/short-description.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/en-US/title.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/fr-FR/full-description.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/fr-FR/short-description.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/fr-FR/title.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/kn-IN/full-description.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/kn-IN/short-description.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/listings/kn-IN/title.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/release-notes/de-DE/default.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/release-notes/en-US/default.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/release-notes/fr-FR/default.txt
+include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/release-notes/kn-IN/default.txt
+include tests/triple-t-2/build/org.piwigo.android/build.gradle
+include tests/triple-t-2/build/org.piwigo.android/settings.gradle
+include tests/triple-t-2/metadata/org.piwigo.android.yml
+include tests/triple-t-anysoftkeyboard/build/com.anysoftkeyboard.languagepack.dutch/addons/languages/dutch/apk/src/main/play/listings/en-US/title.txt
+include tests/triple-t-anysoftkeyboard/build/com.anysoftkeyboard.languagepack.dutch/ime/app/src/main/play/listings/en-US/title.txt
+include tests/triple-t-anysoftkeyboard/build/com.anysoftkeyboard.languagepack.dutch/settings.gradle
+include tests/triple-t-anysoftkeyboard/build/com.menny.android.anysoftkeyboard/addons/languages/dutch/apk/src/main/play/listings/en-US/title.txt
+include tests/triple-t-anysoftkeyboard/build/com.menny.android.anysoftkeyboard/ime/app/src/main/play/listings/en-US/title.txt
+include tests/triple-t-anysoftkeyboard/build/com.menny.android.anysoftkeyboard/settings.gradle
+include tests/triple-t-anysoftkeyboard/metadata/com.anysoftkeyboard.languagepack.dutch.yml
+include tests/triple-t-anysoftkeyboard/metadata/com.menny.android.anysoftkeyboard.yml
+include tests/triple-t-flutter/build/fr.emersion.goguma/android/app/src/main/play/contact-website.txt
+include tests/triple-t-flutter/build/fr.emersion.goguma/android/app/src/main/play/listings/en-US/full-description.txt
+include tests/triple-t-flutter/build/fr.emersion.goguma/android/app/src/main/play/listings/en-US/short-description.txt
+include tests/triple-t-flutter/build/fr.emersion.goguma/android/app/src/main/play/listings/en-US/title.txt
+include tests/triple-t-flutter/metadata/fr.emersion.goguma.yml
+include tests/triple-t-multiple/build/ch.admin.bag.covidcertificate.verifier/settings.gradle
+include tests/triple-t-multiple/build/ch.admin.bag.covidcertificate.verifier/verifier/src/main/play/listings/en-US/title.txt
+include tests/triple-t-multiple/build/ch.admin.bag.covidcertificate.verifier/wallet/src/main/play/listings/en-US/title.txt
+include tests/triple-t-multiple/build/ch.admin.bag.covidcertificate.wallet/settings.gradle
+include tests/triple-t-multiple/build/ch.admin.bag.covidcertificate.wallet/verifier/src/main/play/listings/en-US/title.txt
+include tests/triple-t-multiple/build/ch.admin.bag.covidcertificate.wallet/wallet/src/main/play/listings/en-US/title.txt
+include tests/triple-t-multiple/metadata/ch.admin.bag.covidcertificate.verifier.yml
+include tests/triple-t-multiple/metadata/ch.admin.bag.covidcertificate.wallet.yml
+include tests/urzip-badcert.apk
+include tests/urzip-badsig.apk
+include tests/urzip-release-unsigned.apk
+include tests/urzip-release.apk
+include tests/urzip.apk
+include tests/v2.only.sig_2.apk
+include tests/valid-package-names/random-package-names
+include tests/valid-package-names/RandomPackageNames.java
+include tests/valid-package-names/test.py
+include tests/__init__.py

README

@@ -1,12 +0,0 @@
-F-Droid is an installable catalogue of FOSS (Free and Open Source Software)
-applications for the Android platform. The client makes it easy to browse,
-install, and keep track of updates on your device.
-
-The F-Droid server tools provide various scripts and tools that are used to
-maintain the main F-Droid application repository. You can use these same tools
-to create your own additional or alternative repository for publishing, or to
-assist in creating, testing and submitting metadata to the main repository.
-
-For documentation, please see the docs directory.
-
-Alternatively, visit http://f-droid.org/manual/

README.md

@@ -0,0 +1,133 @@
+<div align="center">
+
+<p><img src="https://gitlab.com/fdroid/artwork/-/raw/master/fdroid-logo-2015/fdroid-logo.svg" width="200"></p>
+
+# F-Droid Server
+### Tools for maintaining an F-Droid repository system.
+
+</div>
+
+---
+
+## What is F-Droid Server?
+
+_fdroidserver_ is a suite of tools to publish and work with collections of
+Android apps (APK files) and other kinds of packages.  It is used to maintain
+the [f-droid.org application repository](https://f-droid.org/packages).  These
+same tools can be used to create additional or alternative repositories for
+publishing, or to assist in creating, testing and submitting metadata to the
+f-droid.org repository, also known as
+[_fdroiddata_](https://gitlab.com/fdroid/fdroiddata).
+
+For documentation, please see <https://f-droid.org/docs>.
+
+In the beginning, _fdroidserver_ was the complete server-side setup that ran
+f-droid.org.  Since then, the website and other parts have been split out into
+their own projects.  The name for this suite of tooling has stayed
+_fdroidserver_ even though it no longer contains any proper server component.
+
+
+## Installing
+
+There are many ways to install _fdroidserver_, including using a range of
+package managers.  All of the options are documented on the website:
+https://f-droid.org/docs/Installing_the_Server_and_Repo_Tools
+
+
+## Releases
+
+The production setup of _fdroidserver_ for f-droid.org is run directly from the
+_master_ branch.  This is put into production on an schedule (currently weekly).
+So development and testing happens in the branches. We track branches using
+merge requests.  Therefore, there are many WIP and long-lived merge requests.
+
+There are also stable releases of _fdroidserver_.  This is mostly intended for
+running custom repositories, where the build process is separate.  It can also
+be useful as a simple way to get started contributing packages to _fdroiddata_,
+since the stable releases are available in package managers.
+
+
+## Tests
+
+To run the full test suite:
+
+    tests/run-tests
+
+To run the tests for individual Python modules, see the `tests/test_*.py` files, e.g.:
+
+    python -m unittest tests/test_metadata.py
+
+It is also possible to run individual tests:
+
+    python -m unittest tests.test_metadata.MetadataTest.test_rewrite_yaml_special_build_params
+
+There is a growing test suite that has good coverage on a number of key parts of
+this code base.  It does not yet cover all the code, and there are some parts
+where the technical debt makes it difficult to write unit tests.  New tests
+should be standard Python _unittest_ test cases.  Whenever possible, the old
+tests written in _bash_ in _tests/run-tests_ should be ported to Python.
+
+This test suite has built over time a bit haphazardly, so it is not as clean,
+organized, or complete as it could be.  We welcome contributions.  The goal is
+to move towards standard Python testing patterns and to expand the unit test
+coverage.  Before rearchitecting any parts of it, be sure to [contact
+us](https://f-droid.org/about) to discuss the changes beforehand.
+
+
+### Additional tests for different linux distributions
+
+These tests are also run on various configurations through GitLab CI. This is
+only enabled for `master@fdroid/fdroidserver` because it takes longer to
+complete than the regular CI tests.  Most of the time you won't need to worry
+about them, but sometimes it might make sense to also run them for your merge
+request. In that case you need to remove [these lines from .gitlab-ci.yml](https://gitlab.com/fdroid/fdroidserver/-/blob/0124b9dde99f9cab19c034cbc7d8cc6005a99b48/.gitlab-ci.yml#L90-91)
+and push this to a new branch of your fork.
+
+Alternatively [run them
+locally](https://docs.gitlab.com/runner/commands/README.html#gitlab-runner-exec)
+like this: `gitlab-runner exec docker ubuntu_lts`
+
+
+## Documentation
+
+The API documentation based on the docstrings gets automatically
+published [here](https://fdroid.gitlab.io/fdroidserver) on every commit
+on the `master` branch.
+
+It can be built locally via
+
+```bash
+pip install -e .[docs]
+cd docs
+sphinx-apidoc -o ./source ../fdroidserver -M -e
+sphinx-autogen -o generated source/*.rst
+make html
+```
+
+To additionally lint the code call
+```bash
+pydocstyle fdroidserver --count
+```
+
+When writing docstrings you should follow the
+[numpy style guide](https://numpydoc.readthedocs.io/en/latest/format.html).
+
+
+## Translation
+
+Everything can be translated.  See
+[Translation and Localization](https://f-droid.org/docs/Translation_and_Localization)
+for more info.
+
+<div align="center">
+
+[![](https://hosted.weblate.org/widgets/f-droid/-/287x66-white.png)](https://hosted.weblate.org/engage/f-droid)
+
+<details>
+<summary>View translation status for all languages.</summary>
+
+[![](https://hosted.weblate.org/widgets/f-droid/-/fdroidserver/multi-auto.svg)](https://hosted.weblate.org/engage/f-droid/?utm_source=widget)
+
+</details>
+
+</div>

buildserver/.gitignore

@@ -1,4 +1,4 @@
 .vagrant
 up.log
 cache/
-Vagrantfile
+Vagrantfile.yaml

buildserver/Dockerfile

@@ -0,0 +1,74 @@
+
+FROM debian:bookworm
+
+ENV LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive
+
+RUN echo Etc/UTC > /etc/timezone \
+	&& echo 'Acquire::Retries "20";' \
+		'APT::Get::Assume-Yes "true";' \
+		'APT::Install-Recommends "0";' \
+		'APT::Install-Suggests "0";' \
+		'Dpkg::Use-Pty "0";' \
+		'quiet "1";' \
+        >> /etc/apt/apt.conf.d/99gitlab
+
+# provision-apt-proxy was deliberately omitted, its not relevant in Docker
+COPY 	provision-android-ndk \
+	provision-android-sdk \
+	provision-apt-get-install \
+	provision-buildserverid \
+	provision-gradle \
+	setup-env-vars \
+	/opt/buildserver/
+
+ARG GIT_REV_PARSE_HEAD=unspecified
+LABEL org.opencontainers.image.revision=$GIT_REV_PARSE_HEAD
+
+# setup 'vagrant' user for compatibility
+RUN useradd --create-home -s /bin/bash vagrant && echo -n 'vagrant:vagrant' | chpasswd
+
+# The provision scripts must be run in the same order as in Vagrantfile
+# - vagrant needs openssh-client iproute2 ssh sudo
+# - ansible needs python3
+#
+# Debian Docker images will soon default to HTTPS for apt sources, so force it.
+# https://github.com/debuerreotype/docker-debian-artifacts/issues/15
+#
+# Ensure fdroidserver's dependencies are marked manual before purging
+# unneeded packages, otherwise, all its dependencies get purged.
+#
+# The official Debian docker images ship without ca-certificates, so
+# TLS certificates cannot be verified until that is installed. The
+# following code temporarily turns off TLS verification, and enables
+# HTTPS, so at least unverified TLS is used for apt-get instead of
+# plain HTTP.  Once ca-certificates is installed, the CA verification
+# is enabled by removing the newly created config file.  This set up
+# makes the initial `apt-get update` and `apt-get install` look the
+# same as verified TLS to the network observer and hides the metadata.
+RUN printf "path-exclude=/usr/share/locale/*\npath-exclude=/usr/share/man/*\npath-exclude=/usr/share/doc/*\npath-include=/usr/share/doc/*/copyright\n" >/etc/dpkg/dpkg.cfg.d/01_nodoc \
+	&& mkdir -p /usr/share/man/man1 \
+	&& echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/99nocacertificates \
+	&& find /etc/apt/sources.list* -type f -exec sed -i s,http:,https:, {} \; \
+	&& apt-get update \
+	&& apt-get install ca-certificates \
+	&& rm /etc/apt/apt.conf.d/99nocacertificates \
+	&& apt-get upgrade \
+	&& apt-get dist-upgrade \
+	&& apt-get install openssh-client iproute2 python3 openssh-server sudo \
+	&& bash /opt/buildserver/setup-env-vars /opt/android-sdk \
+	&& . /etc/profile.d/bsenv.sh \
+	&& bash /opt/buildserver/provision-apt-get-install https://deb.debian.org/debian \
+	&& bash /opt/buildserver/provision-android-sdk "tools;25.2.5" \
+	&& bash /opt/buildserver/provision-android-ndk /opt/android-sdk/ndk \
+	&& bash /opt/buildserver/provision-gradle \
+	&& bash /opt/buildserver/provision-buildserverid $GIT_REV_PARSE_HEAD \
+	&& rm -rf /vagrant/cache \
+	&& apt-get autoremove --purge \
+	&& apt-get clean \
+	&& rm -rf /var/lib/apt/lists/*
+
+# Vagrant sudo setup for compatibility
+RUN echo 'vagrant ALL = NOPASSWD: ALL' > /etc/sudoers.d/vagrant \
+	&& chmod 440 /etc/sudoers.d/vagrant \
+	&& sed -i -e 's/Defaults.*requiretty/#&/' /etc/sudoers

buildserver/Vagrantfile

@@ -0,0 +1,109 @@
+require 'yaml'
+require 'pathname'
+require 'fileutils'
+
+configfile = {
+  'boot_timeout' => 600,
+  'cachedir' => File.join(ENV['HOME'], '.cache', 'fdroidserver'),
+  'cpus' => 1,
+  'debian_mirror' => 'https://deb.debian.org/debian/',
+  'hwvirtex' => 'on',
+  'memory' => 2048,
+  'vm_provider' => 'virtualbox',
+}
+
+srvpath = Pathname.new(File.dirname(__FILE__)).realpath
+configpath = File.join(srvpath, "/Vagrantfile.yaml")
+if File.exist? configpath
+  c = YAML.load_file(configpath)
+  if c and not c.empty?
+    c.each do |k,v|
+      configfile[k] = v
+    end
+  end
+else
+  puts "Copying example file to #{configpath}"
+  FileUtils.cp('../examples/Vagrantfile.yaml', configpath)
+end
+
+Vagrant.configure("2") do |config|
+
+  if Vagrant.has_plugin?("vagrant-cachier")
+    config.cache.scope = :box
+    config.cache.auto_detect = false
+    config.cache.enable :apt
+    config.cache.enable :chef
+  end
+
+  config.vm.box = "debian/bookworm64"
+
+  if not configfile.has_key? "vm_provider" or configfile["vm_provider"] == "virtualbox"
+    # default to VirtualBox if not set
+    config.vm.provider "virtualbox" do |v|
+      v.customize ["modifyvm", :id, "--memory", configfile['memory']]
+      v.customize ["modifyvm", :id, "--cpus", configfile['cpus']]
+      v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']]
+    end
+    synced_folder_type = 'virtualbox'
+  elsif configfile["vm_provider"] == "libvirt"
+    # use KVM/QEMU if this is running in KVM/QEMU
+    config.vm.provider :libvirt do |libvirt|
+      libvirt.driver = configfile["hwvirtex"] == "on" ? "kvm" : "qemu"
+      libvirt.host = "localhost"
+      libvirt.uri = "qemu:///system"
+      libvirt.cpus = configfile["cpus"]
+      libvirt.memory = configfile["memory"]
+      # Debian Vagrant image is only 20G, so allocate more
+      libvirt.machine_virtual_size = 1024
+      if configfile.has_key? "libvirt_disk_bus"
+        libvirt.disk_bus = configfile["libvirt_disk_bus"]
+      end
+      if configfile.has_key? "libvirt_nic_model_type"
+        libvirt.nic_model_type = configfile["libvirt_nic_model_type"]
+      end
+    end
+    if configfile.has_key? "synced_folder_type"
+      synced_folder_type = configfile["synced_folder_type"]
+    else
+      synced_folder_type = '9p'
+    end
+    config.vm.synced_folder './', '/vagrant', type: synced_folder_type,
+                            SharedFoldersEnableSymlinksCreate: false
+  else
+    abort("No supported VM Provider found, set vm_provider in Vagrantfile.yaml!")
+  end
+
+  config.vm.boot_timeout = configfile['boot_timeout']
+
+  if configfile.has_key? "aptproxy"
+    config.vm.provision :shell, path: "provision-apt-proxy",
+      args: [configfile["aptproxy"]]
+  end
+
+  config.vm.synced_folder configfile["cachedir"], '/vagrant/cache',
+                          create: true, type: synced_folder_type
+
+  # Make sure dir exists to mount to, since buildserver/ is
+  # automatically mounted as /vagrant in the guest VM. This is more
+  # necessary with 9p synced folders
+  Dir.mkdir('cache') unless File.exist?('cache')
+
+  # Root partition needs to be resized to the new allocated space
+  config.vm.provision "shell", inline: <<-SHELL
+    growpart -v -u auto /dev/vda 1
+    resize2fs /dev/vda1
+  SHELL
+
+  config.vm.provision "shell", name: "setup-env-vars", path: "setup-env-vars",
+    args: ["/opt/android-sdk"]
+  config.vm.provision "shell", name: "apt-get-install", path: "provision-apt-get-install",
+    args: [configfile['debian_mirror']]
+  config.vm.provision "shell", name: "android-sdk", path: "provision-android-sdk"
+  config.vm.provision "shell", name: "android-ndk", path: "provision-android-ndk",
+    args: ["/opt/android-sdk/ndk"]
+  config.vm.provision "shell", name: "gradle", path: "provision-gradle"
+  config.vm.provision "shell", name: "disable-analytics", path: "provision-disable-analytics"
+  config.vm.provision "shell", name: "buildserverid", path: "provision-buildserverid",
+    args: [`git rev-parse HEAD`]
+
+end

buildserver/config.buildserver.yml

@@ -0,0 +1,2 @@
+sdk_path: /opt/android-sdk
+gradle_version_dir: /opt/gradle/versions

buildserver/cookbooks/android-ndk/recipes/default.rb

@@ -1,36 +0,0 @@
-
-ndk_loc = node[:settings][:ndk_loc]
-user = node[:settings][:user]
-
-execute "add-android-ndk-path" do
-  user user
-  command "echo \"export PATH=\\$PATH:#{ndk_loc} #PATH-NDK\" >> /home/#{user}/.bsenv"
-  not_if "grep PATH-NDK /home/#{user}/.bsenv"
-end
-
-execute "add-android-ndk-var" do
-  user user
-  command "echo \"export ANDROID_NDK=#{ndk_loc}\" >> /home/#{user}/.bsenv"
-  not_if "grep ANDROID_NDK /home/#{user}/.bsenv"
-end
-
-script "setup-android-ndk" do
-  timeout 14400
-  interpreter "bash"
-  user node[:settings][:user]
-  cwd "/tmp"
-  code "
-    if [ `uname -m` == 'x86_64' ] ; then
-       SUFFIX='_64'
-    else
-       SUFFIX=''
-    fi
-    tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2
-    tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2
-    mv android-ndk-r9b #{ndk_loc}
-  "
-  not_if do
-    File.exists?("#{ndk_loc}")
-  end
-end
-

buildserver/cookbooks/android-sdk/recipes/default.rb

@@ -1,99 +0,0 @@
-
-sdk_loc = node[:settings][:sdk_loc]
-user = node[:settings][:user]
-
-script "setup-android-sdk" do
-  timeout 14400
-  interpreter "bash"
-  user user
-  cwd "/tmp"
-  code "
-    tar zxvf /vagrant/cache/android-sdk_r22.3-linux.tgz
-    mv android-sdk-linux #{sdk_loc}
-    #{sdk_loc}/tools/android update sdk --no-ui -t platform-tool
-    #{sdk_loc}/tools/android update sdk --no-ui -t tool
-  "
-  not_if "test -d #{sdk_loc}"
-end
-
-execute "add-android-sdk-path" do
-  user user
-  path = "#{sdk_loc}/tools:#{sdk_loc}/platform-tools"
-  command "echo \"export PATH=\\$PATH:#{path} #PATH-SDK\" >> /home/#{user}/.bsenv"
-  not_if "grep PATH-SDK /home/#{user}/.bsenv"
-end
-
-execute "add-android-home" do
-  user user
-  command "echo \"export ANDROID_HOME=#{sdk_loc}\" >> /home/#{user}/.bsenv"
-  not_if "grep ANDROID_HOME /home/#{user}/.bsenv"
-end
-
-script "add_build_tools" do
-  interpreter "bash"
-  user user
-  ver = "19.0.1"
-  cwd "/tmp"
-  code "
-    if [ -f /vagrant/cache/build-tools/#{ver}.tar.gz ] ; then
-      echo Installing from cache
-      mkdir #{sdk_loc}/build-tools
-      tar -C #{sdk_loc}/build-tools -z -x -f /vagrant/cache/build-tools/#{ver}.tar.gz
-    else
-      #{sdk_loc}/tools/android update sdk --no-ui -a -t build-tools-#{ver} <<X
-y
-
-X
-    fi
-	sed -i '/BTPATH/d' /home/#{user}/.bsenv
-	echo \"export PATH=\\$PATH:#{sdk_loc}/build-tools/#{ver} #BTPATH\" >> /home/#{user}/.bsenv
-  "
-  not_if "test -d #{sdk_loc}/build-tools/#{ver}"
-end
-
-# This is currently 19.0.1
-script "add_platform_tools" do
-  interpreter "bash"
-  user user
-  cwd "/tmp"
-  code "
-    if [ -f /vagrant/cache/platform-tools.tar.gz ] ; then
-      echo Installing from cache
-      mkdir #{sdk_loc}/platform-tools
-      tar -C #{sdk_loc}/platform-tools -z -x -f /vagrant/cache/platform-tools.tar.gz
-    else
-      #{sdk_loc}/tools/android update sdk --no-ui -a -t platform-tools <<X
-y
-
-X
-    fi
-  "
-  not_if "test -d #{sdk_loc}/platform-tools"
-end
-
-%w{android-3 android-4 android-7 android-8 android-10 android-11
-   android-12 android-13 android-14 android-15 android-16 android-17
-   android-18 android-19
-   extra-android-support extra-android-m2repository}.each do |sdk|
-
-  script "add_sdk_#{sdk}" do
-    interpreter "bash"
-    user user
-    cwd "/tmp"
-    code "
-      if [ -f /vagrant/cache/platforms/#{sdk}.tar.gz ] ; then
-        echo Installing from cache
-        tar -C #{sdk_loc}/platforms -z -x -f /vagrant/cache/platforms/#{sdk}.tar.gz
-      else
-        echo Installing via 'android'
-        #{sdk_loc}/tools/android update sdk --no-ui -a -t #{sdk} <<X
-y
-
-X
-      fi
-    "
-    not_if "test -d #{sdk_loc}/platforms/#{sdk}"
-  end
-
-end
-

buildserver/cookbooks/fdroidbuild-general/recipes/default.rb

@@ -1,28 +0,0 @@
-
-user = node[:settings][:user]
-
-execute "apt-get-update" do
-  command "apt-get update"
-end
-
-%w{ant ant-contrib autoconf autopoint bison cmake expect libtool libsaxonb-java libssl1.0.0 libssl-dev maven openjdk-7-jdk javacc python python-magic git-core mercurial subversion bzr git-svn make perlmagick pkg-config zip ruby rubygems librmagick-ruby yasm imagemagick gettext realpath transfig texinfo}.each do |pkg|
-  package pkg do
-    action :install
-  end
-end
-
-if node['kernel']['machine'] == "x86_64"
-  %w{libstdc++6:i386 libgcc1:i386 zlib1g:i386 libncurses5:i386}.each do |pkg|
-    package pkg do
-      action :install
-    end
-  end
-end
-
-execute "add-bsenv" do
-  user user
-  command "echo \". ./.bsenv \" >> /home/#{user}/.bashrc"
-  not_if "grep bsenv /home/#{user}/.bashrc"
-end
-
-

buildserver/cookbooks/gradle/recipes/default.rb

@@ -1,48 +0,0 @@
-
-user = node[:settings][:user]
-
-gradle_script = IO.read(File.join(
-	File.expand_path(File.dirname(__FILE__)), "gradle"))
-
-script "add-gradle-bindir" do
-  cwd "/tmp"
-  interpreter "bash"
-  code "mkdir -p /opt/gradle/bin"
-  not_if "test -d /opt/gradle/bin"
-end
-
-script "add-gradle-verdir" do
-  cwd "/tmp"
-  interpreter "bash"
-  code "mkdir -p /opt/gradle/versions"
-  not_if "test -d /opt/gradle/versions"
-end
-
-%w{1.4 1.6 1.7 1.8 1.9}.each do |ver|
-  script "install-gradle-#{ver}" do
-    cwd "/tmp"
-    interpreter "bash"
-    code "
-      unzip /vagrant/cache/gradle-#{ver}-bin.zip
-      mv gradle-#{ver} /opt/gradle/versions/#{ver}
-    "
-    not_if "test -d /opt/gradle/versions/#{ver}"
-  end
-end
-
-script "add-gradle-wrapper" do
-  cwd "/tmp"
-  interpreter "bash"
-  code "
-    cat << \"EOF\" > /opt/gradle/bin/gradle
-#{gradle_script}
-EOF
-    chmod a+x /opt/gradle/bin/gradle
-  "
-end
-
-execute "add-android-ndk-path" do
-  user user
-  command "echo \"export PATH=\\$PATH:/opt/gradle/bin #PATH-GRADLE\" >> /home/#{user}/.bsenv"
-  not_if "grep PATH-GRADLE /home/#{user}/.bsenv"
-end

buildserver/cookbooks/gradle/recipes/gradle

@@ -1,68 +0,0 @@
-#!/bin/bash
-
-bindir="$(dirname $0)"
-basedir="$(dirname $bindir)"
-verdir="${basedir}/versions"
-args=("$@")
-pushd "${verdir}" &>/dev/null
-
-v_all=(*/)
-v_all=(${v_all[@]%/})
-
-v_def=${v_all[-1]}
-echo "Available gradle versions: ${v_all[@]}"
-
-popd &>/dev/null
-
-run_gradle() {
-	${verdir}/${v_found}/bin/gradle "${args[@]}"
-	exit $?
-}
-
-# key-value pairs of what gradle version each gradle plugin version
-# should accept
-d_plugin_k=(0.7 0.6 0.5 0.4 0.3 0.2)
-d_plugin_v=(1.9 1.8 1.6 1.6 1.4 1.4)
-
-# Latest takes priority
-files=(../build.gradle build.gradle)
-
-for f in ${files[@]}; do
-	[[ -f $f ]] || continue
-	while read l; do
-		if [[ $l == *'com.android.tools.build:gradle:'* ]]; then
-			plugin_pver=$(echo -n "$l" | sed "s/.*com.android.tools.build:gradle:\\([0-9\\.\\+]\\+\\).*/\\1/")
-		elif [[ $l == *'gradleVersion'* ]]; then
-			wrapper_ver=$(echo -n "$l" | sed "s/.*gradleVersion[ ]*=[ ]*[\"']\\([0-9\\.]\\+\\)[\"'].*/\\1/")
-		fi
-	done < $f
-done
-
-if [[ -n $wrapper_ver ]]; then
-	v_found=$wrapper_ver
-	echo "Found $v_found via gradleVersion"
-	run_gradle
-fi
-
-if [[ -n $plugin_pver ]]; then
-	i=0
-	match=false
-	for k in ${d_plugin_k[@]}; do
-		if [[ $plugin_pver == ${k}* ]]; then
-			plugin_ver=${d_plugin_v[$i]}
-			match=true
-			break
-		fi
-		let i++
-	done
-	if $match; then
-		v_found=$plugin_ver
-		echo "Found $v_found via gradle plugin version $k"
-	fi
-fi
-
-[[ -n $v_found ]] && run_gradle
-
-echo "No suitable gradle version found - defaulting to $v_def"
-v_found=$v_def
-run_gradle

buildserver/cookbooks/kivy/recipes/default.rb

@@ -1,36 +0,0 @@
-
-user = node[:settings][:user]
-
-%w{cython python-pygame python-pip python-virtualenv python-opengl python-gst0.10 python-enchant libgl1-mesa-dev libgles2-mesa-dev}.each do |pkg|
-  package pkg do
-    action :install
-  end
-end
-
-script "install-kivy" do
-  cwd "/tmp"
-  interpreter "bash"
-  code "
-    tar xf /vagrant/cache/Kivy-1.7.2.tar.gz
-    cd Kivy-1.7.2
-    python setup.py install
-    cd ..
-    rm -rf Kivy*
-  "
-  not_if "python -c 'import kivy'"
-end
-
-script "install-p4a" do
-  cwd "/home/vagrant"
-  interpreter "bash"
-  code "
-    git clone git://github.com/kivy/python-for-android
-    chown -R vagrant:vagrant python-for-android
-    cd python-for-android
-    git checkout ca369d774e2
-  "
-  not_if "test -d /home/vagrant/python-for-android"
-end
-
-
-

buildserver/fixpaths.sh

@@ -1,23 +0,0 @@
-#!/bin/sh
-
-fixit()
-{
-  #Fix sudoers so the PATH gets passed through, otherwise chef
-  #provisioning doesn't work.
-  if [ -z "$1" ]; then
-    export EDITOR=$0 && sudo -E visudo
-  else
-    echo "Fix sudoers"
-    echo "Defaults exempt_group=admin" >> $1
-  fi
-  #Stick the gems bin onto root's path as well.
-  sudo echo "PATH=$PATH:/var/lib/gems/1.8/bin" >>/root/.bashrc
-  # Restart sudo so it gets the changes straight away
-  sudo /etc/init.d/sudo restart
-}
-
-sudo grep "exempt_group" /etc/sudoers -q
-if [ "$?" -eq "1" ]; then
-  fixit
-fi
-

buildserver/provision-android-ndk

@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# $1 is the root dir to install the NDKs into
+# $2 and after are the NDK releases to install
+
+echo $0
+set -e
+set -x
+
+NDK_BASE=$1
+shift
+
+test -e $NDK_BASE || mkdir -p $NDK_BASE
+cd $NDK_BASE
+
+for version in $@; do
+    if [ ! -e ${NDK_BASE}/${version} ]; then
+        unzip /vagrant/cache/android-ndk-${version}-linux*.zip > /dev/null
+        mv android-ndk-${version} \
+	   `sed -En 's,^Pkg.Revision *= *(.+),\1,p' android-ndk-${version}/source.properties`
+    fi
+done
+
+# allow gradle/etc to install missing NDK versions
+chgrp vagrant $NDK_BASE
+chmod g+w $NDK_BASE
+
+# ensure all users can read and execute the NDK
+chmod -R a+rX $NDK_BASE/
+find $NDK_BASE/ -type f -executable -exec chmod a+x -- {} +

buildserver/provision-android-sdk

@@ -0,0 +1,167 @@
+#!/bin/bash
+
+echo $0
+set -e
+set -x
+
+if [ -z $ANDROID_HOME ]; then
+    echo "ANDROID_HOME env var must be set!"
+    exit 1
+fi
+
+# disable the repositories of proprietary stuff
+disabled="
+@version@=1
+@disabled@https\://dl.google.com/android/repository/extras/intel/addon.xml=disabled
+@disabled@https\://dl.google.com/android/repository/glass/addon.xml=disabled
+@disabled@https\://dl.google.com/android/repository/sys-img/android/sys-img.xml=disabled
+@disabled@https\://dl.google.com/android/repository/sys-img/android-tv/sys-img.xml=disabled
+@disabled@https\://dl.google.com/android/repository/sys-img/android-wear/sys-img.xml=disabled
+@disabled@https\://dl.google.com/android/repository/sys-img/google_apis/sys-img.xml=disabled
+"
+test -d ${HOME}/.android || mkdir ${HOME}/.android
+# there are currently zero user repos
+echo 'count=0' > ${HOME}/.android/repositories.cfg
+for line in $disabled; do
+    echo $line >> ${HOME}/.android/sites-settings.cfg
+done
+
+# Include old makebuildserver cache that is a Vagrant synced_folder
+# for sdkmanager to use.
+cachedir=$HOME/.cache/sdkmanager
+mkdir -p $cachedir
+pushd $cachedir
+for f in /vagrant/cache/*.zip; do
+    test -e $f && ln -s $f
+done
+popd
+
+# TODO do not preinstall 'tools' or 'platform-tools' at all, app builds don't need them
+packages="
+  tools;25.2.5
+  platform-tools
+  build-tools;19.1.0
+  build-tools;20.0.0
+  build-tools;21.1.2
+  build-tools;22.0.1
+  build-tools;23.0.1
+  build-tools;23.0.2
+  build-tools;23.0.3
+  build-tools;24.0.0
+  build-tools;24.0.1
+  build-tools;24.0.2
+  build-tools;24.0.3
+  build-tools;25.0.0
+  build-tools;25.0.1
+  build-tools;25.0.2
+  build-tools;25.0.3
+  build-tools;26.0.0
+  build-tools;26.0.1
+  build-tools;26.0.2
+  build-tools;26.0.3
+  build-tools;27.0.0
+  build-tools;27.0.1
+  build-tools;27.0.2
+  build-tools;27.0.3
+  build-tools;28.0.0
+  build-tools;28.0.1
+  build-tools;28.0.2
+  build-tools;28.0.3
+  build-tools;29.0.2
+  build-tools;29.0.3
+  build-tools;30.0.0
+  build-tools;30.0.1
+  build-tools;30.0.2
+  build-tools;30.0.3
+  build-tools;31.0.0
+  build-tools;32.0.0
+  build-tools;33.0.0
+  platforms;android-10
+  platforms;android-11
+  platforms;android-12
+  platforms;android-13
+  platforms;android-14
+  platforms;android-15
+  platforms;android-16
+  platforms;android-17
+  platforms;android-18
+  platforms;android-19
+  platforms;android-20
+  platforms;android-21
+  platforms;android-22
+  platforms;android-23
+  platforms;android-24
+  platforms;android-25
+  platforms;android-26
+  platforms;android-27
+  platforms;android-28
+  platforms;android-29
+  platforms;android-30
+  platforms;android-31
+  platforms;android-32
+  platforms;android-33
+"
+
+if [ $# -gt 0 ]; then
+    echo found args
+    packages=$@
+fi
+
+# temporary test of whether this script ran. It will change once
+# 'tools' is no longer installed by default.
+if [ ! -x $ANDROID_HOME/tools/bin/sdkmanager ]; then
+    mkdir -p ${ANDROID_HOME}/
+    sdkmanager $packages
+fi
+
+# this hacked cache should not end up in the Vagrant box or Docker image
+rm -rf $cachedir
+
+mkdir -p $ANDROID_HOME/licenses/
+
+cat << EOF > $ANDROID_HOME/licenses/android-sdk-license
+
+8933bad161af4178b1185d1a37fbf41ea5269c55
+
+d56f5187479451eabf01fb78af6dfcb131a6481e
+
+24333f8a63b6825ea9c5514f83c2829b004d1fee
+EOF
+
+cat <<EOF > $ANDROID_HOME/licenses/android-sdk-preview-license
+
+84831b9409646a918e30573bab4c9c91346d8abd
+EOF
+
+cat <<EOF > $ANDROID_HOME/licenses/android-sdk-preview-license-old
+
+79120722343a6f314e0719f863036c702b0e6b2a
+
+84831b9409646a918e30573bab4c9c91346d8abd
+EOF
+
+cat <<EOF > $ANDROID_HOME/licenses/intel-android-extra-license
+
+d975f751698a77b662f1254ddbeed3901e976f5a
+EOF
+
+chmod a+X $(dirname $ANDROID_HOME/)
+chmod -R a+rX $ANDROID_HOME/
+chgrp vagrant $ANDROID_HOME
+chmod g+w $ANDROID_HOME
+find $ANDROID_HOME/ -type f -executable -print0 | xargs -0 chmod a+x
+
+# allow gradle to install newer build-tools and platforms
+mkdir -p $ANDROID_HOME/{build-tools,platforms}
+chgrp vagrant $ANDROID_HOME/{build-tools,platforms}
+chmod g+w $ANDROID_HOME/{build-tools,platforms}
+
+# allow gradle/sdkmanager to install into the new m2repository
+test -d $ANDROID_HOME/extras/m2repository || mkdir -p $ANDROID_HOME/extras/m2repository
+find $ANDROID_HOME/extras/m2repository -type d | xargs chgrp vagrant
+find $ANDROID_HOME/extras/m2repository -type d | xargs chmod g+w
+
+# allow gradle/sdkmanager to install extras;android;m2repository
+test -d $ANDROID_HOME/extras/android || mkdir -p $ANDROID_HOME/extras/android
+find $ANDROID_HOME/extras/android -type d | xargs chgrp vagrant
+find $ANDROID_HOME/extras/android -type d | xargs chmod g+w

buildserver/provision-apt-get-install

@@ -0,0 +1,139 @@
+#!/bin/bash
+
+echo $0
+set -e
+set -x
+
+debian_mirror=$1
+export DEBIAN_FRONTEND=noninteractive
+
+printf 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";\n' \
+       > /etc/apt/apt.conf.d/99no-install-recommends
+
+printf 'Acquire::Retries "20";\n' \
+       > /etc/apt/apt.conf.d/99acquire-retries
+
+cat <<EOF > /etc/apt/apt.conf.d/99no-auto-updates
+APT::Periodic::Enable "0";
+APT::Periodic::Update-Package-Lists "0";
+APT::Periodic::Unattended-Upgrade "0";
+EOF
+
+printf 'APT::Get::Assume-Yes "true";\n' \
+       > /etc/apt/apt.conf.d/99assumeyes
+
+cat <<EOF > /etc/apt/apt.conf.d/99quiet
+Dpkg::Use-Pty "0";
+quiet "1";
+EOF
+
+cat <<EOF > /etc/apt/apt.conf.d/99confdef
+Dpkg::Options { "--force-confdef"; };
+EOF
+
+echo "man-db man-db/auto-update boolean false" | debconf-set-selections
+
+if echo $debian_mirror | grep '^https' 2>&1 > /dev/null; then
+    apt-get update || apt-get update
+    apt-get install ca-certificates
+fi
+
+cat << EOF > /etc/apt/sources.list
+deb ${debian_mirror} bookworm main
+deb https://security.debian.org/debian-security bookworm-security main
+deb ${debian_mirror} bookworm-updates main
+EOF
+echo "deb ${debian_mirror} bookworm-backports main" > /etc/apt/sources.list.d/backports.list
+
+apt-get update || apt-get update
+
+# purge things that might come from the base box, but we don't want
+# https://salsa.debian.org/cloud-team/debian-vagrant-images/-/tree/master/config_space/package_config
+# cat config_space/package_config/* | sort -u | grep -v '[A-Z#]'
+
+purge="
+ apt-listchanges
+ apt-utils
+ bash-completion
+ bind9-*
+ bsdextrautils
+ bzip2
+ chrony
+ cloud-utils
+ cron
+ cron-daemon-common
+ dbus
+ debconf-i18n
+ debian-faq
+ dmidecode
+ doc-debian
+ fdisk
+ file
+ groff-base
+ inetutils-telnet
+ krb5-locales
+ less
+ locales
+ logrotate
+ lsof
+ manpages
+ nano
+ ncurses-term
+ netcat-traditional
+ pciutils
+ reportbug
+ rsyslog
+ tasksel
+ traceroute
+ unattended-upgrades
+ usrmerge
+ vim-*
+ wamerican
+ wget
+ whiptail
+ xz-utils
+"
+# clean up files packages to be purged, then purge the packages
+rm -rf /var/run/dbus /var/log/unattended-upgrades
+apt-get purge $purge
+
+apt-get upgrade --download-only
+apt-get upgrade
+
+# again after upgrade in case of keyring changes
+apt-get update || apt-get update
+
+packages="
+ androguard/bookworm-backports
+ apksigner
+ default-jdk-headless
+ default-jre-headless
+ curl
+ dexdump
+ fdroidserver
+ git-svn
+ gnupg
+ mercurial
+ patch
+ python3-magic
+ python3-packaging
+ rsync
+ sdkmanager/bookworm-backports
+ sudo
+ unzip
+"
+
+apt-get install $packages --download-only
+apt-get install $packages
+
+# fdroidserver comes from git, it was installed just for dependencies
+apt-mark manual `apt-cache depends fdroidserver | sed -nE 's,^[| ]*Depends: ([a-z0-9 -]+),\1,p'`
+apt-get purge fdroidserver
+
+# clean up things that will become outdated anyway
+apt-get autoremove --purge
+apt-get clean
+rm -rf /var/lib/apt/lists/*
+
+highestjava=`update-java-alternatives --list | sort -n | tail -1 | cut -d ' ' -f 1`
+update-java-alternatives --set $highestjava

buildserver/provision-apt-proxy

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo $0
+set -e
+
+rm -f /etc/apt/apt.conf.d/02proxy
+echo "Acquire::ftp::Proxy \"$1\";" >> /etc/apt/apt.conf.d/02proxy
+echo "Acquire::http::Proxy \"$1\";" >> /etc/apt/apt.conf.d/02proxy
+echo "Acquire::https::Proxy \"$1\";" >> /etc/apt/apt.conf.d/02proxy
+
+apt-get update || apt-get update

buildserver/provision-buildserverid

@@ -0,0 +1,9 @@
+#!/bin/bash -e
+
+test -n "$1"
+
+echo "Writing buildserver ID ...ID is $1"
+set -x
+echo "$1" > /home/vagrant/buildserverid
+# sync data before we halt() the machine, we had an empty buildserverid otherwise
+sync

buildserver/provision-disable-analytics

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -ex
+
+# Flutter
+# https://github.com/flutter/flutter/issues/73657
+flutter_conf=/home/vagrant/.flutter
+cat <<EOF > $flutter_conf
+{
+  "enabled": false
+}
+EOF
+chown -R vagrant:vagrant $flutter_conf
+chmod -R 0644 $flutter_conf
+

buildserver/provision-gradle

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+set -ex
+
+
+# version compare magic
+vergte() {
+    printf '%s\n%s' "$1" "$2" | sort -C -V -r
+}
+
+test -e /opt/gradle/versions || mkdir -p /opt/gradle/versions
+cd /opt/gradle/versions
+
+glob="/vagrant/cache/gradle-*.zip"
+if compgen -G $glob; then # test if glob matches anything
+    f=$(ls -1 --sort=version --group-directories-first $glob | tail -1)
+    ver=`echo $f | sed 's,.*gradle-\([0-9][0-9.]*\).*\.zip,\1,'`
+    # only use versions greater or equal 2.2.1
+    if vergte $ver 2.2.1 && [ ! -d /opt/gradle/versions/${ver} ]; then
+        unzip -qq $f
+        mv gradle-${ver} /opt/gradle/versions/${ver}
+    fi
+fi
+
+chmod -R a+rX /opt/gradle
+
+test -e /opt/gradle/bin || mkdir -p /opt/gradle/bin
+git clone --depth 1 https://gitlab.com/fdroid/gradlew-fdroid.git /home/vagrant/gradlew-fdroid/
+chmod 0755 /home/vagrant/gradlew-fdroid/gradlew-fdroid
+chmod -R u+rwX,a+rX,go-w /home/vagrant/gradlew-fdroid/
+ln -fs /home/vagrant/gradlew-fdroid/gradlew-fdroid /opt/gradle/bin/gradle
+ln -fs /home/vagrant/gradlew-fdroid/gradlew-fdroid /usr/local/bin/
+
+chown -h vagrant:vagrant /opt/gradle/bin/gradle
+chown vagrant:vagrant /opt/gradle/versions
+chmod 0755 /opt/gradle/versions
+
+GRADLE_HOME=/home/vagrant/.gradle
+test -d $GRADLE_HOME/ || mkdir $GRADLE_HOME/
+cat <<EOF > $GRADLE_HOME/gradle.properties
+# builds are not reused, so the daemon is a waste of time
+org.gradle.daemon=false
+
+# set network timeouts to 10 minutes
+# https://github.com/gradle/gradle/pull/3371/files
+systemProp.http.connectionTimeout=600000
+systemProp.http.socketTimeout=600000
+systemProp.org.gradle.internal.http.connectionTimeout=600000
+systemProp.org.gradle.internal.http.socketTimeout=600000
+EOF
+
+chown -R vagrant:vagrant $GRADLE_HOME/
+chmod -R a+rX $GRADLE_HOME/

buildserver/setup-env-vars

@@ -0,0 +1,27 @@
+#!/bin/sh
+#
+# sets up the environment vars needed by the build process
+
+set -e
+set -x
+
+bsenv=/etc/profile.d/bsenv.sh
+
+echo "# generated on "`date` > $bsenv
+
+echo export ANDROID_HOME=$1 >> $bsenv
+echo 'export PATH=$PATH:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:/opt/gradle/bin' >> $bsenv
+echo "export DEBIAN_FRONTEND=noninteractive" >> $bsenv
+echo 'export home_vagrant=/home/vagrant' >> $bsenv
+echo 'export fdroidserver=$home_vagrant/fdroidserver' >> $bsenv
+echo 'export LC_ALL=C.UTF-8' >> $bsenv
+
+chmod 0644 $bsenv
+
+# make sure that SSH never hangs at a password or key prompt
+mkdir -p /etc/ssh/ssh_config.d/
+cat << EOF >> /etc/ssh/ssh_config.d/fdroid
+Host *
+    StrictHostKeyChecking yes
+    BatchMode yes
+EOF

completion/bash-completion

@@ -1,8 +1,8 @@
-#!/bin/bash
+# fdroid(1) completion                                   -*- shell-script -*-
 #
 # bash-completion - part of the FDroid server tools
-# Commits updates to apps, allowing you to edit the commit messages
 #
+# Copyright (C) 2013-2017 Hans-Christoph Steiner <hans@eds.org>
 # Copyright (C) 2013, 2014 Daniel Martí <mvdan@mvdan.cc>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -18,17 +18,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# 'fdroid' is completed automatically, but aliases to it are not.
-# For instance, to alias 'fd' to 'fdroid' and have competion available:
-#
-#	alias fd='fdroid'
-#	complete -F _fdroid fd
-#
-# One can use completion on aliased subcommands as follows:
-#
-#	alias fbuild='fdroid build'
-#	complete -F _fdroid_build fbuild
-
 __fdroid_init() {
 	COMPREPLY=()
 	cur="${COMP_WORDS[COMP_CWORD]}"
@@ -37,10 +26,15 @@ __fdroid_init() {
 	(( $# >= 1 )) && __complete_${1}
 }
 
-__package() {
-	files=( metadata/*.txt )
+__get_appid() {
+	files=( metadata/*.yml )
 	files=( ${files[@]#metadata/} )
-	files=${files[@]%.txt}
+	files=${files[@]%.yml}
+	echo "$files"
+}
+
+__package() {
+	files="$(__get_appid)"
 	COMPREPLY=( $( compgen -W "$files" -- $cur ) )
 }
 
@@ -65,41 +59,41 @@ __apk_vercode() {
 }
 
 __vercode() {
-	local p=${cur:0:-1}
-
-	COMPREPLY=( $( compgen -P "${p}:" -W "$( while read line; do
-		if [[ "$line" == "Build Version:"* ]]
-		then
-			line="${line#*,}"
-			printf "${line%%,*} "
-		elif [[ "$line" == "Build:"* ]]
-		then
-			line="${line#*,}"
-			printf "${line%%,*} "
-		fi
-		done < "metadata/${p}.txt" )" -- $cur ) )
+	if [ $prev = ":" ]; then
+	    appid="${COMP_WORDS[COMP_CWORD-2]}"
+	elif [ $cur = ":" ]; then
+	    appid=$prev
+	    cur=""
+	fi
+	versionCodes=`sed -En 's,^ +versionCode: +([0-9]+) *$,\1,p' metadata/${appid}.yml`
+	COMPREPLY=( $( compgen -W "$versionCodes" -- $cur ) )
 }
 
 __complete_options() {
 	case "${cur}" in
 		--*)
-			COMPREPLY=( $( compgen -W "${lopts}" -- $cur ) )
+			COMPREPLY=( $( compgen -W "--help --version ${lopts}" -- $cur ) )
 			return 0;;
 		*)
-			COMPREPLY=( $( compgen -W "${opts} ${lopts}" -- $cur ) )
+			COMPREPLY=( $( compgen -W "-h ${opts} --help --version ${lopts}" -- $cur ) )
 			return 0;;
 	esac
 }
 
 __complete_build() {
-	opts="-h -v -c -l -s -t -f"
-	lopts="--help --verbose --latest --server --resetserver --on-server
- --force --all"
+	opts="-v -q -l -s -t -f -a"
+
+	lopts="--verbose --quiet --latest --stop --test --server --skip-scan --scan-binary --no-tarball --force --all --no-refresh"
+	case "${prev}" in
+		:)
+			__vercode
+			return 0;;
+	esac
 	case "${cur}" in
 		-*)
 			__complete_options
 			return 0;;
-		*:)
+		:)
 			__vercode
 			return 0;;
 		*)
@@ -108,9 +102,15 @@ __complete_build() {
 	esac
 }
 
+__complete_gpgsign() {
+	opts="-v -q"
+	lopts="--verbose --quiet"
+	__complete_options
+}
+
 __complete_install() {
-	opts="-h -v"
-	lopts="--help --verbose --all"
+	opts="-v -q -a -p -n -y"
+	lopts="--verbose --quiet --all --color --no-color --privacy-mode --no-privacy-mode --no --yes"
 	case "${cur}" in
 		-*)
 			__complete_options
@@ -125,9 +125,10 @@ __complete_install() {
 }
 
 __complete_update() {
-	opts="-h -c -v -q -b -i -I -e -w"
-	lopts="--help --createmeta --verbose --quiet --buildreport --interactive
- --icons --editor --wiki --pretty --clean"
+	opts="-c -v -q -i -I -e"
+	lopts="--create-metadata --verbose --quiet
+ --icons --pretty --clean --delete-unknown
+ --nosign --rename-apks --use-date-from-apk"
 	case "${prev}" in
 		-e|--editor)
 			_filedir
@@ -137,8 +138,8 @@ __complete_update() {
 }
 
 __complete_publish() {
-	opts="-h -v"
-	lopts="--help --verbose"
+	opts="-v -q"
+	lopts="--verbose --quiet"
 	case "${cur}" in
 		-*)
 			__complete_options
@@ -153,8 +154,8 @@ __complete_publish() {
 }
 
 __complete_checkupdates() {
-	opts="-h -v"
-	lopts="--help --verbose --auto --autoonly --commit --gplay"
+	opts="-v -q"
+	lopts="--verbose --quiet --auto --autoonly --commit --allow-dirty"
 	case "${cur}" in
 		-*)
 			__complete_options
@@ -166,17 +167,27 @@ __complete_checkupdates() {
 }
 
 __complete_import() {
-	opts="-h -u -s -r"
-	lopts="--help --url --subdir --repo"
+	opts="-c -h -l -q -s -u -v -W"
+	lopts="--categories --license --quiet --rev --subdir --url"
 	case "${prev}" in
-		-u|--url|-r|--repo|-s|--subdir) return 0;;
+		-c|-l|-s|-u|--categories|--license|--quiet|--rev|--subdir|--url)
+		return 0;;
+		-W)
+		COMPREPLY=( $( compgen -W "error warn ignore" -- $cur ) )
+		return 0;;
 	esac
 	__complete_options
 }
 
+__complete_readmeta() {
+	opts="-v -q"
+	lopts="--verbose --quiet"
+	__complete_options
+}
+
 __complete_rewritemeta() {
-	opts="-h -v"
-	lopts="--help --verbose"
+	opts="-v -q -l"
+	lopts="--verbose --quiet --list"
 	case "${cur}" in
 		-*)
 			__complete_options
@@ -188,8 +199,8 @@ __complete_rewritemeta() {
 }
 
 __complete_lint() {
-	opts="-h -v"
-	lopts="--help --verbose"
+	opts="-v -q -f"
+	lopts="--verbose --quiet --force-yamllint --format"
 	case "${cur}" in
 		-*)
 			__complete_options
@@ -201,8 +212,8 @@ __complete_lint() {
 }
 
 __complete_scanner() {
-	opts="-h -v"
-	lopts="--help --verbose --nosvn"
+	opts="-v -q"
+	lopts="--verbose --quiet"
 	case "${cur}" in
 		-*)
 			__complete_options
@@ -217,8 +228,8 @@ __complete_scanner() {
 }
 
 __complete_verify() {
-	opts="-h -v -p"
-	lopts="--help --verbose"
+	opts="-v -q -p"
+	lopts="--verbose --quiet"
 	case "${cur}" in
 		-*)
 			__complete_options
@@ -232,45 +243,94 @@ __complete_verify() {
 	esac
 }
 
-__complete_stats() {
-	opts="-h -v -d"
-	lopts="--help --verbose --download"
+__complete_btlog() {
+	opts="-u"
+	lopts="--git-remote --git-repo --url"
 	__complete_options
 }
 
-__complete_server() {
-	opts="-h -v"
-	lopts="--help --verbose update"
+__complete_mirror() {
+	opts="-v"
+	lopts="--all --archive --build-logs --color --no-color --pgp-signatures --src-tarballs --output-dir"
+	__complete_options
+}
+
+__complete_nightly() {
+	opts="-v -q"
+	lopts="--show-secret-var --archive-older"
+	__complete_options
+}
+
+__complete_deploy() {
+	opts="-i -v -q"
+	lopts="--identity-file --local-copy-dir --sync-from-local-copy-dir
+ --verbose --quiet --no-checksum --no-keep-git-mirror-archive"
+	__complete_options
+}
+
+__complete_signatures() {
+	opts="-v -q"
+	lopts="--verbose --color --no-color --no-check-https"
+	case "${cur}" in
+		-*)
+			__complete_options
+			return 0;;
+	esac
+        _filedir 'apk'
+        return 0
+}
+
+__complete_signindex() {
+	opts="-v -q"
+	lopts="--verbose"
 	__complete_options
 }
 
 __complete_init() {
-	opts="-h -v -d"
-	lopts="--help --verbose --keystore --distinguished-name --repo-keyalias"
+	opts="-v -q -d"
+	lopts="--verbose --quiet --distinguished-name --keystore
+ --repo-keyalias --android-home --no-prompt --color --no-color"
 	__complete_options
 }
 
-_fdroid() {
-	local cmd cmds
-	cmd=${COMP_WORDS[1]}
-	cmds=" build init install update publish checkupdates import \
-rewritemeta lint scanner verify stats server "
+__cmds=" \
+btlog \
+build \
+checkupdates \
+deploy \
+gpgsign \
+import \
+init \
+install \
+lint \
+mirror \
+nightly \
+publish \
+readmeta \
+rewritemeta \
+scanner \
+signatures \
+signindex \
+update \
+verify \
+"
 
-	for c in $cmds; do eval "_fdroid_${c} () {
+for c in $__cmds; do
+	eval "_fdroid_${c} () {
 		local cur prev opts lopts
-		__fdroid_init ${c};
-	}"; done
+		__fdroid_init ${c}
+	}"
+done
 
-	[[ $cmds == *\ $cmd\ * ]] && _fdroid_${cmd} || {
-		(($COMP_CWORD == 1)) && COMPREPLY=( $( compgen -W "${cmds}" -- $cmd ) )
+_fdroid() {
+	local cmd
+	cmd=${COMP_WORDS[1]}
+
+	[[ $__cmds == *\ $cmd\ * ]] && _fdroid_${cmd} || {
+		(($COMP_CWORD == 1)) && COMPREPLY=( $( compgen -W "${__cmds}" -- $cmd ) )
 	}
 }
 
-_fd-commit() {
-	__package
-}
-
 complete -F _fdroid fdroid
-complete -F _fd-commit fd-commit
 
 return 0

config.buildserver.py

@@ -1,5 +0,0 @@
-sdk_path = "/home/vagrant/android-sdk"
-ndk_path = "/home/vagrant/android-ndk"
-build_tools = "19.0.1"
-mvn3 = "mvn"
-gradle = "gradle"

docs/.gitignore

@@ -1,13 +0,0 @@
-/manual/
-# generated docs files
-/fdroid.aux
-/fdroid.cp
-/fdroid.cps
-/fdroid.fn
-/fdroid.info
-/fdroid.ky
-/fdroid.log
-/fdroid.pg
-/fdroid.toc
-/fdroid.tp
-/fdroid.vr

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/fdl.texi

@@ -1,463 +0,0 @@
-@c The GNU Free Documentation License.
-@center Version 1.3, 3 November 2008
-
-@c This file is intended to be included within another document,
-@c hence no sectioning command or @node.
-
-@display
-Copyright @copyright{} 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.
-@uref{http://fsf.org/}
-
-Everyone is permitted to copy and distribute verbatim copies
-of this license document, but changing it is not allowed.
-@end display
-
-@enumerate 0
-@item
-PREAMBLE
-
-The purpose of this License is to make a manual, textbook, or other
-functional and useful document @dfn{free} in the sense of freedom: to
-assure everyone the effective freedom to copy and redistribute it,
-with or without modifying it, either commercially or noncommercially.
-Secondarily, this License preserves for the author and publisher a way
-to get credit for their work, while not being considered responsible
-for modifications made by others.
-
-This License is a kind of ``copyleft'', which means that derivative
-works of the document must themselves be free in the same sense.  It
-complements the GNU General Public License, which is a copyleft
-license designed for free software.
-
-We have designed this License in order to use it for manuals for free
-software, because free software needs free documentation: a free
-program should come with manuals providing the same freedoms that the
-software does.  But this License is not limited to software manuals;
-it can be used for any textual work, regardless of subject matter or
-whether it is published as a printed book.  We recommend this License
-principally for works whose purpose is instruction or reference.
-
-@item
-APPLICABILITY AND DEFINITIONS
-
-This License applies to any manual or other work, in any medium, that
-contains a notice placed by the copyright holder saying it can be
-distributed under the terms of this License.  Such a notice grants a
-world-wide, royalty-free license, unlimited in duration, to use that
-work under the conditions stated herein.  The ``Document'', below,
-refers to any such manual or work.  Any member of the public is a
-licensee, and is addressed as ``you''.  You accept the license if you
-copy, modify or distribute the work in a way requiring permission
-under copyright law.
-
-A ``Modified Version'' of the Document means any work containing the
-Document or a portion of it, either copied verbatim, or with
-modifications and/or translated into another language.
-
-A ``Secondary Section'' is a named appendix or a front-matter section
-of the Document that deals exclusively with the relationship of the
-publishers or authors of the Document to the Document's overall
-subject (or to related matters) and contains nothing that could fall
-directly within that overall subject.  (Thus, if the Document is in
-part a textbook of mathematics, a Secondary Section may not explain
-any mathematics.)  The relationship could be a matter of historical
-connection with the subject or with related matters, or of legal,
-commercial, philosophical, ethical or political position regarding
-them.
-
-The ``Invariant Sections'' are certain Secondary Sections whose titles
-are designated, as being those of Invariant Sections, in the notice
-that says that the Document is released under this License.  If a
-section does not fit the above definition of Secondary then it is not
-allowed to be designated as Invariant.  The Document may contain zero
-Invariant Sections.  If the Document does not identify any Invariant
-Sections then there are none.
-
-The ``Cover Texts'' are certain short passages of text that are listed,
-as Front-Cover Texts or Back-Cover Texts, in the notice that says that
-the Document is released under this License.  A Front-Cover Text may
-be at most 5 words, and a Back-Cover Text may be at most 25 words.
-
-A ``Transparent'' copy of the Document means a machine-readable copy,
-represented in a format whose specification is available to the
-general public, that is suitable for revising the document
-straightforwardly with generic text editors or (for images composed of
-pixels) generic paint programs or (for drawings) some widely available
-drawing editor, and that is suitable for input to text formatters or
-for automatic translation to a variety of formats suitable for input
-to text formatters.  A copy made in an otherwise Transparent file
-format whose markup, or absence of markup, has been arranged to thwart
-or discourage subsequent modification by readers is not Transparent.
-An image format is not Transparent if used for any substantial amount
-of text.  A copy that is not ``Transparent'' is called ``Opaque''.
-
-Examples of suitable formats for Transparent copies include plain
-@sc{ascii} without markup, Texinfo input format, La@TeX{} input
-format, @acronym{SGML} or @acronym{XML} using a publicly available
-@acronym{DTD}, and standard-conforming simple @acronym{HTML},
-PostScript or @acronym{PDF} designed for human modification.  Examples
-of transparent image formats include @acronym{PNG}, @acronym{XCF} and
-@acronym{JPG}.  Opaque formats include proprietary formats that can be
-read and edited only by proprietary word processors, @acronym{SGML} or
-@acronym{XML} for which the @acronym{DTD} and/or processing tools are
-not generally available, and the machine-generated @acronym{HTML},
-PostScript or @acronym{PDF} produced by some word processors for
-output purposes only.
-
-The ``Title Page'' means, for a printed book, the title page itself,
-plus such following pages as are needed to hold, legibly, the material
-this License requires to appear in the title page.  For works in
-formats which do not have any title page as such, ``Title Page'' means
-the text near the most prominent appearance of the work's title,
-preceding the beginning of the body of the text.
-
-The ``publisher'' means any person or entity that distributes copies
-of the Document to the public.
-
-A section ``Entitled XYZ'' means a named subunit of the Document whose
-title either is precisely XYZ or contains XYZ in parentheses following
-text that translates XYZ in another language.  (Here XYZ stands for a
-specific section name mentioned below, such as ``Acknowledgements'',
-``Dedications'', ``Endorsements'', or ``History''.)  To ``Preserve the Title''
-of such a section when you modify the Document means that it remains a
-section ``Entitled XYZ'' according to this definition.
-
-The Document may include Warranty Disclaimers next to the notice which
-states that this License applies to the Document.  These Warranty
-Disclaimers are considered to be included by reference in this
-License, but only as regards disclaiming warranties: any other
-implication that these Warranty Disclaimers may have is void and has
-no effect on the meaning of this License.
-
-@item
-VERBATIM COPYING
-
-You may copy and distribute the Document in any medium, either
-commercially or noncommercially, provided that this License, the
-copyright notices, and the license notice saying this License applies
-to the Document are reproduced in all copies, and that you add no other
-conditions whatsoever to those of this License.  You may not use
-technical measures to obstruct or control the reading or further
-copying of the copies you make or distribute.  However, you may accept
-compensation in exchange for copies.  If you distribute a large enough
-number of copies you must also follow the conditions in section 3.
-
-You may also lend copies, under the same conditions stated above, and
-you may publicly display copies.
-
-@item
-COPYING IN QUANTITY
-
-If you publish printed copies (or copies in media that commonly have
-printed covers) of the Document, numbering more than 100, and the
-Document's license notice requires Cover Texts, you must enclose the
-copies in covers that carry, clearly and legibly, all these Cover
-Texts: Front-Cover Texts on the front cover, and Back-Cover Texts on
-the back cover.  Both covers must also clearly and legibly identify
-you as the publisher of these copies.  The front cover must present
-the full title with all words of the title equally prominent and
-visible.  You may add other material on the covers in addition.
-Copying with changes limited to the covers, as long as they preserve
-the title of the Document and satisfy these conditions, can be treated
-as verbatim copying in other respects.
-
-If the required texts for either cover are too voluminous to fit
-legibly, you should put the first ones listed (as many as fit
-reasonably) on the actual cover, and continue the rest onto adjacent
-pages.
-
-If you publish or distribute Opaque copies of the Document numbering
-more than 100, you must either include a machine-readable Transparent
-copy along with each Opaque copy, or state in or with each Opaque copy
-a computer-network location from which the general network-using
-public has access to download using public-standard network protocols
-a complete Transparent copy of the Document, free of added material.
-If you use the latter option, you must take reasonably prudent steps,
-when you begin distribution of Opaque copies in quantity, to ensure
-that this Transparent copy will remain thus accessible at the stated
-location until at least one year after the last time you distribute an
-Opaque copy (directly or through your agents or retailers) of that
-edition to the public.
-
-It is requested, but not required, that you contact the authors of the
-Document well before redistributing any large number of copies, to give
-them a chance to provide you with an updated version of the Document.
-
-@item
-MODIFICATIONS
-
-You may copy and distribute a Modified Version of the Document under
-the conditions of sections 2 and 3 above, provided that you release
-the Modified Version under precisely this License, with the Modified
-Version filling the role of the Document, thus licensing distribution
-and modification of the Modified Version to whoever possesses a copy
-of it.  In addition, you must do these things in the Modified Version:
-
-@enumerate A
-@item
-Use in the Title Page (and on the covers, if any) a title distinct
-from that of the Document, and from those of previous versions
-(which should, if there were any, be listed in the History section
-of the Document).  You may use the same title as a previous version
-if the original publisher of that version gives permission.
-
-@item
-List on the Title Page, as authors, one or more persons or entities
-responsible for authorship of the modifications in the Modified
-Version, together with at least five of the principal authors of the
-Document (all of its principal authors, if it has fewer than five),
-unless they release you from this requirement.
-
-@item
-State on the Title page the name of the publisher of the
-Modified Version, as the publisher.
-
-@item
-Preserve all the copyright notices of the Document.
-
-@item
-Add an appropriate copyright notice for your modifications
-adjacent to the other copyright notices.
-
-@item
-Include, immediately after the copyright notices, a license notice
-giving the public permission to use the Modified Version under the
-terms of this License, in the form shown in the Addendum below.
-
-@item
-Preserve in that license notice the full lists of Invariant Sections
-and required Cover Texts given in the Document's license notice.
-
-@item
-Include an unaltered copy of this License.
-
-@item
-Preserve the section Entitled ``History'', Preserve its Title, and add
-to it an item stating at least the title, year, new authors, and
-publisher of the Modified Version as given on the Title Page.  If
-there is no section Entitled ``History'' in the Document, create one
-stating the title, year, authors, and publisher of the Document as
-given on its Title Page, then add an item describing the Modified
-Version as stated in the previous sentence.
-
-@item
-Preserve the network location, if any, given in the Document for
-public access to a Transparent copy of the Document, and likewise
-the network locations given in the Document for previous versions
-it was based on.  These may be placed in the ``History'' section.
-You may omit a network location for a work that was published at
-least four years before the Document itself, or if the original
-publisher of the version it refers to gives permission.
-
-@item
-For any section Entitled ``Acknowledgements'' or ``Dedications'', Preserve
-the Title of the section, and preserve in the section all the
-substance and tone of each of the contributor acknowledgements and/or
-dedications given therein.
-
-@item
-Preserve all the Invariant Sections of the Document,
-unaltered in their text and in their titles.  Section numbers
-or the equivalent are not considered part of the section titles.
-
-@item
-Delete any section Entitled ``Endorsements''.  Such a section
-may not be included in the Modified Version.
-
-@item
-Do not retitle any existing section to be Entitled ``Endorsements'' or
-to conflict in title with any Invariant Section.
-
-@item
-Preserve any Warranty Disclaimers.
-@end enumerate
-
-If the Modified Version includes new front-matter sections or
-appendices that qualify as Secondary Sections and contain no material
-copied from the Document, you may at your option designate some or all
-of these sections as invariant.  To do this, add their titles to the
-list of Invariant Sections in the Modified Version's license notice.
-These titles must be distinct from any other section titles.
-
-You may add a section Entitled ``Endorsements'', provided it contains
-nothing but endorsements of your Modified Version by various
-parties---for example, statements of peer review or that the text has
-been approved by an organization as the authoritative definition of a
-standard.
-
-You may add a passage of up to five words as a Front-Cover Text, and a
-passage of up to 25 words as a Back-Cover Text, to the end of the list
-of Cover Texts in the Modified Version.  Only one passage of
-Front-Cover Text and one of Back-Cover Text may be added by (or
-through arrangements made by) any one entity.  If the Document already
-includes a cover text for the same cover, previously added by you or
-by arrangement made by the same entity you are acting on behalf of,
-you may not add another; but you may replace the old one, on explicit
-permission from the previous publisher that added the old one.
-
-The author(s) and publisher(s) of the Document do not by this License
-give permission to use their names for publicity for or to assert or
-imply endorsement of any Modified Version.
-
-@item
-COMBINING DOCUMENTS
-
-You may combine the Document with other documents released under this
-License, under the terms defined in section 4 above for modified
-versions, provided that you include in the combination all of the
-Invariant Sections of all of the original documents, unmodified, and
-list them all as Invariant Sections of your combined work in its
-license notice, and that you preserve all their Warranty Disclaimers.
-
-The combined work need only contain one copy of this License, and
-multiple identical Invariant Sections may be replaced with a single
-copy.  If there are multiple Invariant Sections with the same name but
-different contents, make the title of each such section unique by
-adding at the end of it, in parentheses, the name of the original
-author or publisher of that section if known, or else a unique number.
-Make the same adjustment to the section titles in the list of
-Invariant Sections in the license notice of the combined work.
-
-In the combination, you must combine any sections Entitled ``History''
-in the various original documents, forming one section Entitled
-``History''; likewise combine any sections Entitled ``Acknowledgements'',
-and any sections Entitled ``Dedications''.  You must delete all
-sections Entitled ``Endorsements.''
-
-@item
-COLLECTIONS OF DOCUMENTS
-
-You may make a collection consisting of the Document and other documents
-released under this License, and replace the individual copies of this
-License in the various documents with a single copy that is included in
-the collection, provided that you follow the rules of this License for
-verbatim copying of each of the documents in all other respects.
-
-You may extract a single document from such a collection, and distribute
-it individually under this License, provided you insert a copy of this
-License into the extracted document, and follow this License in all
-other respects regarding verbatim copying of that document.
-
-@item
-AGGREGATION WITH INDEPENDENT WORKS
-
-A compilation of the Document or its derivatives with other separate
-and independent documents or works, in or on a volume of a storage or
-distribution medium, is called an ``aggregate'' if the copyright
-resulting from the compilation is not used to limit the legal rights
-of the compilation's users beyond what the individual works permit.
-When the Document is included in an aggregate, this License does not
-apply to the other works in the aggregate which are not themselves
-derivative works of the Document.
-
-If the Cover Text requirement of section 3 is applicable to these
-copies of the Document, then if the Document is less than one half of
-the entire aggregate, the Document's Cover Texts may be placed on
-covers that bracket the Document within the aggregate, or the
-electronic equivalent of covers if the Document is in electronic form.
-Otherwise they must appear on printed covers that bracket the whole
-aggregate.
-
-@item
-TRANSLATION
-
-Translation is considered a kind of modification, so you may
-distribute translations of the Document under the terms of section 4.
-Replacing Invariant Sections with translations requires special
-permission from their copyright holders, but you may include
-translations of some or all Invariant Sections in addition to the
-original versions of these Invariant Sections.  You may include a
-translation of this License, and all the license notices in the
-Document, and any Warranty Disclaimers, provided that you also include
-the original English version of this License and the original versions
-of those notices and disclaimers.  In case of a disagreement between
-the translation and the original version of this License or a notice
-or disclaimer, the original version will prevail.
-
-If a section in the Document is Entitled ``Acknowledgements'',
-``Dedications'', or ``History'', the requirement (section 4) to Preserve
-its Title (section 1) will typically require changing the actual
-title.
-
-@item
-TERMINATION
-
-You may not copy, modify, sublicense, or distribute the Document
-except as expressly provided under this License.  Any attempt
-otherwise to copy, modify, sublicense, or distribute it is void, and
-will automatically terminate your rights under this License.
-
-However, if you cease all violation of this License, then your license
-from a particular copyright holder is reinstated (a) provisionally,
-unless and until the copyright holder explicitly and finally
-terminates your license, and (b) permanently, if the copyright holder
-fails to notify you of the violation by some reasonable means prior to
-60 days after the cessation.
-
-Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, receipt of a copy of some or all of the same material does
-not give you any rights to use it.
-
-@item
-FUTURE REVISIONS OF THIS LICENSE
-
-The Free Software Foundation may publish new, revised versions
-of the GNU Free Documentation License from time to time.  Such new
-versions will be similar in spirit to the present version, but may
-differ in detail to address new problems or concerns.  See
-@uref{http://www.gnu.org/copyleft/}.
-
-Each version of the License is given a distinguishing version number.
-If the Document specifies that a particular numbered version of this
-License ``or any later version'' applies to it, you have the option of
-following the terms and conditions either of that specified version or
-of any later version that has been published (not as a draft) by the
-Free Software Foundation.  If the Document does not specify a version
-number of this License, you may choose any version ever published (not
-as a draft) by the Free Software Foundation.  If the Document
-specifies that a proxy can decide which future versions of this
-License can be used, that proxy's public statement of acceptance of a
-version permanently authorizes you to choose that version for the
-Document.
-
-@item
-RELICENSING
-
-``Massive Multiauthor Collaboration Site'' (or ``MMC Site'') means any
-World Wide Web server that publishes copyrightable works and also
-provides prominent facilities for anybody to edit those works.  A
-public wiki that anybody can edit is an example of such a server.  A
-``Massive Multiauthor Collaboration'' (or ``MMC'') contained in the
-site means any set of copyrightable works thus published on the MMC
-site.
-
-``CC-BY-SA'' means the Creative Commons Attribution-Share Alike 3.0
-license published by Creative Commons Corporation, a not-for-profit
-corporation with a principal place of business in San Francisco,
-California, as well as future copyleft versions of that license
-published by that same organization.
-
-``Incorporate'' means to publish or republish a Document, in whole or
-in part, as part of another Document.
-
-An MMC is ``eligible for relicensing'' if it is licensed under this
-License, and if all works that were first published under this License
-somewhere other than this MMC, and subsequently incorporated in whole
-or in part into the MMC, (1) had no cover texts or invariant sections,
-and (2) were thus incorporated prior to November 1, 2008.
-
-The operator of an MMC Site may republish an MMC contained in the site
-under CC-BY-SA on the same site at any time before August 1, 2009,
-provided the MMC is eligible for relicensing.
-
-@end enumerate
-

docs/gendocs.sh

@@ -1,466 +0,0 @@
-#!/bin/sh -e
-# gendocs.sh -- generate a GNU manual in many formats.  This script is
-#   mentioned in maintain.texi.  See the help message below for usage details.
-
-scriptversion=2013-02-03.15
-
-# Copyright 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013
-# Free Software Foundation, Inc.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# Original author: Mohit Agarwal.
-# Send bug reports and any other correspondence to bug-texinfo@gnu.org.
-#
-# The latest version of this script, and the companion template, is
-# available from Texinfo CVS:
-# http://savannah.gnu.org/cgi-bin/viewcvs/texinfo/texinfo/util/gendocs.sh
-# http://savannah.gnu.org/cgi-bin/viewcvs/texinfo/texinfo/util/gendocs_template
-#
-# An up-to-date copy is also maintained in Gnulib (gnu.org/software/gnulib).
-
-# TODO:
-# - image importation was only implemented for HTML generated by
-#   makeinfo.  But it should be simple enough to adjust.
-# - images are not imported in the source tarball.  All the needed
-#   formats (PDF, PNG, etc.) should be included.
-
-prog=`basename "$0"`
-srcdir=`pwd`
-
-scripturl="http://savannah.gnu.org/cgi-bin/viewcvs/~checkout~/texinfo/texinfo/util/gendocs.sh"
-templateurl="http://savannah.gnu.org/cgi-bin/viewcvs/~checkout~/texinfo/texinfo/util/gendocs_template"
-
-: ${SETLANG="env LANG= LC_MESSAGES= LC_ALL= LANGUAGE="}
-: ${MAKEINFO="makeinfo"}
-: ${TEXI2DVI="texi2dvi -t @finalout"}
-: ${DOCBOOK2HTML="docbook2html"}
-: ${DOCBOOK2PDF="docbook2pdf"}
-: ${DOCBOOK2TXT="docbook2txt"}
-: ${GENDOCS_TEMPLATE_DIR="."}
-: ${PERL='perl'}
-: ${TEXI2HTML="texi2html"}
-unset CDPATH
-unset use_texi2html
-
-version="gendocs.sh $scriptversion
-
-Copyright 2013 Free Software Foundation, Inc.
-There is NO warranty.  You may redistribute this software
-under the terms of the GNU General Public License.
-For more information about these matters, see the files named COPYING."
-
-usage="Usage: $prog [OPTION]... PACKAGE MANUAL-TITLE
-
-Generate output in various formats from PACKAGE.texinfo (or .texi or
-.txi) source.  See the GNU Maintainers document for a more extensive
-discussion:
-  http://www.gnu.org/prep/maintain_toc.html
-
-Options:
-  --email ADR use ADR as contact in generated web pages; always give this.
-
-  -s SRCFILE   read Texinfo from SRCFILE, instead of PACKAGE.{texinfo|texi|txi}
-  -o OUTDIR    write files into OUTDIR, instead of manual/.
-  -I DIR       append DIR to the Texinfo search path.
-  --common ARG pass ARG in all invocations.
-  --html ARG   pass ARG to makeinfo or texi2html for HTML targets.
-  --info ARG   pass ARG to makeinfo for Info, instead of --no-split.
-  --no-ascii   skip generating the plain text output.
-  --source ARG include ARG in tar archive of sources.
-  --split HOW  make split HTML by node, section, chapter; default node.
-
-  --texi2html  use texi2html to make HTML target, with all split versions.
-  --docbook    convert through DocBook too (xml, txt, html, pdf).
-
-  --help       display this help and exit successfully.
-  --version    display version information and exit successfully.
-
-Simple example: $prog --email bug-gnu-emacs@gnu.org emacs \"GNU Emacs Manual\"
-
-Typical sequence:
-  cd PACKAGESOURCE/doc
-  wget \"$scripturl\"
-  wget \"$templateurl\"
-  $prog --email BUGLIST MANUAL \"GNU MANUAL - One-line description\"
-
-Output will be in a new subdirectory \"manual\" (by default;
-use -o OUTDIR to override).  Move all the new files into your web CVS
-tree, as explained in the Web Pages node of maintain.texi.
-
-Please use the --email ADDRESS option so your own bug-reporting
-address will be used in the generated HTML pages.
-
-MANUAL-TITLE is included as part of the HTML <title> of the overall
-manual/index.html file.  It should include the name of the package being
-documented.  manual/index.html is created by substitution from the file
-$GENDOCS_TEMPLATE_DIR/gendocs_template.  (Feel free to modify the
-generic template for your own purposes.)
-
-If you have several manuals, you'll need to run this script several
-times with different MANUAL values, specifying a different output
-directory with -o each time.  Then write (by hand) an overall index.html
-with links to them all.
-
-If a manual's Texinfo sources are spread across several directories,
-first copy or symlink all Texinfo sources into a single directory.
-(Part of the script's work is to make a tar.gz of the sources.)
-
-As implied above, by default monolithic Info files are generated.
-If you want split Info, or other Info options, use --info to override.
-
-You can set the environment variables MAKEINFO, TEXI2DVI, TEXI2HTML,
-and PERL to control the programs that get executed, and
-GENDOCS_TEMPLATE_DIR to control where the gendocs_template file is
-looked for.  With --docbook, the environment variables DOCBOOK2HTML,
-DOCBOOK2PDF, and DOCBOOK2TXT are also consulted.
-
-By default, makeinfo and texi2dvi are run in the default (English)
-locale, since that's the language of most Texinfo manuals.  If you
-happen to have a non-English manual and non-English web site, see the
-SETLANG setting in the source.
-
-Email bug reports or enhancement requests to bug-texinfo@gnu.org.
-"
-
-MANUAL_TITLE=
-PACKAGE=
-EMAIL=webmasters@gnu.org  # please override with --email
-commonarg= # passed to all makeinfo/texi2html invcations.
-dirargs=   # passed to all tools (-I dir).
-dirs=      # -I's directories.
-htmlarg=
-infoarg=--no-split
-generate_ascii=true
-outdir=manual
-source_extra=
-split=node
-srcfile=
-
-while test $# -gt 0; do
-  case $1 in
-    -s)          shift; srcfile=$1;;
-    -o)          shift; outdir=$1;;
-    -I)          shift; dirargs="$dirargs -I '$1'"; dirs="$dirs $1";;
-    --common)    shift; commonarg=$1;;
-    --docbook)   docbook=yes;;
-    --email)     shift; EMAIL=$1;;
-    --html)      shift; htmlarg=$1;;
-    --info)      shift; infoarg=$1;;
-    --no-ascii)  generate_ascii=false;;
-    --source)    shift; source_extra=$1;;
-    --split)     shift; split=$1;;
-    --texi2html) use_texi2html=1;;
-
-    --help)      echo "$usage"; exit 0;;
-    --version)   echo "$version"; exit 0;;
-    -*)
-      echo "$0: Unknown option \`$1'." >&2
-      echo "$0: Try \`--help' for more information." >&2
-      exit 1;;
-    *)
-      if test -z "$PACKAGE"; then
-        PACKAGE=$1
-      elif test -z "$MANUAL_TITLE"; then
-        MANUAL_TITLE=$1
-      else
-        echo "$0: extra non-option argument \`$1'." >&2
-        exit 1
-      fi;;
-  esac
-  shift
-done
-
-# makeinfo uses the dirargs, but texi2dvi doesn't.
-commonarg=" $dirargs $commonarg"
-
-# For most of the following, the base name is just $PACKAGE
-base=$PACKAGE
-
-if test -n "$srcfile"; then
-  # but here, we use the basename of $srcfile
-  base=`basename "$srcfile"`
-  case $base in
-    *.txi|*.texi|*.texinfo) base=`echo "$base"|sed 's/\.[texinfo]*$//'`;;
-  esac
-  PACKAGE=$base
-elif test -s "$srcdir/$PACKAGE.texinfo"; then
-  srcfile=$srcdir/$PACKAGE.texinfo
-elif test -s "$srcdir/$PACKAGE.texi"; then
-  srcfile=$srcdir/$PACKAGE.texi
-elif test -s "$srcdir/$PACKAGE.txi"; then
-  srcfile=$srcdir/$PACKAGE.txi
-else
-  echo "$0: cannot find .texinfo or .texi or .txi for $PACKAGE in $srcdir." >&2
-  exit 1
-fi
-
-if test ! -r $GENDOCS_TEMPLATE_DIR/gendocs_template; then
-  echo "$0: cannot read $GENDOCS_TEMPLATE_DIR/gendocs_template." >&2
-  echo "$0: it is available from $templateurl." >&2
-  exit 1
-fi
-
-# Function to return size of $1 in something resembling kilobytes.
-calcsize()
-{
-  size=`ls -ksl $1 | awk '{print $1}'`
-  echo $size
-}
-
-# copy_images OUTDIR HTML-FILE...
-# -------------------------------
-# Copy all the images needed by the HTML-FILEs into OUTDIR.  Look
-# for them in the -I directories.
-copy_images()
-{
-  local odir
-  odir=$1
-  shift
-  $PERL -n -e "
-BEGIN {
-  \$me = '$prog';
-  \$odir = '$odir';
-  @dirs = qw($dirs);
-}
-" -e '
-/<img src="(.*?)"/g && ++$need{$1};
-
-END {
-  #print "$me: @{[keys %need]}\n";  # for debugging, show images found.
-  FILE: for my $f (keys %need) {
-    for my $d (@dirs) {
-      if (-f "$d/$f") {
-        use File::Basename;
-        my $dest = dirname ("$odir/$f");
-        #
-        use File::Path;
-        -d $dest || mkpath ($dest)
-          || die "$me: cannot mkdir $dest: $!\n";
-        #
-        use File::Copy;
-        copy ("$d/$f", $dest)
-          || die "$me: cannot copy $d/$f to $dest: $!\n";
-        next FILE;
-      }
-    }
-    die "$me: $ARGV: cannot find image $f\n";
-  }
-}
-' -- "$@" || exit 1
-}
-
-case $outdir in
-  /*) abs_outdir=$outdir;;
-  *)  abs_outdir=$srcdir/$outdir;;
-esac
-
-echo "Making output for $srcfile"
-echo " in `pwd`"
-mkdir -p "$outdir/"
-
-cmd="$SETLANG $MAKEINFO -o $PACKAGE.info $commonarg $infoarg \"$srcfile\""
-echo "Generating info... ($cmd)"
-eval "$cmd"
-tar czf "$outdir/$PACKAGE.info.tar.gz" $PACKAGE.info*
-ls -l "$outdir/$PACKAGE.info.tar.gz"
-info_tgz_size=`calcsize "$outdir/$PACKAGE.info.tar.gz"`
-# do not mv the info files, there's no point in having them available
-# separately on the web.
-
-cmd="$SETLANG $TEXI2DVI $dirargs \"$srcfile\""
-printf "\nGenerating dvi... ($cmd)\n"
-eval "$cmd"
-# compress/finish dvi:
-gzip -f -9 $PACKAGE.dvi
-dvi_gz_size=`calcsize $PACKAGE.dvi.gz`
-mv $PACKAGE.dvi.gz "$outdir/"
-ls -l "$outdir/$PACKAGE.dvi.gz"
-
-cmd="$SETLANG $TEXI2DVI --pdf $dirargs \"$srcfile\""
-printf "\nGenerating pdf... ($cmd)\n"
-eval "$cmd"
-pdf_size=`calcsize $PACKAGE.pdf`
-mv $PACKAGE.pdf "$outdir/"
-ls -l "$outdir/$PACKAGE.pdf"
-
-if $generate_ascii; then
-  opt="-o $PACKAGE.txt --no-split --no-headers $commonarg"
-  cmd="$SETLANG $MAKEINFO $opt \"$srcfile\""
-  printf "\nGenerating ascii... ($cmd)\n"
-  eval "$cmd"
-  ascii_size=`calcsize $PACKAGE.txt`
-  gzip -f -9 -c $PACKAGE.txt >"$outdir/$PACKAGE.txt.gz"
-  ascii_gz_size=`calcsize "$outdir/$PACKAGE.txt.gz"`
-  mv $PACKAGE.txt "$outdir/"
-  ls -l "$outdir/$PACKAGE.txt" "$outdir/$PACKAGE.txt.gz"
-fi
-
-html_split()
-{
-  opt="--split=$1 --node-files $commonarg $htmlarg"
-  cmd="$SETLANG $TEXI2HTML --output $PACKAGE.html $opt \"$srcfile\""
-  printf "\nGenerating html by $1... ($cmd)\n"
-  eval "$cmd"
-  split_html_dir=$PACKAGE.html
-  (
-    cd ${split_html_dir} || exit 1
-    ln -sf ${PACKAGE}.html index.html
-    tar -czf "$abs_outdir/${PACKAGE}.html_$1.tar.gz" -- *.html
-  )
-  eval html_$1_tgz_size=`calcsize "$outdir/${PACKAGE}.html_$1.tar.gz"`
-  rm -f "$outdir"/html_$1/*.html
-  mkdir -p "$outdir/html_$1/"
-  mv ${split_html_dir}/*.html "$outdir/html_$1/"
-  rmdir ${split_html_dir}
-}
-
-if test -z "$use_texi2html"; then
-  opt="--no-split --html -o $PACKAGE.html $commonarg $htmlarg"
-  cmd="$SETLANG $MAKEINFO $opt \"$srcfile\""
-  printf "\nGenerating monolithic html... ($cmd)\n"
-  rm -rf $PACKAGE.html  # in case a directory is left over
-  eval "$cmd"
-  html_mono_size=`calcsize $PACKAGE.html`
-  gzip -f -9 -c $PACKAGE.html >"$outdir/$PACKAGE.html.gz"
-  html_mono_gz_size=`calcsize "$outdir/$PACKAGE.html.gz"`
-  copy_images "$outdir/" $PACKAGE.html
-  mv $PACKAGE.html "$outdir/"
-  ls -l "$outdir/$PACKAGE.html" "$outdir/$PACKAGE.html.gz"
-
-  opt="--html -o $PACKAGE.html --split=$split $commonarg $htmlarg"
-  cmd="$SETLANG $MAKEINFO $opt \"$srcfile\""
-  printf "\nGenerating html by $split... ($cmd)\n"
-  eval "$cmd"
-  split_html_dir=$PACKAGE.html
-  copy_images $split_html_dir/ $split_html_dir/*.html
-  (
-    cd $split_html_dir || exit 1
-    tar -czf "$abs_outdir/$PACKAGE.html_$split.tar.gz" -- *
-  )
-  eval \
-    html_${split}_tgz_size=`calcsize "$outdir/$PACKAGE.html_$split.tar.gz"`
-  rm -rf "$outdir/html_$split/"
-  mv $split_html_dir "$outdir/html_$split/"
-  du -s "$outdir/html_$split/"
-  ls -l "$outdir/$PACKAGE.html_$split.tar.gz"
-
-else # use texi2html:
-  opt="--output $PACKAGE.html $commonarg $htmlarg"
-  cmd="$SETLANG $TEXI2HTML $opt \"$srcfile\""
-  printf "\nGenerating monolithic html with texi2html... ($cmd)\n"
-  rm -rf $PACKAGE.html  # in case a directory is left over
-  eval "$cmd"
-  html_mono_size=`calcsize $PACKAGE.html`
-  gzip -f -9 -c $PACKAGE.html >"$outdir/$PACKAGE.html.gz"
-  html_mono_gz_size=`calcsize "$outdir/$PACKAGE.html.gz"`
-  mv $PACKAGE.html "$outdir/"
-
-  html_split node
-  html_split chapter
-  html_split section
-fi
-
-printf "\nMaking .tar.gz for sources...\n"
-d=`dirname $srcfile`
-(
-  cd "$d"
-  srcfiles=`ls -d *.texinfo *.texi *.txi *.eps $source_extra 2>/dev/null` || true
-  tar czfh "$abs_outdir/$PACKAGE.texi.tar.gz" $srcfiles
-  ls -l "$abs_outdir/$PACKAGE.texi.tar.gz"
-)
-texi_tgz_size=`calcsize "$outdir/$PACKAGE.texi.tar.gz"`
-
-if test -n "$docbook"; then
-  opt="-o - --docbook $commonarg"
-  cmd="$SETLANG $MAKEINFO $opt \"$srcfile\" >${srcdir}/$PACKAGE-db.xml"
-  printf "\nGenerating docbook XML... ($cmd)\n"
-  eval "$cmd"
-  docbook_xml_size=`calcsize $PACKAGE-db.xml`
-  gzip -f -9 -c $PACKAGE-db.xml >"$outdir/$PACKAGE-db.xml.gz"
-  docbook_xml_gz_size=`calcsize "$outdir/$PACKAGE-db.xml.gz"`
-  mv $PACKAGE-db.xml "$outdir/"
-
-  split_html_db_dir=html_node_db
-  opt="$commonarg -o $split_html_db_dir"
-  cmd="$DOCBOOK2HTML $opt \"${outdir}/$PACKAGE-db.xml\""
-  printf "\nGenerating docbook HTML... ($cmd)\n"
-  eval "$cmd"
-  (
-    cd ${split_html_db_dir} || exit 1
-    tar -czf "$abs_outdir/${PACKAGE}.html_node_db.tar.gz" -- *.html
-  )
-  html_node_db_tgz_size=`calcsize "$outdir/${PACKAGE}.html_node_db.tar.gz"`
-  rm -f "$outdir"/html_node_db/*.html
-  mkdir -p "$outdir/html_node_db"
-  mv ${split_html_db_dir}/*.html "$outdir/html_node_db/"
-  rmdir ${split_html_db_dir}
-
-  cmd="$DOCBOOK2TXT \"${outdir}/$PACKAGE-db.xml\""
-  printf "\nGenerating docbook ASCII... ($cmd)\n"
-  eval "$cmd"
-  docbook_ascii_size=`calcsize $PACKAGE-db.txt`
-  mv $PACKAGE-db.txt "$outdir/"
-
-  cmd="$DOCBOOK2PDF \"${outdir}/$PACKAGE-db.xml\""
-  printf "\nGenerating docbook PDF... ($cmd)\n"
-  eval "$cmd"
-  docbook_pdf_size=`calcsize $PACKAGE-db.pdf`
-  mv $PACKAGE-db.pdf "$outdir/"
-fi
-
-printf "\nMaking index file...\n"
-if test -z "$use_texi2html"; then
-  CONDS="/%%IF  *HTML_SECTION%%/,/%%ENDIF  *HTML_SECTION%%/d;\
-         /%%IF  *HTML_CHAPTER%%/,/%%ENDIF  *HTML_CHAPTER%%/d"
-else
-  # should take account of --split here.
-  CONDS="/%%ENDIF.*%%/d;/%%IF  *HTML_SECTION%%/d;/%%IF  *HTML_CHAPTER%%/d"
-fi
-
-curdate=`$SETLANG date '+%B %d, %Y'`
-sed \
-   -e "s!%%TITLE%%!$MANUAL_TITLE!g" \
-   -e "s!%%EMAIL%%!$EMAIL!g" \
-   -e "s!%%PACKAGE%%!$PACKAGE!g" \
-   -e "s!%%DATE%%!$curdate!g" \
-   -e "s!%%HTML_MONO_SIZE%%!$html_mono_size!g" \
-   -e "s!%%HTML_MONO_GZ_SIZE%%!$html_mono_gz_size!g" \
-   -e "s!%%HTML_NODE_TGZ_SIZE%%!$html_node_tgz_size!g" \
-   -e "s!%%HTML_SECTION_TGZ_SIZE%%!$html_section_tgz_size!g" \
-   -e "s!%%HTML_CHAPTER_TGZ_SIZE%%!$html_chapter_tgz_size!g" \
-   -e "s!%%INFO_TGZ_SIZE%%!$info_tgz_size!g" \
-   -e "s!%%DVI_GZ_SIZE%%!$dvi_gz_size!g" \
-   -e "s!%%PDF_SIZE%%!$pdf_size!g" \
-   -e "s!%%ASCII_SIZE%%!$ascii_size!g" \
-   -e "s!%%ASCII_GZ_SIZE%%!$ascii_gz_size!g" \
-   -e "s!%%TEXI_TGZ_SIZE%%!$texi_tgz_size!g" \
-   -e "s!%%DOCBOOK_HTML_NODE_TGZ_SIZE%%!$html_node_db_tgz_size!g" \
-   -e "s!%%DOCBOOK_ASCII_SIZE%%!$docbook_ascii_size!g" \
-   -e "s!%%DOCBOOK_PDF_SIZE%%!$docbook_pdf_size!g" \
-   -e "s!%%DOCBOOK_XML_SIZE%%!$docbook_xml_size!g" \
-   -e "s!%%DOCBOOK_XML_GZ_SIZE%%!$docbook_xml_gz_size!g" \
-   -e "s,%%SCRIPTURL%%,$scripturl,g" \
-   -e "s!%%SCRIPTNAME%%!$prog!g" \
-   -e "$CONDS" \
-$GENDOCS_TEMPLATE_DIR/gendocs_template >"$outdir/index.html"
-
-echo "Done, see $outdir/ subdirectory for new files."
-
-# Local variables:
-# eval: (add-hook 'write-file-hooks 'time-stamp)
-# time-stamp-start: "scriptversion="
-# time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-end: "$"
-# End:

docs/gendocs_template

@@ -1,87 +0,0 @@
-<!--#include virtual="/server/header.html" -->
-<title>%%TITLE%% - GNU Project - Free Software Foundation (FSF)</title>
-<!--#include virtual="/server/banner.html" -->
-<h2>%%TITLE%%</h2>
-
-<address>Free Software Foundation</address>
-<address>last updated %%DATE%%</address>
-
-<p>This manual (%%PACKAGE%%) is available in the following formats:</p>
-
-<ul>
-<li><a href="%%PACKAGE%%.html">HTML
-    (%%HTML_MONO_SIZE%%K bytes)</a> - entirely on one web page.</li>
-<li><a href="html_node/index.html">HTML</a> - with one web page per
-    node.</li>
-%%IF HTML_SECTION%%
-<li><a href="html_section/index.html">HTML</a> - with one web page per
-    section.</li>
-%%ENDIF HTML_SECTION%%
-%%IF HTML_CHAPTER%%
-<li><a href="html_chapter/index.html">HTML</a> - with one web page per
-    chapter.</li>
-%%ENDIF HTML_CHAPTER%%
-<li><a href="%%PACKAGE%%.html.gz">HTML compressed
-    (%%HTML_MONO_GZ_SIZE%%K gzipped characters)</a> - entirely on
-    one web page.</li>
-<li><a href="%%PACKAGE%%.html_node.tar.gz">HTML compressed
-    (%%HTML_NODE_TGZ_SIZE%%K gzipped tar file)</a> -
-    with one web page per node.</li>
-%%IF HTML_SECTION%%
-<li><a href="%%PACKAGE%%.html_section.tar.gz">HTML compressed
-    (%%HTML_SECTION_TGZ_SIZE%%K gzipped tar file)</a> -
-    with one web page per section.</li>
-%%ENDIF HTML_SECTION%%
-%%IF HTML_CHAPTER%%
-<li><a href="%%PACKAGE%%.html_chapter.tar.gz">HTML compressed
-    (%%HTML_CHAPTER_TGZ_SIZE%%K gzipped tar file)</a> -
-    with one web page per chapter.</li>
-%%ENDIF HTML_CHAPTER%%
-<li><a href="%%PACKAGE%%.info.tar.gz">Info document
-    (%%INFO_TGZ_SIZE%%K bytes gzipped tar file)</a>.</li>
-<li><a href="%%PACKAGE%%.txt">ASCII text
-    (%%ASCII_SIZE%%K bytes)</a>.</li>
-<li><a href="%%PACKAGE%%.txt.gz">ASCII text compressed
-    (%%ASCII_GZ_SIZE%%K bytes gzipped)</a>.</li>
-<li><a href="%%PACKAGE%%.dvi.gz">TeX dvi file
-    (%%DVI_GZ_SIZE%%K bytes gzipped)</a>.</li>
-<li><a href="%%PACKAGE%%.pdf">PDF file
-    (%%PDF_SIZE%%K bytes)</a>.</li>
-<li><a href="%%PACKAGE%%.texi.tar.gz">Texinfo source
-    (%%TEXI_TGZ_SIZE%%K bytes gzipped tar file).</a></li>
-</ul>
-
-<p>You can <a href="http://shop.fsf.org/">buy printed copies of
-some manuals</a> (among other items) from the Free Software Foundation;
-this helps support FSF activities.</p>
-
-<p>(This page generated by the <a href="%%SCRIPTURL%%">%%SCRIPTNAME%%
-script</a>.)</p>
-
-<!-- If needed, change the copyright block at the bottom. In general,
-     all pages on the GNU web server should have the section about
-     verbatim copying.  Please do NOT remove this without talking
-     with the webmasters first.
-     Please make sure the copyright date is consistent with the document
-     and that it is like this: "2001, 2002", not this: "2001-2002". -->
-</div><!-- for id="content", starts in the include above -->
-<!--#include virtual="/server/footer.html" -->
-<div id="footer">
-
-<p>Please send general FSF &amp; GNU inquiries to
-<a href="mailto:gnu@gnu.org">&lt;gnu@gnu.org&gt;</a>.
-There are also <a href="/contact/">other ways to contact</a>
-the FSF.<br />
-Please send broken links and other corrections or suggestions to
-<a href="mailto:%%EMAIL%%">&lt;%%EMAIL%%&gt;</a>.</p>
-
-<p>Copyright &copy; 2013 Free Software Foundation, Inc.</p>
-
-<p>Verbatim copying and distribution of this entire article are
-permitted worldwide, without royalty, in any medium, provided this
-notice, and the copyright notice, are preserved.</p>
-
-</div>
-</div>
-</body>
-</html>

docs/index_versions.md

@@ -1,11 +0,0 @@
-### 11 (January 2014)
-
-* Support per-density icon folders (/icons-\*)
-
-### 10 (January 2014)
-
-* First version
-
-### 0 (?)
-
-* No version yet declared

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/source/conf.py

@@ -0,0 +1,78 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath('../../fdroidserver'))
+
+# -- Project information -----------------------------------------------------
+
+project = 'fdroidserver'
+copyright = '2021, The F-Droid Project'
+author = 'The F-Droid Project'
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'numpydoc',
+    'sphinx.ext.autodoc',
+    'sphinx.ext.autosummary',
+    "sphinx.ext.intersphinx",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "pydata_sphinx_theme"
+
+html_theme_options = {
+    "gitlab_url": "https://gitlab.com/fdroid/fdroidserver",
+    "show_prev_next": False,
+    "navbar_end": ["search-field.html", "navbar-icon-links.html"],
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+html_sidebars = {
+    "**": [],
+}
+
+#html_sidebars = {
+#   '**': ['globaltoc.html', 'sourcelink.html', 'searchbox.html'],
+#   'using/windows': ['windowssidebar.html', 'searchbox.html'],
+#}
+
+html_split_index = True
+#numpydoc_validation_checks = {"all"}
+
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3/", None),
+}

docs/source/index.rst

@@ -0,0 +1,20 @@
+.. fdroidserver documentation master file, created by
+   sphinx-quickstart on Mon May  3 10:06:52 2021.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to fdroidserver's documentation!
+========================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+ 
+These pages contain the autogenerated module docu based on the current `sources <https://gitlab.com/fdroid/fdroidserver/-/tree/master/fdroidserver>`_.
+
+Indices and tables
+==================
+
+
+* Under :ref:`modindex` the different fdroidserver modules are listed. 
+* In :ref:`genindex` you'll find all methods sorted alphabetically.

docs/update.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-set -e
-./gendocs.sh --email admin@f-droid.org fdroid "F-Droid Server Manual"
-scp -r manual/* fdroid@f-droid.org:public_html/manual/
-rm fdroid.cps fdroid.ky fdroid.vr fdroid.aux fdroid.fn fdroid.log fdroid.toc
-rm fdroid.cp  fdroid.info fdroid.pg fdroid.tp
-

examples/Vagrantfile.yaml

@@ -0,0 +1,54 @@
+---
+
+# You may want to alter these before running ./makebuildserver
+
+# In the process of setting up the build server, many gigs of files
+# are downloaded (Android SDK components, gradle, etc).  These are
+# cached so that they are not redownloaded each time. By default,
+# these are stored in ~/.cache/fdroidserver
+#
+# cachedir: buildserver/cache
+
+# To specify which Debian mirror the build server VM should use, by
+# default it uses http.debian.net, which auto-detects which is the
+# best mirror to use.
+#
+# debian_mirror: https://debian.osuosl.org/debian/
+
+# The amount of RAM the build server will have (default: 2048)
+# memory: 3584
+
+# The number of CPUs the build server will have
+# cpus: 1
+
+# Debian package proxy server - if you have one
+# aptproxy: http://192.168.0.19:8000
+
+# If this is running on an older machine or on a virtualized system,
+# it can run a lot slower. If the provisioning fails with a warning
+# about the timeout, extend the timeout here. (default: 600 seconds)
+#
+# boot_timeout: 1200
+
+# By default, this whole process uses VirtualBox as the provider, but
+# QEMU+KVM is also supported via the libvirt plugin to vagrant. If
+# this is run within a KVM guest, then libvirt's QEMU+KVM will be used
+# automatically.  It can also be manually enabled by uncommenting
+# below:
+#
+# vm_provider: libvirt
+
+# By default libvirt uses 'virtio' for both network and disk drivers.
+# Some systems (eg. nesting VMware ESXi) do not support virtio. As a
+# workaround for such rare cases, this setting allows to configure
+# KVM/libvirt to emulate hardware rather than using virtio.
+#
+# libvirt_disk_bus: sata
+# libvirt_nic_model_type: rtl8139
+
+# Sometimes, it is not possible to use the 9p synced folder type with
+# libvirt, like if running a KVM buildserver instance inside of a
+# VMware ESXi guest.  In that case, using NFS or another method is
+# required.
+#
+# synced_folder_type: nfs

examples/config.yml

@@ -0,0 +1,433 @@
+---
+# Copy this file to config.yml, then amend the settings below according to
+# your system configuration.
+
+# Custom path to the Android SDK, defaults to $ANDROID_HOME
+# sdk_path: $ANDROID_HOME
+
+# Paths to installed versions of the Android NDK.  This will be
+# automatically filled out from well known sources like
+# $ANDROID_HOME/ndk-bundle and $ANDROID_HOME/ndk/*.  If a required
+# version is missing in the buildserver VM, it will be automatically
+# downloaded and installed into the standard $ANDROID_HOME/ndk/
+# directory.  Manually setting it here will override the auto-detected
+# values.  The keys can either be the "release" (e.g. r21e) or the
+# "revision" (e.g. 21.4.7075529).
+#
+# ndk_paths:
+#   r10e: $ANDROID_HOME/android-ndk-r10e
+#   r17: ""
+#   21.4.7075529: ~/Android/Ndk
+#   r22b: null
+
+# Directory to store downloaded tools in (i.e. gradle versions)
+# By default, these are stored in ~/.cache/fdroidserver
+# cachedir: cache
+
+# Specify paths to each major Java release that you want to support
+# java_paths:
+#   8: /usr/lib/jvm/java-8-openjdk
+
+# Command or path to binary for running Ant
+# ant: ant
+
+# Command or path to binary for running maven 3
+# mvn3: mvn
+
+# Command or path to binary for running Gradle
+# Defaults to using an internal gradle wrapper (gradlew-fdroid).
+# gradle: gradle
+
+# Always scan the APKs produced by `fdroid build` for known non-free classes
+# scan_binary: true
+
+# Set the maximum age (in days) of an index that a client should accept from
+# this repo. Setting it to 0 or not setting it at all disables this
+# functionality. If you do set this to a non-zero value, you need to ensure
+# that your index is updated much more frequently than the specified interval.
+# The same policy is applied to the archive repo, if there is one.
+# repo_maxage: 0
+
+# Canonical URL of the repositoy, needs to end in /repo. Is is used to identity
+# the repo in the client, as well.
+# repo_url: https://MyFirstFDroidRepo.org/fdroid/repo
+#
+# Base URL for per-package pages on the website of this repo,
+# i.e. https://f-droid.org/packages/<appid>/ This should be accessible
+# with a browser. Setting it to null or not setting this disables the
+# feature.
+# repo_web_base_url: https://MyFirstFDroidRepo.org/packages/
+#
+# repo_name: My First F-Droid Repo Demo
+# repo_description: >-
+#   This is a repository of apps to be used with F-Droid. Applications
+#   in this repository are either official binaries built by the
+#   original application developers, or are binaries built from source
+#   by the admin of f-droid.org using the tools on
+#   https://gitlab.com/fdroid.
+
+# As above, but for the archive repo.
+#
+# archive_url: https://f-droid.org/archive
+# archive_web_base_url:
+# archive_name: My First F-Droid Archive Demo
+# archive_description: >-
+#   The repository of older versions of packages from the main demo repository.
+
+# archive_older sets the number of versions kept in the main repo, with all
+# older ones going to the archive. Set it to 0, and there will be no archive
+# repository, and no need to define the other archive_ values.
+#
+# archive_older: 3
+
+# The repo's icon defaults to a file called 'icon.png' in the 'icons'
+# folder for each section, e.g. repo/icons/icon.png and
+# archive/icons/icon.png.  To use a different filename for the icons,
+# set the filename here.  You must still copy it into place in
+# repo/icons/ and/or archive/icons/.
+#
+# repo_icon: myicon.png
+# archive_icon: myicon.png
+
+# This allows a specific kind of insecure APK to be included in the
+# 'repo' section.  Since April 2017, APK signatures that use MD5 are
+# no longer considered valid, jarsigner and apksigner will return an
+# error when verifying.  `fdroid update` will move APKs with these
+# disabled signatures to the archive.  This option stops that
+# behavior, and lets those APKs stay part of 'repo'.
+#
+# allow_disabled_algorithms: true
+
+# Normally, all apps are collected into a single app repository, like on
+# https://f-droid.org. For certain situations, it is better to make a repo
+# that is made up of APKs only from a single app. For example, an automated
+# build server that publishes nightly builds.
+# per_app_repos: true
+
+# `fdroid update` will create a link to the current version of a given app.
+# This provides a static path to the current APK. To disable the creation of
+# this link, uncomment this:
+# make_current_version_link: false
+
+# By default, the "current version" link will be based on the "Name" of the
+# app from the metadata. You can change it to use a different field from the
+# metadata here:
+# current_version_name_source: packageName
+
+# Optionally, override home directory for gpg
+# gpghome: /home/fdroid/somewhere/else/.gnupg
+
+# The ID of a GPG key for making detached signatures for APKs. Optional.
+# gpgkey: 1DBA2E89
+
+# The key (from the keystore defined below) to be used for signing the
+# repository itself. This is the same name you would give to keytool or
+# jarsigner using -alias. (Not needed in an unsigned repository).
+# repo_keyalias: fdroidrepo
+
+# Optionally, the public key for the key defined by repo_keyalias above can
+# be specified here. There is no need to do this, as the public key can and
+# will be retrieved from the keystore when needed. However, specifying it
+# manually can allow some processing to take place without access to the
+# keystore.
+# repo_pubkey: ...
+
+# The keystore to use for release keys when building. This needs to be
+# somewhere safe and secure, and backed up!  The best way to manage these
+# sensitive keys is to use a "smartcard" (aka Hardware Security Module). To
+# configure F-Droid to use a smartcard, set the keystore file using the keyword
+# "NONE" (i.e. keystore: "NONE"). That makes Java find the keystore on the
+# smartcard based on 'smartcardoptions' below.
+# keystore: ~/.local/share/fdroidserver/keystore.jks
+
+# You should not need to change these at all, unless you have a very
+# customized setup for using smartcards in Java with keytool/jarsigner
+# smartcardoptions: |
+#   -storetype PKCS11 -providerName SunPKCS11-OpenSC
+#   -providerClass sun.security.pkcs11.SunPKCS11
+#   -providerArg opensc-fdroid.cfg
+
+# The password for the keystore (at least 6 characters). If this password is
+# different than the keypass below, it can be OK to store the password in this
+# file for real use. But in general, sensitive passwords should not be stored
+# in text files!
+# keystorepass: password1
+
+# The password for keys - the same is used for each auto-generated key as well
+# as for the repository key. You should not normally store this password in a
+# file since it is a sensitive password.
+# keypass: password2
+
+# The distinguished name used for all keys.
+# keydname: CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US
+
+# Use this to override the auto-generated key aliases with specific ones
+# for particular applications. Normally, just leave it empty.
+#
+# keyaliases:
+#   com.example.app: example
+#
+# You can also force an app to use the same key alias as another one, using
+# the @ prefix.
+#
+# keyaliases:
+#   com.example.another.plugin: "@com.example.another"
+
+
+# The full path to the root of the repository. It must be specified in
+# rsync/ssh format for a remote host/path. This is used for syncing a locally
+# generated repo to the server that is it hosted on. It must end in the
+# standard public repo name of "/fdroid", but can be in up to three levels of
+# sub-directories (i.e. /var/www/packagerepos/fdroid). You can include
+# multiple servers to sync to by wrapping the whole thing in {} or [], and
+# including the serverwebroot strings in a comma-separated list.
+#
+# serverwebroot: user@example:/var/www/fdroid
+# serverwebroot:
+#   - foo.com:/usr/share/nginx/www/fdroid
+#   - bar.info:/var/www/fdroid
+#
+# There is a special mode to only deploy the index file:
+#
+# serverwebroot:
+#   - url: 'me@b.az:/srv/fdroid'
+#     index_only: true
+
+
+# When running fdroid processes on a remote server, it is possible to
+# publish extra information about the status.  Each fdroid sub-command
+# can create repo/status/running.json when it starts, then a
+# repo/status/<sub-command>.json when it completes.  The builds logs
+# and other processes will also get published, if they are running in
+# a buildserver VM.  The build logs name scheme is:
+# .../repo/$APPID_$VERCODE.log.gz.  These files are also pushed to all
+# servers configured in 'serverwebroot'.
+#
+# deploy_process_logs: true
+
+# The full URL to a git remote repository. You can include
+# multiple servers to mirror to by adding strings to a YAML list or map.
+# Servers listed here will also be automatically inserted in the mirrors list.
+#
+# servergitmirrors: https://github.com/user/repo
+# servergitmirrors:
+#   - https://github.com/user/repo
+#   - https://gitlab.com/user/repo
+#
+# servergitmirrors:
+#   - url: https://github.com/user/repo
+#   - url: https://gitlab.com/user/repo
+#     index_only: true
+
+
+# These settings allow using `fdroid deploy` for publishing APK files from
+# your repository to GitHub Releases. (You should also run `fdroid update`
+# every time before deploying to GitHub releases to update index files.) Here's
+# an example for this deployment automation:
+# https://github.com/f-droid/fdroidclient/releases/
+#
+# Currently, versions which are assigned to a release channel (e.g. alpha or
+# beta releases) are ignored.
+#
+# In the example below, tokens are read from environment variables. Putting
+# tokens directly into the config file is also supported but discouraged. It is
+# highly recommended to use a "Fine-grained personal access token", which is
+# restricted to the minimum required permissions, which are:
+#  * Metadata - read
+#  * Contents - read/write
+# (https://github.com/settings/personal-access-tokens/new)
+#
+# github_token: {env: GITHUB_TOKEN}
+# github_releases:
+#   - projectUrl: https://github.com/f-droid/fdroidclient
+#     packageNames:
+#       - org.fdroid.basic
+#       - org.fdroid.fdroid
+#     release_notes_prepend: |
+#       Re-post of official F-Droid App release from https://f-droid.org
+#   - projectUrl: https://github.com/example/app
+#     packageNames: com.example.app
+#     token: {env: GITHUB_TOKEN_EXAMPLE}
+
+
+# Most git hosting services have hard size limits for each git repo.
+# `fdroid deploy` will delete the git history when the git mirror repo
+# approaches this limit to ensure that the repo will still fit when
+# pushed.  GitHub recommends 1GB, gitlab.com recommends 10GB.
+#
+# git_mirror_size_limit: 10GB
+
+# Any mirrors of this repo, for example all of the servers declared in
+# serverwebroot and all the servers declared in servergitmirrors,
+# will automatically be used by the client.  If one
+# mirror is not working, then the client will try another.  If the
+# client has Tor enabled, then the client will prefer mirrors with
+# .onion addresses. This base URL will be used for both the main repo
+# and the archive, if it is enabled.  So these URLs should end in the
+# 'fdroid' base of the F-Droid part of the web server like serverwebroot.
+#
+# mirrors:
+#   - https://foo.bar/fdroid
+#   - http://foobarfoobarfoobar.onion/fdroid
+#
+# Or additional metadata can also be included by adding key/value pairs:
+#
+# mirrors:
+#   - url: https://foo.bar/fdroid
+#     countryCode: BA
+#   - url: http://foobarfoobarfoobar.onion/fdroid
+#
+# The list of mirrors can also be maintained in config/mirrors.yml, a
+# standalone YAML file in the optional configuration directory.  In
+# that case, mirrors: should be removed from this file (config.yml).
+
+
+# optionally specify which identity file to use when using rsync or git over SSH
+#
+# identity_file: ~/.ssh/fdroid_id_rsa
+
+
+# If you are running the repo signing process on a completely offline machine,
+# which provides the best security, then you can specify a folder to sync the
+# repo to when running `fdroid deploy`. This is most likely going to
+# be a USB thumb drive, SD Card, or some other kind of removable media. Make
+# sure it is mounted before running `fdroid deploy`. Using the
+# standard folder called 'fdroid' as the specified folder is recommended, like
+# with serverwebroot.
+#
+# local_copy_dir: /media/MyUSBThumbDrive/fdroid
+
+
+# If you are using local_copy_dir on an offline build/signing server, once the
+# thumb drive has been plugged into the online machine, it will need to be
+# synced to the copy on the online machine. To make that happen
+# automatically, set sync_from_local_copy_dir to True:
+#
+# sync_from_local_copy_dir: true
+
+# To deploy to an AWS S3 "bucket" in the US East region, set the
+# bucket name in the config, then set the environment variables
+# AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY using the values from
+# the AWS Management Console. See
+# https://rclone.org/s3/#authentication
+#
+# awsbucket: myawsfdroidbucket
+
+
+# For extended options for syncing to cloud drive and object store
+# services, `fdroid deploy' wraps Rclone. Rclone is a full featured
+# sync tool for a huge variety of cloud services. Set up your services
+# using `rclone config`, then specify each config name to deploy the
+# awsbucket: to.  Using rclone_config: overrides the default AWS S3 US
+# East setup, and will only sync to the services actually specified.
+#
+# awsbucket: myawsfdroidbucket
+# rclone_config:
+#   - aws-sample-config
+#   - rclone-supported-service-config
+
+
+# By default Rclone uses the user's default configuration file at
+# ~/.config/rclone/rclone.conf To specify a custom configuration file,
+# please add the full path to the configuration file as below.
+#
+# path_to_custom_rclone_config: /home/mycomputer/somedir/example.conf
+
+
+# If you want to force 'fdroid server' to use a non-standard serverwebroot.
+# This will allow you to have 'serverwebroot' entries which do not end in
+# '/fdroid'. (Please note that some client features expect repository URLs
+# to end in '/fdroid/repo'.)
+#
+# nonstandardwebroot: false
+
+
+# If you want to upload the release APK file to androidobservatory.org
+#
+# androidobservatory: false
+
+
+# If you want to upload the release APK file to virustotal.com
+# You have to enter your profile apikey to enable the upload.
+#
+# virustotal_apikey: 9872987234982734
+#
+# Or get it from an environment variable:
+#
+# virustotal_apikey: {env: virustotal_apikey}
+
+
+# Keep a log of all generated index files in a git repo to provide a
+# "binary transparency" log for anyone to check the history of the
+# binaries that are published.  This is in the form of a "git remote",
+# which this machine where `fdroid update` is run has already been
+# configured to allow push access (e.g. ssh key, username/password, etc)
+# binary_transparency_remote: git@gitlab.com:fdroid/binary-transparency-log.git
+
+# Set this to true to always use a build server. This saves specifying the
+# --server option on dedicated secure build server hosts.
+# build_server_always: true
+
+# Limit in number of characters that fields can take up
+# Only the fields listed here are supported, defaults shown
+# char_limits:
+#   author: 256
+#   name: 50
+#   summary: 80
+#   description: 4000
+#   video: 256
+#   whatsNew: 500
+
+# It is possible for the server operator to specify lists of apps that
+# must be installed or uninstalled on the client (aka "push installs).
+# If the user has opted in, or the device is already setup to respond
+# to these requests, then F-Droid will automatically install/uninstall
+# the packageNames listed.  This is protected by the same signing key
+# as the app index metadata.
+#
+# install_list:
+#   - at.bitfire.davdroid
+#   - com.fsck.k9
+#   - us.replicant
+#
+# uninstall_list:
+#   - com.facebook.orca
+#   - com.android.vending
+
+# `fdroid lint` checks licenses in metadata against a built white list.  By
+# default we will require license metadata to be present and only allow
+# licenses approved either by FSF or OSI.  We're using the standardized SPDX
+# license IDs.  (https://spdx.org/licenses/)
+#
+# We use `python3 -m spdx-license-list print --filter-fsf-or-osi` for
+# generating our default list.  (https://pypi.org/project/spdx-license-list)
+#
+# You can override our default list of allowed licenes by setting this option.
+# Just supply a custom list of licene names you would like to allow. To disable
+# checking licenses by the linter, assign an empty value to lint_licenses.
+#
+# lint_licenses:
+#   - Custom-License-A
+#   - Another-License
+
+# `fdroid scanner` can scan for signatures from various sources. By default
+# it's configured to only use F-Droids official SUSS collection. We have
+# support for these special collections:
+#  * 'exodus' - official exodus-privacy.org signatures
+#  * 'etip' - exodus privacy investigation platfrom community contributed
+#           signatures
+#  * 'suss' - official F-Droid: Suspicious or Unwanted Software Signatures
+# You can also configure scanner to use custom collections of signatures here.
+# They have to follow the format specified in the SUSS readme.
+# (https://gitlab.com/fdroid/fdroid-suss/#cache-file-data-format)
+#
+# scanner_signature_sources:
+#   - suss
+#   - exodus
+#   - https://example.com/signatures.json
+
+# The scanner can use signature sources from the internet. These are
+# cached locally.  To force them to be refreshed from the network on
+# every run, set this to true:
+#
+# refresh_scanner: true

examples/fdroid_clean_repos.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+#
+# an fdroid plugin for resetting app VCSs to the latest version for the metadata
+
+import argparse
+import logging
+
+from fdroidserver import _, common, metadata
+from fdroidserver.exception import VCSException
+
+fdroid_summary = 'reset app VCSs to the latest version'
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]"
+    )
+    common.setup_global_opts(parser)
+    parser.add_argument(
+        "appid",
+        nargs='*',
+        help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"),
+    )
+    metadata.add_metadata_arguments(parser)
+    options = common.parse_args(parser)
+    apps = common.read_app_args(
+        options.appid, allow_version_codes=True, sort_by_time=True
+    )
+    common.read_config()
+
+    for appid, app in apps.items():
+        if "Builds" in app and len(app["Builds"]) > 0:
+            build = app.get('Builds')[-1]
+            logging.info(_("Cleaning up '{appid}' VCS").format(appid=appid))
+            try:
+                vcs, build_dir = common.setup_vcs(app)
+                vcs.gotorevision(build.commit)
+                if build.submodules:
+                    vcs.initsubmodules()
+
+            except VCSException:
+                pass
+
+
+if __name__ == "__main__":
+    main()

examples/fdroid_export_keystore_to_nitrokey.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+#
+# an fdroid plugin for exporting a repo's keystore in standard PEM format
+
+import os
+from argparse import ArgumentParser
+
+from fdroidserver import common
+from fdroidserver.common import FDroidPopen
+from fdroidserver.exception import BuildException
+
+fdroid_summary = "export the repo's keystore file to a NitroKey HSM"
+
+
+def run(cmd, error):
+    envs = {'LC_ALL': 'C.UTF-8',
+            'PIN': config['smartcard_pin'],
+            'FDROID_KEY_STORE_PASS': config['keystorepass'],
+            'FDROID_KEY_PASS': config['keypass']}
+    p = FDroidPopen(cmd, envs=envs)
+    if p.returncode != 0:
+        raise BuildException(error, p.output)
+
+
+def main():
+    global config
+    parser = ArgumentParser()
+    common.setup_global_opts(parser)
+    common.parse_args(parser)
+    config = common.read_config()
+    destkeystore = config['keystore'].replace('.jks', '.p12').replace('/', '_')
+    exportkeystore = config['keystore'].replace('.jks', '.pem').replace('/', '_')
+    if os.path.exists(destkeystore) or os.path.exists(exportkeystore):
+        raise BuildException('%s exists!' % exportkeystore)
+    run([config['keytool'], '-importkeystore',
+         '-srckeystore', config['keystore'],
+         '-srcalias', config['repo_keyalias'],
+         '-srcstorepass:env', 'FDROID_KEY_STORE_PASS',
+         '-srckeypass:env', 'FDROID_KEY_PASS',
+         '-destkeystore', destkeystore,
+         '-deststorepass:env', 'FDROID_KEY_STORE_PASS',
+         '-deststoretype', 'PKCS12'],
+        'Failed to convert to PKCS12!')
+#    run(['openssl', 'pkcs12', '-in', destkeystore,
+#         '-passin', 'env:FDROID_KEY_STORE_PASS', '-nokeys',
+#         '-out', exportkeystore,
+#         '-passout', 'env:FDROID_KEY_STORE_PASS'],
+#        'Failed to convert to PEM!')
+    run(['pkcs15-init', '--delete-objects', 'privkey,pubkey',
+         '--id', '3', '--store-private-key', destkeystore,
+         '--format', 'pkcs12', '--auth-id', '3',
+         '--verify-pin', '--pin', 'env:PIN'],
+        '')
+    run(['pkcs15-init', '--delete-objects', 'privkey,pubkey',
+         '--id', '2', '--store-private-key', destkeystore,
+         '--format', 'pkcs12', '--auth-id', '3',
+         '--verify-pin', '--pin', 'env:PIN'],
+        '')
+
+
+if __name__ == "__main__":
+    main()

examples/fdroid_exportkeystore.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+#
+# an fdroid plugin for exporting a repo's keystore in standard PEM format
+
+import os
+from argparse import ArgumentParser
+
+from fdroidserver import common
+from fdroidserver.common import FDroidPopen
+from fdroidserver.exception import BuildException
+
+fdroid_summary = 'export the keystore in standard PEM format'
+
+
+def main():
+    parser = ArgumentParser()
+    common.setup_global_opts(parser)
+    common.parse_args(parser)
+    config = common.read_config()
+    env_vars = {'LC_ALL': 'C.UTF-8',
+                'FDROID_KEY_STORE_PASS': config['keystorepass'],
+                'FDROID_KEY_PASS': config['keypass']}
+    destkeystore = config['keystore'].replace('.jks', '.p12').replace('/', '_')
+    exportkeystore = config['keystore'].replace('.jks', '.pem').replace('/', '_')
+    if os.path.exists(destkeystore) or os.path.exists(exportkeystore):
+        raise BuildException('%s exists!' % exportkeystore)
+    p = FDroidPopen([config['keytool'], '-importkeystore',
+                     '-srckeystore', config['keystore'],
+                     '-srcalias', config['repo_keyalias'],
+                     '-srcstorepass:env', 'FDROID_KEY_STORE_PASS',
+                     '-srckeypass:env', 'FDROID_KEY_PASS',
+                     '-destkeystore', destkeystore,
+                     '-deststoretype', 'PKCS12',
+                     '-deststorepass:env', 'FDROID_KEY_STORE_PASS',
+                     '-destkeypass:env', 'FDROID_KEY_PASS'],
+                    envs=env_vars)
+    if p.returncode != 0:
+        raise BuildException("Failed to convert to PKCS12!", p.output)
+    p = FDroidPopen(['openssl', 'pkcs12', '-in', destkeystore,
+                     '-passin', 'env:FDROID_KEY_STORE_PASS', '-nokeys',
+                     '-out', exportkeystore,
+                     '-passout', 'env:FDROID_KEY_STORE_PASS'],
+                    envs=env_vars)
+    if p.returncode != 0:
+        raise BuildException("Failed to convert to PEM!", p.output)
+
+
+if __name__ == "__main__":
+    main()

examples/fdroid_extract_repo_pubkey.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+#
+# an fdroid plugin print the repo_pubkey from a repo's keystore
+#
+
+from argparse import ArgumentParser
+
+from fdroidserver import common, index
+
+fdroid_summary = 'export the keystore in standard PEM format'
+
+
+def main():
+    parser = ArgumentParser()
+    common.setup_global_opts(parser)
+    common.parse_args(parser)
+    common.read_config()
+    pubkey, repo_pubkey_fingerprint = index.extract_pubkey()
+    print('repo_pubkey = "%s"' % pubkey.decode())
+
+
+if __name__ == "__main__":
+    main()

examples/fdroid_fetchsrclibs.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+#
+# an fdroid plugin for setting up srclibs
+#
+# The 'fdroid build' gitlab-ci job uses --on-server, which does not
+# set up the srclibs.  This plugin does the missing setup.
+
+import argparse
+import os
+import pprint
+
+from fdroidserver import _, common, metadata
+
+fdroid_summary = 'prepare the srclibs for `fdroid build --on-server`'
+
+
+def main():
+    parser = argparse.ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
+    common.setup_global_opts(parser)
+    parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
+    metadata.add_metadata_arguments(parser)
+    options = common.parse_args(parser)
+    apps = common.read_app_args(options.appid, allow_version_codes=True, sort_by_time=True)
+    common.read_config()
+    srclib_dir = os.path.join('build', 'srclib')
+    os.makedirs(srclib_dir, exist_ok=True)
+    srclibpaths = []
+    for appid, app in apps.items():
+        vcs, _ignored = common.setup_vcs(app)
+        for build in app.get('Builds', []):
+            vcs.gotorevision(build.commit, refresh=False)
+            if build.submodules:
+                vcs.initsubmodules()
+            else:
+                vcs.deinitsubmodules()
+            for lib in build.srclibs:
+                srclibpaths.append(common.getsrclib(lib, srclib_dir, prepare=False, build=build))
+    print('Set up srclibs:')
+    pprint.pprint(srclibpaths)
+
+
+if __name__ == "__main__":
+    main()

examples/fdroid_nitrokeyimport.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+from argparse import ArgumentParser
+
+from fdroidserver import common
+from fdroidserver.common import FDroidPopen
+from fdroidserver.exception import BuildException
+
+fdroid_summary = 'import the local keystore into a SmartCard HSM'
+
+
+def main():
+    parser = ArgumentParser()
+    common.setup_global_opts(parser)
+    common.parse_args(parser)
+    config = common.read_config()
+    env_vars = {
+        'LC_ALL': 'C.UTF-8',
+        'FDROID_KEY_STORE_PASS': config['keystorepass'],
+        'FDROID_KEY_PASS': config['keypass'],
+        'SMARTCARD_PIN': str(config['smartcard_pin']),
+    }
+    p = FDroidPopen([config['keytool'], '-importkeystore',
+                     '-srcalias', config['repo_keyalias'],
+                     '-srckeystore', config['keystore'],
+                     '-srcstorepass:env', 'FDROID_KEY_STORE_PASS',
+                     '-srckeypass:env', 'FDROID_KEY_PASS',
+                     '-destalias', config['repo_keyalias'],
+                     '-destkeystore', 'NONE',
+                     '-deststoretype', 'PKCS11',
+                     '-providerName', 'SunPKCS11-OpenSC',
+                     '-providerClass', 'sun.security.pkcs11.SunPKCS11',
+                     '-providerArg', 'opensc-fdroid.cfg',
+                     '-deststorepass:env', 'SMARTCARD_PIN',
+                     '-J-Djava.security.debug=sunpkcs11'],
+                    envs=env_vars)
+    if p.returncode != 0:
+        raise BuildException("Failed to import into HSM!", p.output)
+
+
+if __name__ == "__main__":
+    main()

examples/opensc-fdroid.cfg

@@ -0,0 +1,4 @@
+name = OpenSC
+description = SunPKCS11 w/ OpenSC Smart card Framework
+library = /usr/lib/opensc-pkcs11.so
+slotListIndex = 1

examples/public-read-only-s3-bucket-policy.json

@@ -0,0 +1,11 @@
+{
+    "Version":"2012-10-17",
+    "Statement":[
+        {"Sid":"AddPerm",
+         "Effect":"Allow",
+         "Principal":"*",
+         "Action":"s3:GetObject",
+         "Resource":"arn:aws:s3:::examplebucket/fdroid/*"
+        }
+    ]
+}

examples/template.yml

@@ -0,0 +1,21 @@
+AuthorName: .
+WebSite: ''
+Bitcoin: null
+Litecoin: null
+Donate: null
+
+License: Unknown
+Categories:
+  - Internet
+
+IssueTracker: ''
+SourceCode: ''
+Changelog: ''
+
+Name: .
+Summary: .
+Description: |
+  .
+
+ArchivePolicy: 2 versions
+RequiresRoot: false

fd-commit

@@ -1,89 +0,0 @@
-#!/bin/bash
-#
-# fd-commit - part of the FDroid server tools
-# Commits updates to apps, allowing you to edit the commit messages
-#
-# Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-commands=()
-
-if [ ! -d metadata ]; then
-	[ -d ../metadata ] && cd .. || { echo "No metadata files found!"; exit 2; }
-fi
-
-while read line; do
-	if [[ "$line" == *M*metadata/*.txt ]]; then
-		file=${line##* }
-
-		id=${file##*/}
-		id=${id%.txt*}
-		if [ $# -gt 0 ]; then
-			found=false
-			for arg in "$@"; do
-				if [ "$id" == "$arg" ]; then
-					found=true
-					break
-				fi
-			done
-			$found || continue
-		fi
-
-		[ -d metadata/$id ] && extra=metadata/$id || extra=
-
-		name= autoname=
-		while read l; do
-			if [[ "$l" == "Auto Name:"* ]]; then
-				autoname=${l##*:}
-			elif [[ "$l" == "Name:"* ]]; then
-				name=${l##*:}
-			fi
-		done < "$file"
-
-		if [ -n "$name" ]; then
-			fullname="$name ($id)"
-		elif [ -n "$autoname" ]; then
-			fullname="$autoname ($id)"
-		else
-			fullname="$id"
-		fi
-
-		newbuild=false
-		while read l; do
-			if [[ "$l" == "+Build:"* ]]; then
-				newbuild=true
-				build=${l#*:}
-				version=${build%%,*}
-				build=${build#*,}
-				vercode=${build%%,*}
-			fi
-		done < <(git diff HEAD -- "$file")
-
-		if $newbuild ; then
-			message="Update $fullname to $version ($vercode)"
-		else
-			message="$fullname:"
-		fi
-
-		message=${message//\"/\\\"}
-		commands+=("git add -- $file $extra && git commit -m \"$message\" -e -v")
-	fi
-done < <(git status --porcelain)
-
-git reset >/dev/null
-for cmd in "${commands[@]}"; do
-	eval "$cmd"
-	git reset >/dev/null
-done

fdroid

@@ -1,9 +1,7 @@
-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 #
 # fdroid.py - part of the FDroid server tools
-# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
-# Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
+# Copyright (C) 2020 Michael Pöhn <michael.poehn@fsfe.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,50 +16,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import sys
 
-commands = [
-        "build",
-        "init",
-        "install",
-        "update",
-        "publish",
-        "verify",
-        "checkupdates",
-        "import",
-        "rewritemeta",
-        "lint",
-        "scanner",
-        "stats",
-        "server"]
-
-def print_help():
-    print "Valid commands are:"
-    for command in commands:
-        print "  " + command
-    print "Use '%s <command> --help' for more info about that command."%sys.argv[0]
-
-def main():
-
-    if len(sys.argv) <= 1:
-        print_help()
-        sys.exit(0)
-
-    command = sys.argv[1]
-    if not command in commands:
-        if command not in ('-h', '--help'):
-            print "Command '" + command + "' not recognised.\n"
-        print_help()
-        sys.exit(1)
-
-    # Trick optparse into displaying the right usage when --help is used.
-    sys.argv[0] += ' ' + command
-
-    del sys.argv[1]
-    mod = __import__('fdroidserver.' + command, None, None, [command])
-    mod.main()
-    sys.exit(0)
-
-if __name__ == "__main__":
-    main()
+import fdroidserver.__main__
 
+fdroidserver.__main__.main()

fdroidserver/__init__.py

@@ -0,0 +1,78 @@
+import gettext
+import glob
+import os
+import sys
+
+# support running straight from git and standard installs
+rootpaths = [
+    os.path.realpath(os.path.join(os.path.dirname(__file__), '..')),
+    os.path.realpath(
+        os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'share')
+    ),
+    os.path.join(sys.prefix, 'share'),
+]
+
+localedir = None
+for rootpath in rootpaths:
+    found_mo = glob.glob(
+        os.path.join(rootpath, 'locale', '*', 'LC_MESSAGES', 'fdroidserver.mo')
+    )
+    if len(found_mo) > 0:
+        localedir = os.path.join(rootpath, 'locale')
+        break
+
+gettext.bindtextdomain('fdroidserver', localedir)
+gettext.textdomain('fdroidserver')
+_ = gettext.gettext
+
+
+from fdroidserver.exception import (
+    FDroidException,
+    MetaDataException,
+    VerificationException,  # NOQA: E402
+)
+
+FDroidException  # NOQA: B101
+MetaDataException  # NOQA: B101
+VerificationException  # NOQA: B101
+
+from fdroidserver.common import genkeystore as generate_keystore  # NOQA: E402
+from fdroidserver.common import verify_apk_signature
+
+verify_apk_signature  # NOQA: B101
+generate_keystore  # NOQA: B101
+from fdroidserver.index import (
+    download_repo_index,
+    download_repo_index_v1,
+    download_repo_index_v2,
+    get_mirror_service_urls,
+)
+from fdroidserver.index import make as make_index  # NOQA: E402
+
+download_repo_index  # NOQA: B101
+download_repo_index_v1  # NOQA: B101
+download_repo_index_v2  # NOQA: B101
+get_mirror_service_urls  # NOQA: B101
+make_index  # NOQA: B101
+from fdroidserver.update import (
+    process_apk,
+    process_apks,
+    scan_apk,
+    scan_repo_files,  # NOQA: E402
+)
+
+process_apk  # NOQA: B101
+process_apks  # NOQA: B101
+scan_apk  # NOQA: B101
+scan_repo_files  # NOQA: B101
+from fdroidserver.deploy import (
+    update_awsbucket,
+    update_servergitmirrors,
+    update_serverwebroot,  # NOQA: E402
+    update_serverwebroots,
+)
+
+update_awsbucket  # NOQA: B101
+update_servergitmirrors  # NOQA: B101
+update_serverwebroots  # NOQA: B101
+update_serverwebroot  # NOQA: B101

fdroidserver/__main__.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+#
+# fdroidserver/__main__.py - part of the FDroid server tools
+# Copyright (C) 2020 Michael Pöhn <michael.poehn@fsfe.org>
+# Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
+# Copyright (C) 2013-2014 Daniel Marti <mvdan@mvdan.cc>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import importlib.metadata
+import logging
+import os
+import pkgutil
+import re
+import sys
+from argparse import ArgumentError
+from collections import OrderedDict
+
+import git
+
+import fdroidserver.common
+import fdroidserver.metadata
+from fdroidserver import _
+
+COMMANDS = OrderedDict([
+    ("build", _("Build a package from source")),
+    ("init", _("Quickly start a new repository")),
+    ("publish", _("Sign and place packages in the repo")),
+    ("gpgsign", _("Add PGP signatures using GnuPG for packages in repo")),
+    ("update", _("Update repo information for new packages")),
+    ("deploy", _("Interact with the repo HTTP server")),
+    ("verify", _("Verify the integrity of downloaded packages")),
+    ("checkupdates", _("Check for updates to applications")),
+    ("import", _("Extract application metadata from a source repository")),
+    ("install", _("Install built packages on devices")),
+    ("readmeta", _("Read all the metadata files and exit")),
+    ("rewritemeta", _("Rewrite all the metadata files")),
+    ("lint", _("Warn about possible metadata errors")),
+    ("scanner", _("Scan the source code of a package")),
+    ("signindex", _("Sign indexes created using update --nosign")),
+    ("btlog", _("Update the binary transparency log for a URL")),
+    ("signatures", _("Extract signatures from APKs")),
+    ("nightly", _("Set up an app build for a nightly build repo")),
+    ("mirror", _("Download complete mirrors of small repos")),
+])
+
+
+def print_help(available_plugins=None):
+    print(_("usage: ") + _("fdroid [<command>] [-h|--help|--version|<args>]"))
+    print("")
+    print(_("Valid commands are:"))
+    for cmd, summary in COMMANDS.items():
+        print("   " + cmd + ' ' * (15 - len(cmd)) + summary)
+    if available_plugins:
+        print(_('commands from plugin modules:'))
+        for command in sorted(available_plugins.keys()):
+            print('   {:15}{}'.format(command, available_plugins[command]['summary']))
+    print("")
+
+
+def preparse_plugin(module_name, module_dir):
+    """No summary.
+
+    Simple regex based parsing for plugin scripts.
+
+    So we don't have to import them when we just need the summary,
+    but not plan on executing this particular plugin.
+    """
+    if '.' in module_name:
+        raise ValueError("No '.' allowed in fdroid plugin modules: '{}'"
+                         .format(module_name))
+    path = os.path.join(module_dir, module_name + '.py')
+    if not os.path.isfile(path):
+        path = os.path.join(module_dir, module_name, '__main__.py')
+        if not os.path.isfile(path):
+            raise ValueError("unable to find main plugin script "
+                             "for module '{n}' ('{d}')"
+                             .format(n=module_name,
+                                     d=module_dir))
+    summary = None
+    main = None
+    with open(path, 'r', encoding='utf-8') as f:
+        re_main = re.compile(r'^(\s*def\s+main\s*\(.*\)\s*:'
+                             r'|\s*main\s*=\s*lambda\s*:.+)$')
+        re_summary = re.compile(r'^\s*fdroid_summary\s*=\s["\'](?P<text>.+)["\']$')
+        for line in f:
+            m_summary = re_summary.match(line)
+            if m_summary:
+                summary = m_summary.group('text')
+            if re_main.match(line):
+                main = True
+
+    if summary is None:
+        raise NameError("could not find 'fdroid_summary' in: '{}' plugin"
+                        .format(module_name))
+    if main is None:
+        raise NameError("could not find 'main' function in: '{}' plugin"
+                        .format(module_name))
+    return {'name': module_name, 'summary': summary}
+
+
+def find_plugins():
+    found_plugins = [{'name': x[1], 'dir': x[0].path} for x in pkgutil.iter_modules() if x[1].startswith('fdroid_')]
+    plugin_infos = {}
+    for plugin_def in found_plugins:
+        command_name = plugin_def['name'][7:]
+        try:
+            plugin_infos[command_name] = preparse_plugin(plugin_def['name'],
+                                                         plugin_def['dir'])
+        except Exception as e:
+            # We need to keep module lookup fault tolerant because buggy
+            # modules must not prevent fdroidserver from functioning
+            if len(sys.argv) > 1 and sys.argv[1] == command_name:
+                # only raise exeption when a user specifies the broken
+                # plugin in explicitly in command line
+                raise e
+    return plugin_infos
+
+
+def main():
+    available_plugins = find_plugins()
+
+    if len(sys.argv) <= 1:
+        print_help(available_plugins=available_plugins)
+        sys.exit(0)
+
+    command = sys.argv[1]
+    if command not in COMMANDS and command not in available_plugins:
+        if command in ('-h', '--help'):
+            print_help(available_plugins=available_plugins)
+            sys.exit(0)
+        elif command == 'server':
+            print(_("""ERROR: The "server" subcommand has been removed, use "deploy"!"""))
+            sys.exit(1)
+        elif command == '--version':
+            try:
+                print(importlib.metadata.version("fdroidserver"))
+                sys.exit(0)
+            except importlib.metadata.PackageNotFoundError:
+                pass
+            try:
+                print(
+                    git.repo.Repo(
+                        os.path.dirname(os.path.dirname(__file__))
+                    ).git.describe(always=True, tags=True)
+                )
+                sys.exit(0)
+            except git.exc.InvalidGitRepositoryError:
+                print(_('No version information could be found.'))
+                sys.exit(1)
+        else:
+            print(_("Command '%s' not recognised.\n" % command))
+            print_help(available_plugins=available_plugins)
+            sys.exit(1)
+
+    verbose = any(s in sys.argv for s in ['-v', '--verbose'])
+    quiet = any(s in sys.argv for s in ['-q', '--quiet'])
+
+    # Helpful to differentiate warnings from errors even when on quiet
+    logformat = '%(asctime)s %(levelname)s: %(message)s'
+    loglevel = logging.INFO
+    if verbose:
+        loglevel = logging.DEBUG
+    elif quiet:
+        loglevel = logging.WARN
+
+    logging.basicConfig(format=logformat, level=loglevel)
+
+    if verbose and quiet:
+        logging.critical(_("Conflicting arguments: '--verbose' and '--quiet' "
+                           "can not be specified at the same time."))
+        sys.exit(1)
+
+    # Trick argparse into displaying the right usage when --help is used.
+    sys.argv[0] += ' ' + command
+
+    del sys.argv[1]
+    if command in COMMANDS.keys():
+        # import is named import_subcommand internally b/c import is reserved by Python
+        command = 'import_subcommand' if command == 'import' else command
+        mod = __import__('fdroidserver.' + command, None, None, [command])
+    else:
+        mod = __import__(available_plugins[command]['name'], None, None, [command])
+
+    system_encoding = sys.getdefaultencoding()
+    if system_encoding is None or system_encoding.lower() not in ('utf-8', 'utf8'):
+        logging.warning(_("Encoding is set to '{enc}' fdroid might run "
+                          "into encoding issues. Please set it to 'UTF-8' "
+                          "for best results.".format(enc=system_encoding)))
+
+    try:
+        mod.main()
+    # These are ours, contain a proper message and are "expected"
+    except (fdroidserver.common.FDroidException,
+            fdroidserver.metadata.MetaDataException) as e:
+        if verbose:
+            raise
+        else:
+            logging.critical(str(e))
+        sys.exit(1)
+    except ArgumentError as e:
+        logging.critical(str(e))
+        sys.exit(1)
+    except KeyboardInterrupt:
+        print('')
+        fdroidserver.common.force_exit(1)
+    # These should only be unexpected crashes due to bugs in the code
+    # str(e) often doesn't contain a reason, so just show the backtrace
+    except Exception as e:
+        logging.critical(_("Unknown exception found!"))
+        raise e
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/_yaml.py

@@ -0,0 +1,64 @@
+# Copyright (C) 2025, Hans-Christoph Steiner <hans@eds.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Standard YAML parsing and dumping.
+
+YAML 1.2 is the preferred format for all data files.  When loading
+F-Droid formats like config.yml and <Application ID>.yml, YAML 1.2 is
+forced, and older YAML constructs should be considered an error.
+
+It is OK to load and dump files in other YAML versions if they are
+externally defined formats, like FUNDING.yml.  In those cases, these
+common instances might not be appropriate to use.
+
+There is a separate instance for dumping based on the "round trip" aka
+"rt" mode.  The "rt" mode maintains order while the "safe" mode sorts
+the output.  Also, yaml.version is not forced in the dumper because that
+makes it write out a "%YAML 1.2" header.  F-Droid's formats are
+explicitly defined as YAML 1.2 and meant to be human-editable.  So that
+header gets in the way.
+
+"""
+
+import ruamel.yaml
+
+yaml = ruamel.yaml.YAML(typ='safe')
+yaml.version = (1, 2)
+
+yaml_dumper = ruamel.yaml.YAML(typ='rt')
+
+
+def config_dump(config, fp=None):
+    """Dump config data in YAML 1.2 format without headers.
+
+    This outputs YAML in a string that is suitable for use in regexps
+    and string replacements, as well as complete files.  It is therefore
+    explicitly set up to avoid writing out headers and footers.
+
+    This is modeled after PyYAML's yaml.dump(), which can dump to a file
+    or return a string.
+
+    https://yaml.dev/doc/ruamel.yaml/example/#Output_of_%60dump()%60_as_a_string
+
+    """
+    dumper = ruamel.yaml.YAML(typ='rt')
+    dumper.default_flow_style = False
+    dumper.explicit_start = False
+    dumper.explicit_end = False
+    if fp is None:
+        with ruamel.yaml.compat.StringIO() as fp:
+            dumper.dump(config, fp)
+            return fp.getvalue()
+    dumper.dump(config, fp)

fdroidserver/asynchronousfilereader/__init__.py

@@ -0,0 +1,56 @@
+"""Simple thread based asynchronous file reader for Python.
+
+AsynchronousFileReader
+======================
+
+see https://github.com/soxofaan/asynchronousfilereader
+
+MIT License
+Copyright (c) 2014 Stefaan Lippens
+"""
+
+__version__ = '0.2.1'
+
+import threading
+
+try:
+    # Python 2
+    from Queue import Queue
+except ImportError:
+    # Python 3
+    from queue import Queue
+
+
+class AsynchronousFileReader(threading.Thread):
+    """Helper class to implement asynchronous reading of a file in a separate thread.
+
+    Pushes read lines on a queue to be consumed in another thread.
+    """
+
+    def __init__(self, fd, queue=None, autostart=True):
+        self._fd = fd
+        if queue is None:
+            queue = Queue()
+        self.queue = queue
+
+        threading.Thread.__init__(self)
+
+        if autostart:
+            self.start()
+
+    def run(self):
+        """Read lines and put them on the queue (the body of the tread)."""
+        while True:
+            line = self._fd.readline()
+            if not line:
+                break
+            self.queue.put(line)
+
+    def eof(self):
+        """Check whether there is no more content to expect."""
+        return not self.is_alive() and self.queue.empty()
+
+    def readlines(self):
+        """Get currently available lines."""
+        while not self.queue.empty():
+            yield self.queue.get()

fdroidserver/btlog.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+"""Update the binary transparency log for a URL."""
+#
+# btlog.py - part of the FDroid server tools
+# Copyright (C) 2017, Hans-Christoph Steiner <hans@eds.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# This is for creating a binary transparency log in a git repo for any
+# F-Droid repo accessible via HTTP.  It is meant to run very often,
+# even once a minute in a cronjob, so it uses HEAD requests and the
+# HTTP ETag to check if the file has changed.  HEAD requests should
+# not count against the download counts.  This pattern of a HEAD then
+# a GET is what fdroidclient uses to avoid ETags being abused as
+# cookies. This also uses the same HTTP User Agent as the F-Droid
+# client app so its not easy for the server to distinguish this from
+# the F-Droid client.
+
+import collections
+import glob
+import json
+import logging
+import os
+import shutil
+import tempfile
+import zipfile
+from argparse import ArgumentParser
+from typing import Optional
+
+import defusedxml.minidom
+import git
+import requests
+
+from . import _, common, deploy
+from .exception import FDroidException
+
+
+def make_binary_transparency_log(
+    repodirs: collections.abc.Iterable,
+    btrepo: str = 'binary_transparency',
+    url: Optional[str] = None,
+    commit_title: str = 'fdroid update',
+):
+    """Log the indexes in a standalone git repo to serve as a "binary transparency" log.
+
+    Parameters
+    ----------
+    repodirs
+        The directories of the F-Droid repository to generate the binary
+        transparency log for.
+    btrepo
+        The path to the Git repository of the binary transparency log.
+    url
+        The URL of the F-Droid repository to generate the binary transparency
+        log for.
+    commit_title
+        The commit title for commits in the binary transparency log Git
+        repository.
+
+    Notes
+    -----
+    Also see https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies .
+    """
+    logging.info('Committing indexes to ' + btrepo)
+    if os.path.exists(os.path.join(btrepo, '.git')):
+        gitrepo = git.Repo(btrepo)
+    else:
+        if not os.path.exists(btrepo):
+            os.mkdir(btrepo)
+        gitrepo = git.Repo.init(btrepo, initial_branch=deploy.GIT_BRANCH)
+
+        if not url:
+            url = common.config['repo_url'].rstrip('/')
+        with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
+            fp.write(
+                """
+# Binary Transparency Log for %s
+
+This is a log of the signed app index metadata.  This is stored in a
+git repo, which serves as an imperfect append-only storage mechanism.
+People can then check that any file that they received from that
+F-Droid repository was a publicly released file.
+
+For more info on this idea:
+* https://wiki.mozilla.org/Security/Binary_Transparency
+"""
+                % url[: url.rindex('/')]  # strip '/repo'
+            )
+        gitrepo.index.add(['README.md'])
+        gitrepo.index.commit('add README')
+
+    for repodir in repodirs:
+        cpdir = os.path.join(btrepo, repodir)
+        if not os.path.exists(cpdir):
+            os.mkdir(cpdir)
+        for f in ('index.xml', 'index-v1.json', 'index-v2.json', 'entry.json'):
+            repof = os.path.join(repodir, f)
+            if not os.path.exists(repof):
+                continue
+            dest = os.path.join(cpdir, f)
+            if f.endswith('.xml'):
+                doc = defusedxml.minidom.parse(repof)
+                output = doc.toprettyxml(encoding='utf-8')
+                with open(dest, 'wb') as f:
+                    f.write(output)
+            elif f.endswith('.json'):
+                with open(repof) as fp:
+                    output = json.load(fp, object_pairs_hook=collections.OrderedDict)
+                with open(dest, 'w') as fp:
+                    json.dump(output, fp, indent=2)
+            gitrepo.index.add([repof])
+        for f in ('index.jar', 'index-v1.jar', 'entry.jar'):
+            repof = os.path.join(repodir, f)
+            if not os.path.exists(repof):
+                continue
+            dest = os.path.join(cpdir, f)
+            jarin = zipfile.ZipFile(repof, 'r')
+            jarout = zipfile.ZipFile(dest, 'w')
+            for info in jarin.infolist():
+                if info.filename.startswith('META-INF/'):
+                    jarout.writestr(info, jarin.read(info.filename))
+            jarout.close()
+            jarin.close()
+            gitrepo.index.add([repof])
+
+        output_files = []
+        for root, dirs, files in os.walk(repodir):
+            for f in files:
+                output_files.append(os.path.relpath(os.path.join(root, f), repodir))
+        output = collections.OrderedDict()
+        for f in sorted(output_files):
+            repofile = os.path.join(repodir, f)
+            stat = os.stat(repofile)
+            output[f] = (
+                stat.st_size,
+                stat.st_ctime_ns,
+                stat.st_mtime_ns,
+                stat.st_mode,
+                stat.st_uid,
+                stat.st_gid,
+            )
+        fslogfile = os.path.join(cpdir, 'filesystemlog.json')
+        with open(fslogfile, 'w') as fp:
+            json.dump(output, fp, indent=2)
+        gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json')])
+
+        for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')):
+            gitrepo.index.add([os.path.join(repodir, os.path.basename(f))])
+
+    gitrepo.index.commit(commit_title)
+
+
+def main():
+    """Generate or update a binary transparency log for a F-Droid repository.
+
+    The behaviour of this function is influenced by the configuration file as
+    well as command line parameters.
+
+    Raises
+    ------
+    :exc:`~fdroidserver.exception.FDroidException`
+        If the specified or default Git repository does not exist.
+
+    """
+    parser = ArgumentParser()
+    common.setup_global_opts(parser)
+    parser.add_argument(
+        "--git-repo",
+        default=os.path.join(os.getcwd(), 'binary_transparency'),
+        help=_("Path to the git repo to use as the log"),
+    )
+    parser.add_argument(
+        "-u",
+        "--url",
+        default='https://f-droid.org',
+        help=_("The base URL for the repo to log (default: https://f-droid.org)"),
+    )
+    parser.add_argument(
+        "--git-remote",
+        default=None,
+        help=_("Push the log to this git remote repository"),
+    )
+    options = common.parse_args(parser)
+
+    if options.verbose:
+        logging.getLogger("requests").setLevel(logging.INFO)
+        logging.getLogger("urllib3").setLevel(logging.INFO)
+    else:
+        logging.getLogger("requests").setLevel(logging.WARNING)
+        logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+    if not os.path.exists(options.git_repo):
+        raise FDroidException(
+            '"%s" does not exist! Create it, or use --git-repo' % options.git_repo
+        )
+
+    session = requests.Session()
+
+    new_files = False
+    repodirs = ('repo', 'archive')
+    tempdirbase = tempfile.mkdtemp(prefix='.fdroid-btlog-')
+    for repodir in repodirs:
+        # TODO read HTTP headers for etag from git repo
+        tempdir = os.path.join(tempdirbase, repodir)
+        os.makedirs(tempdir, exist_ok=True)
+        gitrepodir = os.path.join(options.git_repo, repodir)
+        os.makedirs(gitrepodir, exist_ok=True)
+        for f in (
+            'entry.jar',
+            'entry.json',
+            'index-v1.jar',
+            'index-v1.json',
+            'index-v2.json',
+            'index.jar',
+            'index.xml',
+        ):
+            dlfile = os.path.join(tempdir, f)
+            dlurl = options.url + '/' + repodir + '/' + f
+            http_headers_file = os.path.join(gitrepodir, f + '.HTTP-headers.json')
+
+            headers = {'User-Agent': 'F-Droid 0.102.3'}
+            etag = None
+            if os.path.exists(http_headers_file):
+                with open(http_headers_file) as fp:
+                    etag = json.load(fp)['ETag']
+
+            r = session.head(dlurl, headers=headers, allow_redirects=False)
+            if r.status_code != 200:
+                logging.debug(
+                    'HTTP Response (%d), did not download %s' % (r.status_code, dlurl)
+                )
+                continue
+            if etag and etag == r.headers.get('ETag'):
+                logging.debug('ETag matches, did not download ' + dlurl)
+                continue
+
+            r = session.get(dlurl, headers=headers, allow_redirects=False)
+            if r.status_code == 200:
+                with open(dlfile, 'wb') as f:
+                    for chunk in r:
+                        f.write(chunk)
+
+                dump = dict()
+                for k, v in r.headers.items():
+                    dump[k] = v
+                with open(http_headers_file, 'w') as fp:
+                    json.dump(dump, fp, indent=2, sort_keys=True)
+                new_files = True
+
+    if new_files:
+        os.chdir(tempdirbase)
+        make_binary_transparency_log(
+            repodirs, options.git_repo, options.url, 'fdroid btlog'
+        )
+    if options.git_remote:
+        deploy.push_binary_transparency(options.git_repo, options.git_remote)
+    shutil.rmtree(tempdirbase, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/exception.py

@@ -0,0 +1,58 @@
+class FDroidException(Exception):
+    def __init__(self, value=None, detail=None):
+        super().__init__()
+        self.value = value
+        self.detail = detail
+
+    def shortened_detail(self):
+        if len(self.detail) < 16000:
+            return self.detail
+        return '[...]\n' + self.detail[-16000:]
+
+    def __str__(self):
+        if self.value is None:
+            ret = __name__
+        else:
+            ret = str(self.value)
+        if self.detail:
+            ret += (
+                "\n==== detail begin ====\n%s\n==== detail end ===="
+                % ''.join(self.detail).strip()
+            )
+        return ret
+
+
+class MetaDataException(Exception):
+    def __init__(self, value):
+        super().__init__()
+        self.value = value
+
+    def __str__(self):
+        return self.value
+
+
+class VCSException(FDroidException):
+    pass
+
+
+class NoVersionCodeException(FDroidException):
+    pass
+
+
+class NoSubmodulesException(VCSException):
+    pass
+
+
+class BuildException(FDroidException):
+    pass
+
+
+class VerificationException(FDroidException):
+    pass
+
+
+class ConfigurationException(FDroidException):
+    def __init__(self, value=None, detail=None):
+        super().__init__()
+        self.value = value
+        self.detail = detail

fdroidserver/getsig/getsig.java

@@ -1,105 +0,0 @@
-import java.io.IOException;
-import java.io.InputStream;
-import java.math.BigInteger;
-import java.security.Signature;
-import java.security.cert.*;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.Enumeration;
-import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
-
-public class getsig {
-
-    public static void main(String[] args) {
-
-        String apkPath = null;
-        boolean full = false;
-
-        if(args.length == 1) {
-            apkPath = args[0];
-        } else if (args.length == 2) {
-            if(!args[0].equals("-f")) {
-                System.out.println("Only -f is supported");
-                System.exit(1);
-            }
-            apkPath = args[1];
-            full = true;
-        } else {
-            System.out.println("Specify the APK file to get the signature from!");
-            System.exit(1);
-        }
-
-        try {
-
-            JarFile apk = new JarFile(apkPath);
-            java.security.cert.Certificate[] certs = null;
-
-            Enumeration entries = apk.entries();
-            while (entries.hasMoreElements()) {
-                JarEntry je = (JarEntry) entries.nextElement();
-                if (!je.isDirectory() && !je.getName().startsWith("META-INF/")) {
-                    // Just need to read the stream (discarding the data) to get
-                    // it to process the certificate...
-                    byte[] b = new byte[4096];
-                    InputStream is = apk.getInputStream(je);
-                    while (is.read(b, 0, b.length) != -1);
-                    is.close();
-                    certs = je.getCertificates();
-                    if(certs != null)
-                        break;
-                }
-            }
-            apk.close();
-
-            if (certs == null) {
-                System.out.println("Not signed");
-                System.exit(1);
-            }
-            if (certs.length != 1) {
-                System.out.println("One signature expected");
-                System.exit(1);
-            }
-
-            // Get the signature in the same form that is returned by
-            // android.content.pm.Signature.toCharsString() (but in the
-            // form of a byte array so we can pass it to the MD5 function)...
-            byte[] sig = certs[0].getEncoded();
-            byte[] csig = new byte[sig.length * 2];
-            for (int j=0; j<sig.length; j++) {
-                byte v = sig[j];
-                int d = (v>>4)&0xf;
-                csig[j*2] = (byte)(d >= 10 ? ('a' + d - 10) : ('0' + d));
-                d = v&0xf;
-                csig[j*2+1] = (byte)(d >= 10 ? ('a' + d - 10) : ('0' + d));
-            }
-
-            String result;
-            if(full) {
-                result = new String(csig);
-            } else {
-                // Get the MD5 sum...
-                MessageDigest md;
-                md = MessageDigest.getInstance("MD5");
-                byte[] md5sum = new byte[32];
-                md.update(csig);
-                md5sum = md.digest();
-                BigInteger bigInt = new BigInteger(1, md5sum);
-                String md5hash = bigInt.toString(16);
-                while (md5hash.length() < 32)
-                    md5hash = "0" + md5hash;
-                result = md5hash;
-            }
-
-            System.out.println("Result:" + result);
-            System.exit(0);
-
-        } catch (Exception e) {
-            System.out.println("Exception:" + e);
-            System.exit(1);
-        }
-    }
-
-}
-
-

fdroidserver/getsig/make.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-javac getsig.java

fdroidserver/getsig/run.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-java getsig $1 $2 $3

fdroidserver/github.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+#
+# github.py - part of the FDroid server tools
+# Copyright (C) 2024, Michael Pöhn, michael@poehn.at
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import pathlib
+import urllib.parse
+import urllib.request
+
+
+class GithubApi:
+    """Wrapper for some select calls to GitHub Json/REST API.
+
+    This class wraps some calls to api.github.com. This is not intended to be a
+    general API wrapper. Instead it's purpose is to return pre-filtered and
+    transformed data that's playing well with other fdroidserver functions.
+
+    With the GitHub API, the token is optional, but it has pretty
+    severe rate limiting.
+
+    """
+
+    def __init__(self, api_token, repo_path):
+        self._api_token = api_token
+        if repo_path.startswith("https://github.com/"):
+            self._repo_path = repo_path[19:]
+        else:
+            self._repo_path = repo_path
+
+    def _req(self, url, data=None):
+        h = {
+            "Accept": "application/vnd.github+json",
+            "X-GitHub-Api-Version": "2022-11-28",
+        }
+        if self._api_token:
+            h["Authorization"] = f"Bearer {self._api_token}"
+        return urllib.request.Request(
+            url,
+            headers=h,
+            data=data,
+        )
+
+    def list_released_tags(self):
+        """List of all tags that are associated with a release for this repo on GitHub."""
+        names = []
+        req = self._req(f"https://api.github.com/repos/{self._repo_path}/releases")
+        with urllib.request.urlopen(req) as resp:  # nosec CWE-22 disable bandit warning
+            releases = json.load(resp)
+            for release in releases:
+                names.append(release['tag_name'])
+        return names
+
+    def list_unreleased_tags(self):
+        all_tags = self.list_all_tags()
+        released_tags = self.list_released_tags()
+        return [x for x in all_tags if x not in released_tags]
+
+    def get_latest_apk(self):
+        req = self._req(
+            f"https://api.github.com/repos/{self._repo_path}/releases/latest"
+        )
+        with urllib.request.urlopen(req) as resp:  # nosec CWE-22 disable bandit warning
+            assets = json.load(resp)['assets']
+            for asset in assets:
+                url = asset.get('browser_download_url')
+                if url and url.endswith('.apk'):
+                    return url
+
+    def tag_exists(self, tag):
+        """
+        Check if git tag is present on github.
+
+        https://docs.github.com/en/rest/git/refs?apiVersion=2022-11-28#list-matching-references--fine-grained-access-tokens
+        """
+        req = self._req(
+            f"https://api.github.com/repos/{self._repo_path}/git/matching-refs/tags/{tag}"
+        )
+        with urllib.request.urlopen(req) as resp:  # nosec CWE-22 disable bandit warning
+            rd = json.load(resp)
+            return len(rd) == 1 and rd[0].get("ref", False) == f"refs/tags/{tag}"
+        return False
+
+    def list_all_tags(self):
+        """Get list of all tags for this repo on GitHub."""
+        tags = []
+        req = self._req(
+            f"https://api.github.com/repos/{self._repo_path}/git/matching-refs/tags/"
+        )
+        with urllib.request.urlopen(req) as resp:  # nosec CWE-22 disable bandit warning
+            refs = json.load(resp)
+            for ref in refs:
+                r = ref.get('ref', '')
+                if r.startswith('refs/tags/'):
+                    tags.append(r[10:])
+        return tags
+
+    def create_release(self, tag, files, body=''):
+        """
+        Create a new release on github.
+
+        also see: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#create-a-release
+
+        :returns: True if release was created, False if release already exists
+        :raises: urllib exceptions in case of network or api errors, also
+                 raises an exception when the tag doesn't exists.
+        """
+        # Querying github to create a new release for a non-existent tag, will
+        # also create that tag on github. So we need an additional check to
+        # prevent this behavior.
+        if not self.tag_exists(tag):
+            raise Exception(
+                f"can't create github release for {self._repo_path} {tag}, tag doesn't exists"
+            )
+        # create the relase on github
+        req = self._req(
+            f"https://api.github.com/repos/{self._repo_path}/releases",
+            data=json.dumps(
+                {
+                    "tag_name": tag,
+                    "body": body,
+                }
+            ).encode("utf-8"),
+        )
+        try:
+            with urllib.request.urlopen(  # nosec CWE-22 disable bandit warning
+                req
+            ) as resp:
+                release_id = json.load(resp)['id']
+        except urllib.error.HTTPError as e:
+            if e.status == 422:
+                codes = [x['code'] for x in json.load(e).get('errors', [])]
+                if "already_exists" in codes:
+                    return False
+            raise e
+
+        # attach / upload all files for the relase
+        for file in files:
+            self._create_release_asset(release_id, file)
+
+        return True
+
+    def _create_release_asset(self, release_id, file):
+        """
+        Attach a file to a release on GitHub.
+
+        This uploads a file to github relases, it will be attached to the supplied release
+
+        also see: https://docs.github.com/en/rest/releases/assets?apiVersion=2022-11-28#upload-a-release-asset
+        """
+        file = pathlib.Path(file)
+        with open(file, 'rb') as f:
+            req = urllib.request.Request(
+                f"https://uploads.github.com/repos/{self._repo_path}/releases/{release_id}/assets?name={file.name}",
+                headers={
+                    "Accept": "application/vnd.github+json",
+                    "Authorization": f"Bearer {self._api_token}",
+                    "X-GitHub-Api-Version": "2022-11-28",
+                    "Content-Type": "application/octet-stream",
+                },
+                data=f.read(),
+            )
+            with urllib.request.urlopen(req):  # nosec CWE-22 disable bandit warning
+                return True
+            return False

fdroidserver/gpgsign.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+#
+# gpgsign.py - part of the FDroid server tools
+# Copyright (C) 2014, Ciaran Gultnieks, ciaran@ciarang.com
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import glob
+import logging
+import os
+import time
+from argparse import ArgumentParser
+
+from . import _, common
+from .common import FDroidPopen
+from .exception import FDroidException
+
+config = None
+start_timestamp = time.gmtime()
+
+
+def status_update_json(signed):
+    """Output a JSON file with metadata about this run."""
+    logging.debug(_('Outputting JSON'))
+    output = common.setup_status_output(start_timestamp)
+    if signed:
+        output['signed'] = signed
+    common.write_status_json(output)
+
+
+def main():
+    global config
+
+    # Parse command line...
+    parser = ArgumentParser()
+    common.setup_global_opts(parser)
+    common.parse_args(parser)
+
+    config = common.read_config()
+
+    repodirs = ['repo']
+    if config['archive_older'] != 0:
+        repodirs.append('archive')
+
+    signed = []
+    for output_dir in repodirs:
+        if not os.path.isdir(output_dir):
+            raise FDroidException(
+                _("Missing output directory") + " '" + output_dir + "'"
+            )
+
+        # Process any apks that are waiting to be signed...
+        for f in sorted(glob.glob(os.path.join(output_dir, '*.*'))):
+            if not common.is_repo_file(f, for_gpg_signing=True):
+                continue
+            filename = os.path.basename(f)
+            sigfilename = filename + ".asc"
+            sigpath = os.path.join(output_dir, sigfilename)
+
+            if not os.path.exists(sigpath):
+                gpgargs = ['gpg', '-a', '--output', sigpath, '--detach-sig']
+                if 'gpghome' in config:
+                    gpgargs.extend(['--homedir', config['gpghome']])
+                if 'gpgkey' in config:
+                    gpgargs.extend(['--local-user', config['gpgkey']])
+                gpgargs.append(os.path.join(output_dir, filename))
+                p = FDroidPopen(gpgargs)
+                if p.returncode != 0:
+                    raise FDroidException("Signing failed.")
+
+                signed.append(filename)
+                logging.info('Signed ' + filename)
+    status_update_json(signed)
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/import.py

@@ -1,303 +0,0 @@
-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
-#
-# import.py - part of the FDroid server tools
-# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
-# Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import sys
-import os
-import shutil
-import urllib
-from optparse import OptionParser
-from ConfigParser import ConfigParser
-import common, metadata
-
-# Get the repo type and address from the given web page. The page is scanned
-# in a rather naive manner for 'git clone xxxx', 'hg clone xxxx', etc, and
-# when one of these is found it's assumed that's the information we want.
-# Returns repotype, address, or None, reason
-def getrepofrompage(url):
-
-    req = urllib.urlopen(url)
-    if req.getcode() != 200:
-        return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
-    page = req.read()
-
-    # Works for Google Code and BitBucket...
-    index = page.find('hg clone')
-    if index != -1:
-        repotype = 'hg'
-        repo = page[index + 9:]
-        index = repo.find('<')
-        if index == -1:
-            return (None, "Error while getting repo address")
-        repo = repo[:index]
-        repo = repo.split('"')[0]
-        return (repotype, repo)
-
-    # Works for Google Code and BitBucket...
-    index=page.find('git clone')
-    if index != -1:
-        repotype = 'git'
-        repo = page[index + 10:]
-        index = repo.find('<')
-        if index == -1:
-            return (None, "Error while getting repo address")
-        repo = repo[:index]
-        repo = repo.split('"')[0]
-        return (repotype, repo)
-
-    # Google Code only...
-    index=page.find('svn checkout')
-    if index != -1:
-        repotype = 'git-svn'
-        repo = page[index + 13:]
-        prefix = '<strong><em>http</em></strong>'
-        if not repo.startswith(prefix):
-            return (None, "Unexpected checkout instructions format")
-        repo = 'http' + repo[len(prefix):]
-        index = repo.find('<')
-        if index == -1:
-            return (None, "Error while getting repo address - no end tag? '" + repo + "'")
-            sys.exit(1)
-        repo = repo[:index]
-        index = repo.find(' ')
-        if index == -1:
-            return (None, "Error while getting repo address - no space? '" + repo + "'")
-        repo = repo[:index]
-        repo = repo.split('"')[0]
-        return (repotype, repo)
-
-    return (None, "No information found." + page)
-
-config = None
-options = None
-
-def main():
-
-    global config, options
-
-    # Parse command line...
-    parser = OptionParser()
-    parser.add_option("-u", "--url", default=None,
-                      help="Project URL to import from.")
-    parser.add_option("-s", "--subdir", default=None,
-                      help="Path to main android project subdirectory, if not in root.")
-    parser.add_option("-r", "--repo", default=None,
-                      help="Allows a different repo to be specified for a multi-repo google code project")
-    parser.add_option("--rev", default=None,
-                      help="Allows a different revision (or git branch) to be specified for the initial import")
-    (options, args) = parser.parse_args()
-
-    config = common.read_config(options)
-
-    if not options.url:
-        print "Specify project url."
-        sys.exit(1)
-    url = options.url
-
-    tmp_dir = 'tmp'
-    if not os.path.isdir(tmp_dir):
-        print "Creating temporary directory"
-        os.makedirs(tmp_dir)
-
-    # Get all apps...
-    apps = metadata.read_metadata()
-
-    # Figure out what kind of project it is...
-    projecttype = None
-    issuetracker = None
-    license = None
-    website = url #by default, we might override it
-    if url.startswith('git://'):
-        projecttype = 'git'
-        repo = url
-        repotype = 'git'
-        sourcecode = ""
-        website = ""
-    elif url.startswith('https://github.com'):
-        if url.endswith('/'):
-            url = url[:-1]
-        if url.endswith('.git'):
-            print "A github URL should point to the project, not the git repo"
-            sys.exit(1)
-        projecttype = 'github'
-        repo = url + '.git'
-        repotype = 'git'
-        sourcecode = url
-        issuetracker = url + '/issues'
-    elif url.startswith('https://gitorious.org/'):
-        projecttype = 'gitorious'
-        repo = 'https://git.gitorious.org/' + url[22:] + '.git'
-        repotype = 'git'
-        sourcecode = url
-    elif url.startswith('https://bitbucket.org/'):
-        if url.endswith('/'):
-            url = url[:-1]
-        projecttype = 'bitbucket'
-        sourcecode = url + '/src'
-        issuetracker = url + '/issues'
-        # Figure out the repo type and adddress...
-        repotype, repo = getrepofrompage(sourcecode)
-        if not repotype:
-            print "Unable to determine vcs type. " + repo
-            sys.exit(1)
-    elif url.startswith('http://code.google.com/p/'):
-        if not url.endswith('/'):
-            url += '/';
-        projecttype = 'googlecode'
-        sourcecode = url + 'source/checkout'
-        if options.repo:
-            sourcecode += "?repo=" + options.repo
-        issuetracker = url + 'issues/list'
-
-        # Figure out the repo type and adddress...
-        repotype, repo = getrepofrompage(sourcecode)
-        if not repotype:
-            print "Unable to determine vcs type. " + repo
-            sys.exit(1)
-
-        # Figure out the license...
-        req = urllib.urlopen(url)
-        if req.getcode() != 200:
-            print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode())
-            sys.exit(1)
-        page = req.read()
-        index = page.find('Code license')
-        if index == -1:
-            print "Couldn't find license data"
-            sys.exit(1)
-        ltext = page[index:]
-        lprefix = 'rel="nofollow">'
-        index = ltext.find(lprefix)
-        if index == -1:
-            print "Couldn't find license text"
-            sys.exit(1)
-        ltext = ltext[index + len(lprefix):]
-        index = ltext.find('<')
-        if index == -1:
-            print "License text not formatted as expected"
-            sys.exit(1)
-        ltext = ltext[:index]
-        if ltext == 'GNU GPL v3':
-            license = 'GPLv3'
-        elif ltext == 'GNU GPL v2':
-            license = 'GPLv2'
-        elif ltext == 'Apache License 2.0':
-            license = 'Apache2'
-        elif ltext == 'MIT License':
-            license = 'MIT'
-        elif ltext == 'GNU Lesser GPL':
-            license = 'LGPL'
-        elif ltext == 'Mozilla Public License 1.1':
-            license = 'MPL'
-        elif ltext == 'New BSD License':
-            license = 'NewBSD'
-        else:
-            print "License " + ltext + " is not recognised"
-            sys.exit(1)
-
-    if not projecttype:
-        print "Unable to determine the project type."
-        print "The URL you supplied was not in one of the supported formats. Please consult"
-        print "the manual for a list of supported formats, and supply one of those."
-        sys.exit(1)
-
-    # Get a copy of the source so we can extract some info...
-    print 'Getting source from ' + repotype + ' repo at ' + repo
-    src_dir = os.path.join(tmp_dir, 'importer')
-    if os.path.exists(src_dir):
-        shutil.rmtree(src_dir)
-    vcs = common.getvcs(repotype, repo, src_dir)
-    vcs.gotorevision(options.rev)
-    if options.subdir:
-        root_dir = os.path.join(src_dir, options.subdir)
-    else:
-        root_dir = src_dir
-
-    # Extract some information...
-    paths = common.manifest_paths(root_dir, None)
-    if paths:
-
-        version, vercode, package = common.parse_androidmanifests(paths)
-        if not package:
-            print "Couldn't find package ID"
-            sys.exit(1)
-        if not version:
-            print "WARNING: Couldn't find latest version name"
-        if not vercode:
-            print "WARNING: Couldn't find latest version code"
-    else:
-        spec = os.path.join(root_dir, 'buildozer.spec')
-        if os.path.exists(spec):
-            defaults = {'orientation': 'landscape', 'icon': '',
-                    'permissions': '', 'android.api': "18"}
-            bconfig = ConfigParser(defaults, allow_no_value=True)
-            bconfig.read(spec)
-            package = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
-            version = bconfig.get('app', 'version')
-            vercode = None
-        else:
-            print "No android or kivy project could be found. Specify --subdir?"
-            sys.exit(1)
-
-    # Make sure it's actually new...
-    for app in apps:
-        if app['id'] == package:
-            print "Package " + package + " already exists"
-            sys.exit(1)
-
-    # Construct the metadata...
-    app = metadata.parse_metadata(None)
-    app['id'] = package
-    app['Web Site'] = website
-    app['Source Code'] = sourcecode
-    if issuetracker:
-        app['Issue Tracker'] = issuetracker
-    if license:
-        app['License'] = license
-    app['Repo Type'] = repotype
-    app['Repo'] = repo
-    app['Update Check Mode'] = "Tags"
-
-    # Create a build line...
-    build = {}
-    build['version'] = version if version else '?'
-    build['vercode'] = vercode if vercode else '?'
-    build['commit'] = '?'
-    build['disable'] = 'Generated by import.py - check/set version fields and commit id'
-    if options.subdir:
-        build['subdir'] = options.subdir
-    if os.path.exists(os.path.join(root_dir, 'jni')):
-        build['buildjni'] = 'yes'
-    app['builds'].append(build)
-
-    # Keep the repo directory to save bandwidth...
-    if not os.path.exists('build'):
-        os.mkdir('build')
-    shutil.move(src_dir, os.path.join('build', package))
-    with open('build/.fdroidvcs-' + package, 'w') as f:
-        f.write(repotype + ' ' + repo)
-
-    metafile = os.path.join('metadata', package + '.txt')
-    metadata.write_metadata(metafile, app)
-    print "Wrote " + metafile
-
-
-if __name__ == "__main__":
-    main()
-

fdroidserver/import_subcommand.py

@@ -0,0 +1,470 @@
+#!/usr/bin/env python3
+"""Extract application metadata from a source repository."""
+#
+# import_subcommand.py - part of the FDroid server tools
+# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
+# Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import logging
+import os
+import re
+import shutil
+import stat
+import sys
+import urllib
+from argparse import ArgumentParser
+from pathlib import Path
+from typing import Optional
+
+import git
+import yaml
+
+try:
+    from yaml import CSafeLoader as SafeLoader
+except ImportError:
+    from yaml import SafeLoader
+
+from . import _, common, metadata
+from .exception import FDroidException
+
+config = None
+
+SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?')
+GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''')
+APPLICATION_ID_REGEX = re.compile(r'''\s*applicationId\s=?\s?['"].*['"]''')
+
+
+def get_all_gradle_and_manifests(build_dir):
+    paths = []
+    for root, dirs, files in os.walk(build_dir):
+        for f in sorted(files):
+            if f == 'AndroidManifest.xml' or f.endswith(('.gradle', '.gradle.kts')):
+                full = Path(root) / f
+                paths.append(full)
+    return paths
+
+
+def get_gradle_subdir(build_dir, paths):
+    """Get the subdir where the gradle build is based."""
+    first_gradle_dir = None
+    for path in paths:
+        if not first_gradle_dir:
+            first_gradle_dir = path.parent.relative_to(build_dir)
+        if path.exists() and SETTINGS_GRADLE_REGEX.match(path.name):
+            for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text(encoding='utf-8')):
+                for f in (path.parent / m.group(1)).glob('build.gradle*'):
+                    with f.open(encoding='utf-8') as fp:
+                        for line in fp:
+                            if common.ANDROID_PLUGIN_REGEX.match(
+                                line
+                            ) or APPLICATION_ID_REGEX.match(line):
+                                return f.parent.relative_to(build_dir)
+    if first_gradle_dir and first_gradle_dir != Path('.'):
+        return first_gradle_dir
+
+
+def handle_retree_error_on_windows(function, path, excinfo):
+    """Python can't remove a readonly file on Windows so chmod first."""
+    if function in (os.unlink, os.rmdir, os.remove) and excinfo[0] == PermissionError:
+        os.chmod(path, stat.S_IWRITE)
+        function(path)
+
+
+def clone_to_tmp_dir(app: metadata.App, rev=None) -> Path:
+    """Clone the source repository of an app to a temporary directory for further processing.
+
+    Parameters
+    ----------
+    app
+        The App instance to clone the source of.
+
+    Returns
+    -------
+    tmp_dir
+        The (temporary) directory the apps source has been cloned into.
+
+    """
+    tmp_dir = Path('tmp')
+    tmp_dir.mkdir(exist_ok=True)
+
+    tmp_dir = tmp_dir / 'importer'
+
+    if tmp_dir.exists():
+        shutil.rmtree(str(tmp_dir), onerror=handle_retree_error_on_windows)
+    vcs = common.getvcs(app.RepoType, app.Repo, tmp_dir)
+    vcs.gotorevision(rev)
+
+    return tmp_dir
+
+
+def getrepofrompage(url: str) -> tuple[Optional[str], str]:
+    """Get the repo type and address from the given web page.
+
+    The page is scanned in a rather naive manner for 'git clone xxxx',
+    'hg clone xxxx', etc, and when one of these is found it's assumed
+    that's the information we want.  Returns repotype, address, or
+    None, reason
+
+    Parameters
+    ----------
+    url
+        The url to look for repository information at.
+
+    Returns
+    -------
+    repotype_or_none
+        The found repository type or None if an error occured.
+    address_or_reason
+        The address to the found repository or the reason if an error occured.
+
+    """
+    if not url.startswith('http'):
+        return (None, _('{url} does not start with "http"!'.format(url=url)))
+    req = urllib.request.urlopen(url)  # nosec B310 non-http URLs are filtered out
+    if req.getcode() != 200:
+        return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
+    page = req.read().decode(req.headers.get_content_charset())
+
+    # Works for BitBucket
+    m = re.search('data-fetch-url="(.*)"', page)
+    if m is not None:
+        repo = m.group(1)
+
+        if repo.endswith('.git'):
+            return ('git', repo)
+
+        return ('hg', repo)
+
+    # Works for BitBucket (obsolete)
+    index = page.find('hg clone')
+    if index != -1:
+        repotype = 'hg'
+        repo = page[index + 9 :]
+        index = repo.find('<')
+        if index == -1:
+            return (None, _("Error while getting repo address"))
+        repo = repo[:index]
+        repo = repo.split('"')[0]
+        return (repotype, repo)
+
+    # Works for BitBucket (obsolete)
+    index = page.find('git clone')
+    if index != -1:
+        repotype = 'git'
+        repo = page[index + 10 :]
+        index = repo.find('<')
+        if index == -1:
+            return (None, _("Error while getting repo address"))
+        repo = repo[:index]
+        repo = repo.split('"')[0]
+        return (repotype, repo)
+
+    return (None, _("No information found.") + page)
+
+
+def get_app_from_url(url: str) -> metadata.App:
+    """Guess basic app metadata from the URL.
+
+    The URL must include a network hostname, unless it is an lp:,
+    file:, or git/ssh URL.  This throws ValueError on bad URLs to
+    match urlparse().
+
+    Parameters
+    ----------
+    url
+        The URL to look to look for app metadata at.
+
+    Returns
+    -------
+    app
+        App instance with the found metadata.
+
+    Raises
+    ------
+    :exc:`~fdroidserver.exception.FDroidException`
+        If the VCS type could not be determined.
+    :exc:`ValueError`
+        If the URL is invalid.
+
+    """
+    parsed = urllib.parse.urlparse(url)
+    invalid_url = False
+    if not parsed.scheme or not parsed.path:
+        invalid_url = True
+
+    app = metadata.App()
+    app.Repo = url
+    if url.startswith('git://') or url.startswith('git@'):
+        app.RepoType = 'git'
+    elif parsed.netloc == 'github.com':
+        app.RepoType = 'git'
+        app.SourceCode = url
+        app.IssueTracker = url + '/issues'
+    elif parsed.netloc in ('gitlab.com', 'framagit.org'):
+        # git can be fussy with gitlab URLs unless they end in .git
+        if url.endswith('.git'):
+            url = url[:-4]
+        app.Repo = url + '.git'
+        app.RepoType = 'git'
+        app.SourceCode = url
+        app.IssueTracker = url + '/issues'
+    elif parsed.netloc == 'notabug.org':
+        if url.endswith('.git'):
+            url = url[:-4]
+        app.Repo = url + '.git'
+        app.RepoType = 'git'
+        app.SourceCode = url
+        app.IssueTracker = url + '/issues'
+    elif parsed.netloc == 'bitbucket.org':
+        if url.endswith('/'):
+            url = url[:-1]
+        app.SourceCode = url + '/src'
+        app.IssueTracker = url + '/issues'
+        # Figure out the repo type and adddress...
+        app.RepoType, app.Repo = getrepofrompage(url)
+    elif parsed.netloc == 'codeberg.org':
+        app.RepoType = 'git'
+        app.SourceCode = url
+        app.IssueTracker = url + '/issues'
+    elif url.startswith('https://') and url.endswith('.git'):
+        app.RepoType = 'git'
+
+    if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'):
+        invalid_url = True
+
+    if invalid_url:
+        raise ValueError(_('"{url}" is not a valid URL!'.format(url=url)))
+
+    if not app.RepoType:
+        raise FDroidException("Unable to determine vcs type. " + app.Repo)
+
+    return app
+
+
+def main():
+    """Extract app metadata and write it to a file.
+
+    The behaviour of this function is influenced by the configuration file as
+    well as command line parameters.
+
+    Raises
+    ------
+    :exc:`~fdroidserver.exception.FDroidException`
+        If the repository already has local metadata, no URL is specified and
+        the current directory is not a Git repository, no application ID could
+        be found, no Gradle project could be found or there is already metadata
+        for the found application ID.
+
+    """
+    global config
+
+    # Parse command line...
+    parser = ArgumentParser()
+    common.setup_global_opts(parser)
+    parser.add_argument("-u", "--url", help=_("Project URL to import from."))
+    parser.add_argument(
+        "-s",
+        "--subdir",
+        help=_("Path to main Android project subdirectory, if not in root."),
+    )
+    parser.add_argument(
+        "-c",
+        "--categories",
+        help=_("Comma separated list of categories."),
+    )
+    parser.add_argument("-l", "--license", help=_("Overall license of the project."))
+    parser.add_argument(
+        "--omit-disable",
+        action="store_true",
+        help=_("Do not add 'disable:' to the generated build entries"),
+    )
+    parser.add_argument(
+        "--rev",
+        help=_(
+            "Allows a different revision (or git branch) to be specified for the initial import"
+        ),
+    )
+    metadata.add_metadata_arguments(parser)
+    options = common.parse_args(parser)
+    metadata.warnings_action = options.W
+
+    config = common.read_config()
+
+    apps = metadata.read_metadata()
+    app = None
+
+    tmp_importer_dir = None
+
+    local_metadata_files = common.get_local_metadata_files()
+    if local_metadata_files:
+        raise FDroidException(
+            _("This repo already has local metadata: %s") % local_metadata_files[0]
+        )
+
+    build = metadata.Build()
+    app = metadata.App()
+    if options.url is None and Path('.git').is_dir():
+        app.RepoType = 'git'
+        tmp_importer_dir = Path.cwd()
+        git_repo = git.Repo(tmp_importer_dir)
+        for remote in git.Remote.iter_items(git_repo):
+            if remote.name == 'origin':
+                url = git_repo.remotes.origin.url
+                app = get_app_from_url(url)
+                break
+        write_local_file = True
+    elif options.url:
+        app = get_app_from_url(options.url)
+        tmp_importer_dir = clone_to_tmp_dir(app, options.rev)
+        git_repo = git.Repo(tmp_importer_dir)
+
+        if not options.omit_disable:
+            build.disable = (
+                'Generated by `fdroid import` - check version fields and commitid'
+            )
+        write_local_file = False
+    else:
+        raise FDroidException("Specify project url.")
+
+    app.AutoUpdateMode = 'Version'
+    app.UpdateCheckMode = 'Tags'
+    build.commit = common.get_head_commit_id(tmp_importer_dir)
+
+    # Extract some information...
+    paths = get_all_gradle_and_manifests(tmp_importer_dir)
+    gradle_subdir = get_gradle_subdir(tmp_importer_dir, paths)
+    if paths:
+        versionName, versionCode, appid = common.parse_androidmanifests(paths, app)
+        if not appid:
+            raise FDroidException(_("Couldn't find Application ID"))
+        if not versionName:
+            logging.warning(_('Could not find latest versionName'))
+        if not versionCode:
+            logging.warning(_('Could not find latest versionCode'))
+    else:
+        raise FDroidException(_("No gradle project could be found. Specify --subdir?"))
+
+    # Make sure it's actually new...
+    if appid in apps:
+        raise FDroidException(_('Package "{appid}" already exists').format(appid=appid))
+
+    # Create a build line...
+    build.versionName = versionName or 'Unknown'
+    app.CurrentVersion = build.versionName
+    build.versionCode = versionCode or 0
+    app.CurrentVersionCode = build.versionCode
+    if options.subdir:
+        build.subdir = options.subdir
+    elif gradle_subdir:
+        build.subdir = gradle_subdir.as_posix()
+    # subdir might be None
+    subdir = Path(tmp_importer_dir / build.subdir) if build.subdir else tmp_importer_dir
+
+    if options.license:
+        app.License = options.license
+    if options.categories:
+        app.Categories = options.categories.split(',')
+    if (subdir / 'jni').exists():
+        build.buildjni = ['yes']
+    if (subdir / 'build.gradle').exists() or (subdir / 'build.gradle.kts').exists():
+        build.gradle = ['yes']
+
+    app.AutoName = common.fetch_real_name(subdir, build.gradle)
+
+    package_json = tmp_importer_dir / 'package.json'  # react-native
+    pubspec_yaml = tmp_importer_dir / 'pubspec.yaml'  # flutter
+    if package_json.exists():
+        build.sudo = [
+            'sysctl fs.inotify.max_user_watches=524288 || true',
+            'apt-get update',
+            'apt-get install -y npm',
+        ]
+        build.init = ['npm install --build-from-source']
+        with package_json.open() as fp:
+            data = json.load(fp)
+        app.AutoName = app.AutoName or data.get('name')
+        app.License = data.get('license', app.License)
+        app.Description = data.get('description', app.Description)
+        app.WebSite = data.get('homepage', app.WebSite)
+        app_json = tmp_importer_dir / 'app.json'
+        build.scanignore = ['android/build.gradle']
+        build.scandelete = ['node_modules']
+        if app_json.exists():
+            with app_json.open() as fp:
+                data = json.load(fp)
+            app.AutoName = app.AutoName or data.get('name')
+    if pubspec_yaml.exists():
+        with pubspec_yaml.open() as fp:
+            data = yaml.load(fp, Loader=SafeLoader)
+        app.AutoName = app.AutoName or data.get('name')
+        app.License = data.get('license', app.License)
+        app.Description = data.get('description', app.Description)
+        app.UpdateCheckData = 'pubspec.yaml|version:\\s.+\\+(\\d+)|.|version:\\s(.+)\\+'
+        build.srclibs = ['flutter@stable']
+        build.output = 'build/app/outputs/flutter-apk/app-release.apk'
+        build.subdir = None
+        build.gradle = None
+        build.prebuild = [
+            'export PUB_CACHE=$(pwd)/.pub-cache',
+            '$$flutter$$/bin/flutter config --no-analytics',
+            '$$flutter$$/bin/flutter packages pub get',
+        ]
+        build.scandelete = [
+            '.pub-cache',
+        ]
+        build.build = [
+            'export PUB_CACHE=$(pwd)/.pub-cache',
+            '$$flutter$$/bin/flutter build apk',
+        ]
+
+    git_modules = tmp_importer_dir / '.gitmodules'
+    if git_modules.exists():
+        build.submodules = True
+
+    metadata.post_parse_yaml_metadata(app)
+
+    app['Builds'].append(build)
+
+    if write_local_file:
+        metadata.write_metadata(Path('.fdroid.yml'), app)
+    else:
+        # Keep the repo directory to save bandwidth...
+        Path('build').mkdir(exist_ok=True)
+        build_dir = Path('build') / appid
+        if build_dir.exists():
+            logging.warning(
+                _('{path} already exists, ignoring import results!').format(
+                    path=build_dir
+                )
+            )
+            sys.exit(1)
+        elif tmp_importer_dir:
+            # For Windows: Close the repo or a git.exe instance holds handles to repo
+            try:
+                git_repo.close()
+            except AttributeError:  # Debian/stretch's version does not have close()
+                pass
+            shutil.move(tmp_importer_dir, build_dir)
+        Path('build/.fdroidvcs-' + appid).write_text(app.RepoType + ' ' + app.Repo)
+
+        metadatapath = Path('metadata') / (appid + '.yml')
+        metadata.write_metadata(metadatapath, app)
+        logging.info("Wrote " + str(metadatapath))
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/init.py

@@ -1,9 +1,8 @@
-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 #
-# update.py - part of the FDroid server tools
+# init.py - part of the FDroid server tools
 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
-# Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
+# Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
 # Copyright (C) 2013 Hans-Christoph Steiner <hans@eds.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -19,206 +18,281 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import hashlib
+import glob
+import logging
 import os
 import re
 import shutil
 import socket
-import subprocess
 import sys
-from optparse import OptionParser
-
-import common
-from common import FDroidPopen, BuildException
+from argparse import ArgumentParser
 
+from . import _, common
+from .exception import FDroidException
 
 config = {}
-options = None
 
-def write_to_config(key, value):
-    '''write a key/value to the local config.py'''
-    with open('config.py', 'r') as f:
-        data = f.read()
-    pattern = key + '\s*=.*'
-    repl = key + ' = "' + value + '"'
+
+def disable_in_config(key, value):
+    """Write a key/value to the local config.yml, then comment it out."""
+    import yaml
+
+    with open(common.CONFIG_FILE) as fp:
+        data = fp.read()
+    pattern = r'\n[\s#]*' + key + r':.*'
+    repl = '\n#' + yaml.dump({key: value}, default_flow_style=False)
     data = re.sub(pattern, repl, data)
-    with open('config.py', 'w') as f:
-        f.writelines(data)
-
-
-def genpassword():
-    '''generate a random password for when generating keys'''
-    h = hashlib.sha256()
-    h.update(os.urandom(16)) # salt
-    h.update(bytes(socket.getfqdn()))
-    return h.digest().encode('base64').strip()
-
-
-def genkey(keystore, repo_keyalias, password, keydname):
-    '''generate a new keystore with a new key in it for signing repos'''
-    print('Generating a new key in "' + keystore + '"...')
-    p = FDroidPopen(['keytool', '-genkey',
-                '-keystore', keystore, '-alias', repo_keyalias,
-                '-keyalg', 'RSA', '-keysize', '4096',
-                '-sigalg', 'SHA256withRSA',
-                '-validity', '10000',
-                '-storepass', password, '-keypass', password,
-                '-dname', keydname])
-    if p.returncode != 0:
-        raise BuildException("Failed to generate key", p.stdout, p.stderr)
-    # now show the lovely key that was just generated
-    p = subprocess.Popen(['keytool', '-list', '-v',
-                '-keystore', keystore, '-alias', repo_keyalias],
-                stdin=subprocess.PIPE,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE)
-    output = p.communicate(password)[0]
-    print(output.lstrip().strip() + '\n\n')
+    with open(common.CONFIG_FILE, 'w') as fp:
+        fp.writelines(data)
 
 
 def main():
-
-    global options, config
+    global config
 
     # Parse command line...
-    parser = OptionParser()
-    parser.add_option("-v", "--verbose", action="store_true", default=False,
-                      help="Spew out even more information than normal")
-    parser.add_option("-d", "--distinguished-name", default=None,
-                      help="X.509 'Distiguished Name' used when generating keys")
-    parser.add_option("--keystore", default=None,
-                      help="Path to the keystore for the repo signing key")
-    parser.add_option("--repo-keyalias", default=None,
-                      help="Alias of the repo signing key in the keystore")
-    (options, args) = parser.parse_args()
+    parser = ArgumentParser()
+    common.setup_global_opts(parser)
+    parser.add_argument(
+        "-d",
+        "--distinguished-name",
+        default=None,
+        help=_("X.509 'Distinguished Name' used when generating keys"),
+    )
+    parser.add_argument(
+        "--keystore",
+        default=None,
+        help=_("Path to the keystore for the repo signing key"),
+    )
+    parser.add_argument(
+        "--repo-keyalias",
+        default=None,
+        help=_("Alias of the repo signing key in the keystore"),
+    )
+    parser.add_argument(
+        "--android-home",
+        default=None,
+        help=_("Path to the Android SDK (sometimes set in ANDROID_HOME)"),
+    )
+    parser.add_argument(
+        "--no-prompt",
+        action="store_true",
+        default=False,
+        help=_("Do not prompt for Android SDK path, just fail"),
+    )
+    options = common.parse_args(parser)
 
-    # find root install prefix
-    tmp = os.path.dirname(sys.argv[0])
-    if os.path.basename(tmp) == 'bin':
-        prefix = os.path.dirname(tmp)
-        examplesdir = prefix + '/share/doc/fdroidserver/examples'
-    else:
-        # we're running straight out of the git repo
-        prefix = tmp
-        examplesdir = prefix
+    common.set_console_logging(options.verbose, options.color)
 
     fdroiddir = os.getcwd()
+    test_config = dict()
+    examplesdir = common.get_examples_dir()
+    common.fill_config_defaults(test_config)
 
-    if not os.path.exists('config.py') and not os.path.exists('repo'):
-        # 'metadata' and 'tmp' are created in fdroid
-        os.mkdir('repo')
-        shutil.copy(os.path.join(examplesdir, 'fdroid-icon.png'), fdroiddir)
-        shutil.copyfile(os.path.join(examplesdir, 'config.sample.py'), 'config.py')
-        os.chmod('config.py', 0o0600)
-    else:
-        print('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...')
-        sys.exit()
-
-    # now that we have a local config.py, read configuration...
-    config = common.read_config(options)
-
-    # track down where the Android SDK is
-    if os.path.isdir(config['sdk_path']):
-        print('Using "' + config['sdk_path'] + '" for the Android SDK')
-        sdk_path = config['sdk_path']
-    elif 'ANDROID_HOME' in os.environ.keys():
-        sdk_path = os.environ['ANDROID_HOME']
-    else:
-        default_sdk_path = '/opt/android-sdk'
-        while True:
-            s = raw_input('Enter the path to the Android SDK (' + default_sdk_path + '): ')
-            if re.match('^\s*$', s) != None:
-                sdk_path = default_sdk_path
-            else:
-                sdk_path = s
-            if os.path.isdir(os.path.join(sdk_path, 'build-tools')):
-                break
-            else:
-                print('"' + s + '" does not contain the Android SDK! Try again...')
-    if os.path.isdir(sdk_path):
-        write_to_config('sdk_path', sdk_path)
-
-    # try to find a working aapt, in all the recent possible paths
-    build_tools = os.path.join(sdk_path, 'build-tools')
-    aaptdirs = []
-    aaptdirs.append(os.path.join(build_tools, config['build_tools']))
-    aaptdirs.append(build_tools)
-    for f in sorted(os.listdir(build_tools), reverse=True):
-        if os.path.isdir(os.path.join(build_tools, f)):
-            aaptdirs.append(os.path.join(build_tools, f))
-    for d in aaptdirs:
-        if os.path.isfile(os.path.join(d, 'aapt')):
-            aapt = os.path.join(d, 'aapt')
-            break
-    if os.path.isfile(aapt):
-        dirname = os.path.basename(os.path.dirname(aapt))
-        if dirname == 'build-tools':
-            # this is the old layout, before versioned build-tools
-            write_to_config('build_tools', '')
+    # track down where the Android SDK is, the default is to use the path set
+    # in ANDROID_HOME if that exists, otherwise None
+    if options.android_home is not None:
+        test_config['sdk_path'] = options.android_home
+    elif not common.test_sdk_exists(test_config):
+        # if neither --android-home nor the default sdk_path
+        # exist, prompt the user using platform-specific default
+        # and if the user leaves it blank, ignore and move on.
+        default_sdk_path = ''
+        if sys.platform in ('win32', 'cygwin'):
+            p = os.path.join(
+                os.getenv('USERPROFILE'), 'AppData', 'Local', 'Android', 'android-sdk'
+            )
+        elif sys.platform == 'darwin':
+            # on OSX, Homebrew is common and has an easy path to detect
+            p = '/usr/local/opt/android-sdk'
+        elif os.path.isdir('/usr/lib/android-sdk'):
+            # if the Debian packages are installed, suggest them
+            p = '/usr/lib/android-sdk'
         else:
-            write_to_config('build_tools', dirname)
+            p = '/opt/android-sdk'
+        if os.path.exists(p):
+            default_sdk_path = p
+            test_config['sdk_path'] = default_sdk_path
 
-    # track down where the Android NDK is
-    ndk_path = '/opt/android-ndk'
-    if os.path.isdir(config['ndk_path']):
-        ndk_path = config['ndk_path']
-    elif 'ANDROID_NDK' in os.environ.keys():
-        print('using ANDROID_NDK')
-        ndk_path = os.environ['ANDROID_NDK']
-    if os.path.isdir(ndk_path):
-        write_to_config('ndk_path', ndk_path)
-    # the NDK is optional so we don't prompt the user for it if its not found
+        if not common.test_sdk_exists(test_config):
+            del test_config['sdk_path']
+            while not options.no_prompt:
+                try:
+                    s = input(
+                        _('Enter the path to the Android SDK (%s) here:\n> ')
+                        % default_sdk_path
+                    )
+                except KeyboardInterrupt:
+                    print('')
+                    sys.exit(1)
+                if re.match(r'^\s*$', s) is not None:
+                    test_config['sdk_path'] = default_sdk_path
+                else:
+                    test_config['sdk_path'] = s
+                if common.test_sdk_exists(test_config):
+                    break
+                default_sdk_path = ''
+
+    if test_config.get('sdk_path') and not common.test_sdk_exists(test_config):
+        raise FDroidException(
+            _("Android SDK not found at {path}!").format(path=test_config['sdk_path'])
+        )
+
+    if not os.path.exists(common.CONFIG_FILE):
+        # 'metadata' and 'tmp' are created in fdroid
+        if not os.path.exists('repo'):
+            os.mkdir('repo')
+        example_config_yml = os.path.join(examplesdir, common.CONFIG_FILE)
+        if os.path.exists(example_config_yml):
+            shutil.copyfile(example_config_yml, common.CONFIG_FILE)
+        else:
+            from pkg_resources import get_distribution
+
+            versionstr = get_distribution('fdroidserver').version
+            if not versionstr:
+                versionstr = 'master'
+            with open(common.CONFIG_FILE, 'w') as fp:
+                fp.write('# see https://gitlab.com/fdroid/fdroidserver/blob/')
+                fp.write(versionstr)
+                fp.write(f'/examples/{common.CONFIG_FILE}\n')
+        os.chmod(common.CONFIG_FILE, 0o0600)
+        # If android_home is None, test_config['sdk_path'] will be used and
+        # "$ANDROID_HOME" may be used if the env var is set up correctly.
+        # If android_home is not None, the path given from the command line
+        # will be directly written in the config.
+        if 'sdk_path' in test_config:
+            common.write_to_config(test_config, 'sdk_path', options.android_home)
+    else:
+        logging.warning(
+            'Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...'
+        )
+        logging.info('Try running `fdroid init` in an empty directory.')
+        raise FDroidException('Repository already exists.')
+
+    # now that we have a local config.yml, read configuration...
+    config = common.read_config()
+
+    # the NDK is optional and there may be multiple versions of it, so it's
+    # left for the user to configure
 
     # find or generate the keystore for the repo signing key. First try the
-    # path written in the default config.py.  Then check if the user has
+    # path written in the default config.yml.  Then check if the user has
     # specified a path from the command line, which will trump all others.
-    # Otherwise, create ~/.local/share/fdroidserver and stick it in there.
+    # Otherwise, create ~/.local/share/fdroidserver and stick it in there.  If
+    # keystore is set to NONE, that means that Java will look for keys in a
+    # Hardware Security Module aka Smartcard.
     keystore = config['keystore']
     if options.keystore:
-        if os.path.isfile(options.keystore):
+        keystore = os.path.abspath(options.keystore)
+        if options.keystore == 'NONE':
             keystore = options.keystore
-            write_to_config('keystore', keystore)
         else:
-            print('"' + options.keystore + '" does not exist or is not a file!')
-            sys.exit(1)
+            keystore = os.path.abspath(options.keystore)
+            if not os.path.exists(keystore):
+                logging.info(
+                    '"' + keystore + '" does not exist, creating a new keystore there.'
+                )
+    common.write_to_config(test_config, 'keystore', keystore)
+    repo_keyalias = None
+    keydname = None
     if options.repo_keyalias:
         repo_keyalias = options.repo_keyalias
-        write_to_config('repo_keyalias', repo_keyalias)
+        common.write_to_config(test_config, 'repo_keyalias', repo_keyalias)
     if options.distinguished_name:
         keydname = options.distinguished_name
-        write_to_config('keydname', keydname)
-    if not os.path.isfile(keystore):
-        # no existing or specified keystore, generate the whole thing
-        keystoredir = os.path.join(os.getenv('HOME'),
-                                   '.local', 'share', 'fdroidserver')
-        if not os.path.exists(keystoredir):
-            os.makedirs(keystoredir, mode=0o700)
-        keystore = os.path.join(keystoredir, 'keystore.jks')
-        write_to_config('keystore', keystore)
-        password = genpassword()
-        write_to_config('keystorepass', password)
-        write_to_config('keypass', password)
-        if not options.repo_keyalias:
-            repo_keyalias = socket.getfqdn()
-            write_to_config('repo_keyalias', repo_keyalias)
-        if not options.distinguished_name:
-            keydname = 'CN=' + repo_keyalias + ', OU=F-Droid'
-            write_to_config('keydname', keydname)
-        genkey(keystore, repo_keyalias, password, keydname)
+        common.write_to_config(test_config, 'keydname', keydname)
+    if keystore == 'NONE':  # we're using a smartcard
+        common.write_to_config(
+            test_config, 'repo_keyalias', '1'
+        )  # seems to be the default
+        disable_in_config('keypass', 'never used with smartcard')
+        common.write_to_config(
+            test_config,
+            'smartcardoptions',
+            (
+                '-storetype PKCS11 '
+                + '-providerClass sun.security.pkcs11.SunPKCS11 '
+                + '-providerArg opensc-fdroid.cfg'
+            ),
+        )
+        # find opensc-pkcs11.so
+        if not os.path.exists('opensc-fdroid.cfg'):
+            if os.path.exists('/usr/lib/opensc-pkcs11.so'):
+                opensc_so = '/usr/lib/opensc-pkcs11.so'
+            elif os.path.exists('/usr/lib64/opensc-pkcs11.so'):
+                opensc_so = '/usr/lib64/opensc-pkcs11.so'
+            else:
+                files = glob.glob(
+                    '/usr/lib/' + os.uname()[4] + '-*-gnu/opensc-pkcs11.so'
+                )
+                if len(files) > 0:
+                    opensc_so = files[0]
+                else:
+                    opensc_so = '/usr/lib/opensc-pkcs11.so'
+                    logging.warning(
+                        'No OpenSC PKCS#11 module found, '
+                        + 'install OpenSC then edit "opensc-fdroid.cfg"!'
+                    )
+            with open('opensc-fdroid.cfg', 'w') as f:
+                f.write('name = OpenSC\nlibrary = ')
+                f.write(opensc_so)
+                f.write('\n')
+        logging.info(
+            "Repo setup using a smartcard HSM. Please edit keystorepass and repo_keyalias in config.yml."
+        )
+        logging.info(
+            "If you want to generate a new repo signing key in the HSM you can do that with 'fdroid update "
+            "--create-key'."
+        )
+    elif os.path.exists(keystore):
+        to_set = ['keystorepass', 'keypass', 'repo_keyalias', 'keydname']
+        if repo_keyalias:
+            to_set.remove('repo_keyalias')
+        if keydname:
+            to_set.remove('keydname')
+        logging.warning(
+            '\n'
+            + _('Using existing keystore "{path}"').format(path=keystore)
+            + '\n'
+            + _('Now set these in config.yml:')
+            + ' '
+            + ', '.join(to_set)
+            + '\n'
+        )
+    else:
+        password = common.genpassword()
+        c = dict(test_config)
+        c['keystorepass'] = password
+        c['keypass'] = password
+        c['repo_keyalias'] = repo_keyalias or socket.getfqdn()
+        c['keydname'] = 'CN=' + c['repo_keyalias'] + ', OU=F-Droid'
+        common.write_to_config(test_config, 'keystorepass', password)
+        common.write_to_config(test_config, 'keypass', password)
+        common.write_to_config(test_config, 'repo_keyalias', c['repo_keyalias'])
+        common.write_to_config(test_config, 'keydname', c['keydname'])
+        common.genkeystore(c)
 
-    print('Built repo based in "' + fdroiddir + '"')
-    print('with this config:')
-    print('  Android SDK:\t\t\t' + sdk_path)
-    print('  Android SDK Build Tools:\t' + os.path.dirname(aapt))
-    print('  Android NDK (optional):\t' + ndk_path)
-    print('  Keystore for signing key:\t' + keystore)
-    print('\nTo complete the setup, add your APKs to "' +
-          os.path.join(fdroiddir, 'repo') + '"' +
-'''
+    msg = '\n'
+    msg += _('Built repo based in "%s" with this config:') % fdroiddir
+    msg += '\n\n  Android SDK:\t\t\t' + config['sdk_path']
+    msg += '\n  ' + _('Keystore for signing key:\t') + keystore
+    if repo_keyalias is not None:
+        msg += '\n  Alias for key in store:\t' + repo_keyalias
+    msg += '\n\n'
+    msg += (
+        _(
+            """To complete the setup, add your APKs to "%s"
 then run "fdroid update -c; fdroid update".  You might also want to edit
-"config.py" to set the URL, repo name, and more.  You should also set up
-a signing key.
+"config.yml" to set the URL, repo name, and more.  You should also set up
+a signing key (a temporary one might have been automatically generated).
 
-For more info: https://f-droid.org/manual/fdroid.html#Simple-Binary-Repository
-and https://f-droid.org/manual/fdroid.html#Signing
-''')
+For more info: https://f-droid.org/docs/Setup_an_F-Droid_App_Repo
+and https://f-droid.org/docs/Signing_Process"""
+        )
+        % os.path.join(fdroiddir, 'repo')
+    )
+    if not options.quiet:
+        # normally, INFO is only shown with --verbose, but show this unless --quiet
+        logger = logging.getLogger()
+        logger.setLevel(logging.INFO)
+        logger.info(msg)
+    logging.shutdown()

fdroidserver/install.py

@@ -1,9 +1,8 @@
-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 #
-# verify.py - part of the FDroid server tools
+# install.py - part of the FDroid server tools
 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
-# Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
+# Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,98 +17,396 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import sys
-import os
 import glob
-from optparse import OptionParser, OptionError
+import locale
+import logging
+import os
+import sys
+import termios
+import tty
+from argparse import ArgumentParser, BooleanOptionalAction
+from pathlib import Path
+from urllib.parse import urlencode, urlparse, urlunparse
 
-import common
-from common import FDroidPopen
+import defusedxml.ElementTree as XMLElementTree
+
+from . import _, common, github, index, net
+from .exception import FDroidException
+
+DEFAULT_IPFS_GATEWAYS = ("https://gateway.ipfs.io/ipfs/",)
+MAVEN_CENTRAL_MIRRORS = [
+    {
+        "url": "https://repo1.maven.org/maven2/",
+        "dnsA": ["199.232.16.209"],
+        "worksWithoutSNI": True,
+    },
+    {
+        "url": "https://repo.maven.apache.org/maven2/",
+        "dnsA": ["199.232.16.215"],
+        "worksWithoutSNI": True,
+    },
+    {
+        "url": "https://maven-central-asia.storage-download.googleapis.com/maven2/",
+    },
+    {
+        "url": "https://maven-central-eu.storage-download.googleapis.com/maven2/",
+    },
+    {
+        "url": "https://maven-central.storage-download.googleapis.com/maven2/",
+    },
+]
+
+
+# pylint: disable=unused-argument
+def download_apk(appid='org.fdroid.fdroid', privacy_mode=False):
+    """Download an APK from F-Droid via the first mirror that works."""
+    url = urlunparse(
+        urlparse(common.FDROIDORG_MIRRORS[0]['url'])._replace(
+            query=urlencode({'fingerprint': common.FDROIDORG_FINGERPRINT})
+        )
+    )
+
+    data, _ignored = index.download_repo_index_v2(url)
+    app = data.get('packages', dict()).get(appid)
+    preferred_version = None
+    for version in app['versions'].values():
+        if not preferred_version:
+            # if all else fails, use the first one
+            preferred_version = version
+        if not version.get('releaseChannels'):
+            # prefer APK in default release channel
+            preferred_version = version
+            break
+
+    mirrors = common.append_filename_to_mirrors(
+        preferred_version['file']['name'][1:], common.FDROIDORG_MIRRORS
+    )
+    ipfsCIDv1 = preferred_version['file'].get('ipfsCIDv1')
+    if ipfsCIDv1:
+        for gateway in DEFAULT_IPFS_GATEWAYS:
+            mirrors.append({'url': os.path.join(gateway, ipfsCIDv1)})
+    f = net.download_using_mirrors(mirrors)
+    if f and os.path.exists(f):
+        versionCode = preferred_version['manifest']['versionCode']
+        f = Path(f)
+        return str(f.rename(f.with_stem(f'{appid}_{versionCode}')).resolve())
+
+
+def download_fdroid_apk(privacy_mode=False):  # pylint: disable=unused-argument
+    """Directly download the current F-Droid APK and verify it.
+
+    This downloads the "download button" link, which is the version
+    that is best tested for new installs.
+
+    """
+    mirror = common.FDROIDORG_MIRRORS[0]
+    mirror['url'] = urlunparse(urlparse(mirror['url'])._replace(path='F-Droid.apk'))
+    return net.download_using_mirrors([mirror])
+
+
+def download_fdroid_apk_from_github(privacy_mode=False):
+    """Download F-Droid.apk from F-Droid's GitHub Releases."""
+    if common.config and not privacy_mode:
+        token = common.config.get('github_token')
+    else:
+        token = None
+    gh = github.GithubApi(token, 'https://github.com/f-droid/fdroidclient')
+    latest_apk = gh.get_latest_apk()
+    filename = os.path.basename(latest_apk)
+    return net.download_file(latest_apk, os.path.join(common.get_cachedir(), filename))
+
+
+def download_fdroid_apk_from_ipns(privacy_mode=False):
+    """Download the F-Droid APK from an IPNS repo."""
+    cid = 'k51qzi5uqu5dl4hbcksbdmplanu9n4hivnqsupqe6vzve1pdbeh418ssptldd3'
+    mirrors = [
+        {"url": f"https://ipfs.io/ipns/{cid}/F-Droid.apk"},
+    ]
+    if not privacy_mode:
+        mirrors.append({"url": f"https://{cid}.ipns.dweb.link/F-Droid.apk"})
+    return net.download_using_mirrors(mirrors)
+
+
+def download_fdroid_apk_from_maven(privacy_mode=False):
+    """Download F-Droid.apk from Maven Central and official mirrors."""
+    path = 'org/fdroid/fdroid/F-Droid'
+    if privacy_mode:
+        mirrors = MAVEN_CENTRAL_MIRRORS[:2]  # skip the Google servers
+    else:
+        mirrors = MAVEN_CENTRAL_MIRRORS
+    metadata = net.download_using_mirrors(
+        common.append_filename_to_mirrors(
+            os.path.join(path, 'maven-metadata.xml'), mirrors
+        )
+    )
+    version = XMLElementTree.parse(metadata).getroot().findall('*.//latest')[0].text
+    mirrors = common.append_filename_to_mirrors(
+        os.path.join(path, version, f'F-Droid-{version}.apk'), mirrors
+    )
+    return net.download_using_mirrors(mirrors)
+
+
+def install_fdroid_apk(privacy_mode=False):
+    """Download and install F-Droid.apk using all tricks we can muster.
+
+    By default, this first tries to fetch the official install APK
+    which is offered when someone clicks the "download" button on
+    https://f-droid.org/.  Then it will try all the mirrors and
+    methods until it gets something successful, or runs out of
+    options.
+
+    There is privacy_mode which tries to download from mirrors first,
+    so that this downloads from a mirror that has many different kinds
+    of files available, thereby breaking the clear link to F-Droid.
+
+    Returns
+    -------
+    None for success or the error message.
+
+    """
+    country_code = locale.getlocale()[0].split('_')[-1]
+    if privacy_mode is None and country_code in ('CN', 'HK', 'IR', 'TM'):
+        logging.warning(
+            _('Privacy mode was enabled based on your locale ({country_code}).').format(
+                country_code=country_code
+            )
+        )
+        privacy_mode = True
+
+    if privacy_mode or not (common.config and common.config.get('jarsigner')):
+        download_methods = [
+            download_fdroid_apk_from_maven,
+            download_fdroid_apk_from_ipns,
+            download_fdroid_apk_from_github,
+        ]
+    else:
+        download_methods = [
+            download_apk,
+            download_fdroid_apk_from_maven,
+            download_fdroid_apk_from_github,
+            download_fdroid_apk_from_ipns,
+            download_fdroid_apk,
+        ]
+    for method in download_methods:
+        try:
+            f = method(privacy_mode=privacy_mode)
+            break
+        except Exception as e:
+            logging.info(e)
+    else:
+        return _('F-Droid.apk could not be downloaded from any known source!')
+
+    fingerprint = common.apk_signer_fingerprint(f)
+    if fingerprint.upper() != common.FDROIDORG_FINGERPRINT:
+        return _('{path} has the wrong fingerprint ({fingerprint})!').format(
+            path=f, fingerprint=fingerprint
+        )
+    install_apk(f)
+
+
+def install_apk(f):
+    if common.config and common.config.get('apksigner'):
+        # TODO this should always verify, but that requires APK sig verification in Python #94
+        logging.info(_('Verifying package {path} with apksigner.').format(path=f))
+        common.verify_apk_signature(f)
+    if common.config and common.config.get('adb'):
+        if devices():
+            install_apks_to_devices([f])
+            os.remove(f)
+        else:
+            os.remove(f)
+            return _('No devices found for `adb install`! Please plug one in.')
 
-options = None
-config = None
 
 def devices():
-    p = FDroidPopen(["adb", "devices"])
+    """Get the list of device serials for use with adb commands."""
+    p = common.SdkToolsPopen(['adb', "devices"])
     if p.returncode != 0:
-        raise Exception("An error occured when finding devices: %s" % p.stderr)
-    lines = p.stdout.splitlines()
-    if lines[0].startswith('* daemon not running'):
-        lines = lines[2:]
-    if len(lines) < 3:
-        return []
-    lines = lines[1:-1]
-    return [l.split()[0] for l in lines]
+        raise FDroidException("An error occured when finding devices: %s" % p.output)
+    serials = list()
+    for line in p.output.splitlines():
+        columns = line.strip().split("\t", maxsplit=1)
+        if len(columns) == 2:
+            serial, status = columns
+            if status == 'device':
+                serials.append(serial)
+            else:
+                d = {'serial': serial, 'status': status}
+                logging.warning(_('adb reports {serial} is "{status}"!'.format(**d)))
+    return serials
+
+
+def install_apks_to_devices(apks):
+    """Install the list of APKs to all Android devices reported by `adb devices`."""
+    for apk in apks:
+        # Get device list each time to avoid device not found errors
+        devs = devices()
+        if not devs:
+            raise FDroidException(_("No attached devices found"))
+        logging.info(_("Installing %s...") % apk)
+        for dev in devs:
+            logging.info(
+                _("Installing '{apkfilename}' on {dev}...").format(
+                    apkfilename=apk, dev=dev
+                )
+            )
+            p = common.SdkToolsPopen(['adb', "-s", dev, "install", apk])
+            fail = ""
+            for line in p.output.splitlines():
+                if line.startswith("Failure"):
+                    fail = line[9:-1]
+            if not fail:
+                continue
+
+            if fail == "INSTALL_FAILED_ALREADY_EXISTS":
+                logging.warning(
+                    _('"{apkfilename}" is already installed on {dev}.').format(
+                        apkfilename=apk, dev=dev
+                    )
+                )
+            else:
+                raise FDroidException(
+                    _("Failed to install '{apkfilename}' on {dev}: {error}").format(
+                        apkfilename=apk, dev=dev, error=fail
+                    )
+                )
+
+
+def read_char():
+    """Read input from the terminal prompt one char at a time."""
+    fd = sys.stdin.fileno()
+    old_settings = termios.tcgetattr(fd)
+    try:
+        tty.setraw(fd)
+        ch = sys.stdin.read(1)
+    finally:
+        termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+    return ch
+
+
+def strtobool(val):
+    """Convert a localized string representation of truth to True or False."""
+    return val.lower() in ('', 'y', 'yes', _('yes'), _('true'))  # '' is pressing Enter
+
+
+def prompt_user(yes, msg):
+    """Prompt user for yes/no, supporting Enter and Esc as accepted answers."""
+    run_install = yes
+    if yes is None and sys.stdout.isatty():
+        print(msg, end=' ', flush=True)
+        answer = ''
+        while True:
+            in_char = read_char()
+            if in_char == '\r':  # Enter key
+                break
+            if not in_char.isprintable():
+                sys.exit(1)
+            print(in_char, end='', flush=True)
+            answer += in_char
+        run_install = strtobool(answer)
+        print()
+    return run_install
 
 
 def main():
+    parser = ArgumentParser(
+        usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]"
+    )
+    common.setup_global_opts(parser)
+    parser.add_argument(
+        "appid",
+        nargs='*',
+        help=_("application ID with optional versionCode in the form APPID[:VERCODE]"),
+    )
+    parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        default=False,
+        help=_("Install all signed applications available"),
+    )
+    parser.add_argument(
+        "-p",
+        "--privacy-mode",
+        action=BooleanOptionalAction,
+        default=None,
+        help=_("Download F-Droid.apk using mirrors that leak less to the network"),
+    )
+    parser.add_argument(
+        "-y",
+        "--yes",
+        action="store_true",
+        default=None,
+        help=_("Automatic yes to all prompts."),
+    )
+    parser.add_argument(
+        "-n",
+        "--no",
+        action="store_false",
+        dest='yes',
+        help=_("Automatic no to all prompts."),
+    )
+    options = common.parse_args(parser)
 
-    global options, config
+    common.set_console_logging(options.verbose, options.color)
+    logging.captureWarnings(True)  # for SNIMissingWarning
 
-    # Parse command line...
-    parser = OptionParser(usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
-    parser.add_option("-v", "--verbose", action="store_true", default=False,
-                      help="Spew out even more information than normal")
-    parser.add_option("-a", "--all", action="store_true", default=False,
-                      help="Install all signed applications available")
-    (options, args) = parser.parse_args()
+    common.get_config()
 
-    if not args and not options.all:
-        raise OptionError("If you really want to install all the signed apps, use --all", "all")
-
-    config = common.read_config(options)
+    if not options.appid and not options.all:
+        run_install = prompt_user(
+            options.yes,
+            _('Would you like to download and install F-Droid.apk via adb? (YES/no)'),
+        )
+        if run_install:
+            sys.exit(install_fdroid_apk(options.privacy_mode))
+        sys.exit(1)
 
     output_dir = 'repo'
-    if not os.path.isdir(output_dir):
-        print "No signed output directory - nothing to do"
-        sys.exit(0)
+    if (options.appid or options.all) and not os.path.isdir(output_dir):
+        logging.error(_("No signed output directory - nothing to do"))
+        run_install = prompt_user(
+            options.yes,
+            _('Would you like to download the app(s) from f-droid.org? (YES/no)'),
+        )
+        if run_install:
+            for appid in options.appid:
+                f = download_apk(appid)
+                install_apk(f)
+            sys.exit(install_fdroid_apk(options.privacy_mode))
+        sys.exit(1)
 
-    if args:
+    if options.appid:
+        vercodes = common.read_pkg_args(options.appid, True)
+        common.get_metadata_files(vercodes)  # only check appids
+        apks = {appid: None for appid in vercodes}
 
-        vercodes = common.read_pkg_args(args, True)
-        apks = { appid : None for appid in vercodes }
-
-        # Get the signed apk with the highest vercode
+        # Get the signed APK with the highest vercode
         for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))):
-
-            appid, vercode = common.apknameinfo(apkfile)
+            try:
+                appid, vercode = common.publishednameinfo(apkfile)
+            except FDroidException:
+                continue
             if appid not in apks:
                 continue
             if vercodes[appid] and vercode not in vercodes[appid]:
                 continue
             apks[appid] = apkfile
 
-        for appid, apk in apks.iteritems():
+        for appid, apk in apks.items():
             if not apk:
-                raise Exception("No signed apk available for %s" % appid)
+                raise FDroidException(_("No signed APK available for %s") % appid)
+        install_apks_to_devices(apks.values())
 
-    else:
+    elif options.all:
+        apks = {
+            common.publishednameinfo(apkfile)[0]: apkfile
+            for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk')))
+        }
+        install_apks_to_devices(apks.values())
 
-        apks = { common.apknameinfo(apkfile)[0] : apkfile for apkfile in
-                sorted(glob.glob(os.path.join(output_dir, '*.apk'))) }
+    logging.info('\n' + _('Finished'))
 
-    for appid, apk in apks.iteritems():
-        # Get device list each time to avoid device not found errors
-        devs = devices()
-        if not devs:
-            raise Exception("No attached devices found")
-        print "Installing %s..." % apk
-        for dev in devs:
-            print "Installing %s on %s..." % (apk, dev)
-            p = FDroidPopen(["adb", "-s", dev, "install", apk ])
-            fail= ""
-            for line in p.stdout.splitlines():
-                if line.startswith("Failure"):
-                    fail = line[9:-1]
-            if fail:
-                if fail == "INSTALL_FAILED_ALREADY_EXISTS":
-                    print "%s is already installed on %s." % (apk, dev)
-                else:
-                    raise Exception("Failed to install %s on %s: %s" % (
-                        apk, dev, fail))
-
-    print "\nFinished"
 
 if __name__ == "__main__":
     main()
-

fdroidserver/looseversion.py

@@ -0,0 +1,300 @@
+# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+# --------------------------------------------
+#
+# 1. This LICENSE AGREEMENT is between the Python Software Foundation
+# ("PSF"), and the Individual or Organization ("Licensee") accessing and
+# otherwise using this software ("Python") in source or binary form and
+# its associated documentation.
+#
+# 2. Subject to the terms and conditions of this License Agreement, PSF hereby
+# grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+# analyze, test, perform and/or display publicly, prepare derivative works,
+# distribute, and otherwise use Python alone or in any derivative version,
+# provided, however, that PSF's License Agreement and PSF's notice of copyright,
+# i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+# 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 Python Software Foundation;
+# All Rights Reserved" are retained in Python alone or in any derivative version
+# prepared by Licensee.
+#
+# 3. In the event Licensee prepares a derivative work that is based on
+# or incorporates Python or any part thereof, and wants to make
+# the derivative work available to others as provided herein, then
+# Licensee hereby agrees to include in any such work a brief summary of
+# the changes made to Python.
+#
+# 4. PSF is making Python available to Licensee on an "AS IS"
+# basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+# IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+# INFRINGE ANY THIRD PARTY RIGHTS.
+#
+# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+# FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+#
+# 6. This License Agreement will automatically terminate upon a material
+# breach of its terms and conditions.
+#
+# 7. Nothing in this License Agreement shall be deemed to create any
+# relationship of agency, partnership, or joint venture between PSF and
+# Licensee.  This License Agreement does not grant permission to use PSF
+# trademarks or trade name in a trademark sense to endorse or promote
+# products or services of Licensee, or any third party.
+#
+# 8. By copying, installing or otherwise using Python, Licensee
+# agrees to be bound by the terms and conditions of this License
+# Agreement.
+#
+# SPDX-License-Identifier: Python-2.0
+#
+# downloaded from:
+# https://github.com/effigies/looseversion/blob/e1a5a176a92dc6825deda4205c1be6d05e9ed352/src/looseversion/__init__.py
+
+"""Provides classes to represent module version numbers (one class for
+each style of version numbering).  There are currently two such classes
+implemented: StrictVersion and LooseVersion.
+
+Every version number class implements the following interface:
+  * the 'parse' method takes a string and parses it to some internal
+    representation; if the string is an invalid version number,
+    'parse' raises a ValueError exception
+  * the class constructor takes an optional string argument which,
+    if supplied, is passed to 'parse'
+  * __str__ reconstructs the string that was passed to 'parse' (or
+    an equivalent string -- ie. one that will generate an equivalent
+    version number instance)
+  * __repr__ generates Python code to recreate the version number instance
+  * _cmp compares the current instance with either another instance
+    of the same class or a string (which will be parsed to an instance
+    of the same class, thus must follow the same rules)
+"""
+import re
+import sys
+
+__license__ = "Python License 2.0"
+
+# The rules according to Greg Stein:
+# 1) a version number has 1 or more numbers separated by a period or by
+#    sequences of letters. If only periods, then these are compared
+#    left-to-right to determine an ordering.
+# 2) sequences of letters are part of the tuple for comparison and are
+#    compared lexicographically
+# 3) recognize the numeric components may have leading zeroes
+#
+# The LooseVersion class below implements these rules: a version number
+# string is split up into a tuple of integer and string components, and
+# comparison is a simple tuple comparison.  This means that version
+# numbers behave in a predictable and obvious way, but a way that might
+# not necessarily be how people *want* version numbers to behave.  There
+# wouldn't be a problem if people could stick to purely numeric version
+# numbers: just split on period and compare the numbers as tuples.
+# However, people insist on putting letters into their version numbers;
+# the most common purpose seems to be:
+#   - indicating a "pre-release" version
+#     ('alpha', 'beta', 'a', 'b', 'pre', 'p')
+#   - indicating a post-release patch ('p', 'pl', 'patch')
+# but of course this can't cover all version number schemes, and there's
+# no way to know what a programmer means without asking him.
+#
+# The problem is what to do with letters (and other non-numeric
+# characters) in a version number.  The current implementation does the
+# obvious and predictable thing: keep them as strings and compare
+# lexically within a tuple comparison.  This has the desired effect if
+# an appended letter sequence implies something "post-release":
+# eg. "0.99" < "0.99pl14" < "1.0", and "5.001" < "5.001m" < "5.002".
+#
+# However, if letters in a version number imply a pre-release version,
+# the "obvious" thing isn't correct.  Eg. you would expect that
+# "1.5.1" < "1.5.2a2" < "1.5.2", but under the tuple/lexical comparison
+# implemented here, this just isn't so.
+#
+# Two possible solutions come to mind.  The first is to tie the
+# comparison algorithm to a particular set of semantic rules, as has
+# been done in the StrictVersion class above.  This works great as long
+# as everyone can go along with bondage and discipline.  Hopefully a
+# (large) subset of Python module programmers will agree that the
+# particular flavor of bondage and discipline provided by StrictVersion
+# provides enough benefit to be worth using, and will submit their
+# version numbering scheme to its domination.  The free-thinking
+# anarchists in the lot will never give in, though, and something needs
+# to be done to accommodate them.
+#
+# Perhaps a "moderately strict" version class could be implemented that
+# lets almost anything slide (syntactically), and makes some heuristic
+# assumptions about non-digits in version number strings.  This could
+# sink into special-case-hell, though; if I was as talented and
+# idiosyncratic as Larry Wall, I'd go ahead and implement a class that
+# somehow knows that "1.2.1" < "1.2.2a2" < "1.2.2" < "1.2.2pl3", and is
+# just as happy dealing with things like "2g6" and "1.13++".  I don't
+# think I'm smart enough to do it right though.
+#
+# In any case, I've coded the test suite for this module (see
+# ../test/test_version.py) specifically to fail on things like comparing
+# "1.2a2" and "1.2".  That's not because the *code* is doing anything
+# wrong, it's because the simple, obvious design doesn't match my
+# complicated, hairy expectations for real-world version numbers.  It
+# would be a snap to fix the test suite to say, "Yep, LooseVersion does
+# the Right Thing" (ie. the code matches the conception).  But I'd rather
+# have a conception that matches common notions about version numbers.
+
+
+if sys.version_info >= (3,):
+
+    class _Py2Int(int):
+        """Integer object that compares < any string"""
+
+        def __gt__(self, other):
+            if isinstance(other, str):
+                return False
+            return super().__gt__(other)
+
+        def __lt__(self, other):
+            if isinstance(other, str):
+                return True
+            return super().__lt__(other)
+
+else:
+    _Py2Int = int
+
+
+class LooseVersion(object):
+    """Version numbering for anarchists and software realists.
+    Implements the standard interface for version number classes as
+    described above.  A version number consists of a series of numbers,
+    separated by either periods or strings of letters.  When comparing
+    version numbers, the numeric components will be compared
+    numerically, and the alphabetic components lexically.  The following
+    are all valid version numbers, in no particular order:
+
+        1.5.1
+        1.5.2b2
+        161
+        3.10a
+        8.02
+        3.4j
+        1996.07.12
+        3.2.pl0
+        3.1.1.6
+        2g6
+        11g
+        0.960923
+        2.2beta29
+        1.13++
+        5.5.kw
+        2.0b1pl0
+
+    In fact, there is no such thing as an invalid version number under
+    this scheme; the rules for comparison are simple and predictable,
+    but may not always give the results you want (for some definition
+    of "want").
+    """
+
+    component_re = re.compile(r"(\d+ | [a-z]+ | \.)", re.VERBOSE)
+
+    def __init__(self, vstring=None):
+        if vstring:
+            self.parse(vstring)
+
+    def __eq__(self, other):
+        c = self._cmp(other)
+        if c is NotImplemented:
+            return NotImplemented
+        return c == 0
+
+    def __lt__(self, other):
+        c = self._cmp(other)
+        if c is NotImplemented:
+            return NotImplemented
+        return c < 0
+
+    def __le__(self, other):
+        c = self._cmp(other)
+        if c is NotImplemented:
+            return NotImplemented
+        return c <= 0
+
+    def __gt__(self, other):
+        c = self._cmp(other)
+        if c is NotImplemented:
+            return NotImplemented
+        return c > 0
+
+    def __ge__(self, other):
+        c = self._cmp(other)
+        if c is NotImplemented:
+            return NotImplemented
+        return c >= 0
+
+    def parse(self, vstring):
+        # I've given up on thinking I can reconstruct the version string
+        # from the parsed tuple -- so I just store the string here for
+        # use by __str__
+        self.vstring = vstring
+        components = [x for x in self.component_re.split(vstring) if x and x != "."]
+        for i, obj in enumerate(components):
+            try:
+                components[i] = int(obj)
+            except ValueError:
+                pass
+
+        self.version = components
+
+    def __str__(self):
+        return self.vstring
+
+    def __repr__(self):
+        return "LooseVersion ('%s')" % str(self)
+
+    def _cmp(self, other):
+        other = self._coerce(other)
+        if other is NotImplemented:
+            return NotImplemented
+
+        if self.version == other.version:
+            return 0
+        if self.version < other.version:
+            return -1
+        if self.version > other.version:
+            return 1
+        return NotImplemented
+
+    @classmethod
+    def _coerce(cls, other):
+        if isinstance(other, cls):
+            return other
+        elif isinstance(other, str):
+            return cls(other)
+        elif "distutils" in sys.modules:
+            # Using this check to avoid importing distutils and suppressing the warning
+            try:
+                from distutils.version import LooseVersion as deprecated
+            except ImportError:
+                return NotImplemented
+            if isinstance(other, deprecated):
+                return cls(str(other))
+        return NotImplemented
+
+
+class LooseVersion2(LooseVersion):
+    """LooseVersion variant that restores Python 2 semantics
+
+    In Python 2, comparing LooseVersions where paired components could be string
+    and int always resulted in the string being "greater". In Python 3, this produced
+    a TypeError.
+    """
+
+    def parse(self, vstring):
+        # I've given up on thinking I can reconstruct the version string
+        # from the parsed tuple -- so I just store the string here for
+        # use by __str__
+        self.vstring = vstring
+        components = [x for x in self.component_re.split(vstring) if x and x != "."]
+        for i, obj in enumerate(components):
+            try:
+                components[i] = _Py2Int(obj)
+            except ValueError:
+                pass
+
+        self.version = components

fdroidserver/mirror.py

@@ -0,0 +1,278 @@
+#!/usr/bin/env python3
+
+import ipaddress
+import logging
+import os
+import posixpath
+import socket
+import subprocess
+import sys
+import urllib.parse
+from argparse import ArgumentParser
+
+from . import _, common, index, update
+
+
+def _run_wget(path, urls, verbose=False):
+    if verbose:
+        verbose = '--verbose'
+    else:
+        verbose = '--no-verbose'
+
+    if not urls:
+        return
+    logging.debug(_('Running wget in {path}').format(path=path))
+    cwd = os.getcwd()
+    os.makedirs(path, exist_ok=True)
+    os.chdir(path)
+    urls_file = '.fdroid-mirror-wget-input-file'
+    with open(urls_file, 'w') as fp:
+        for url in urls:
+            fp.write(url.split('?')[0] + '\n')  # wget puts query string in the filename
+    subprocess.call(
+        [
+            'wget',
+            verbose,
+            '--continue',
+            '--user-agent="fdroid mirror"',
+            '--input-file=' + urls_file,
+        ]
+    )
+    os.remove(urls_file)
+    os.chdir(cwd)  # leave the working env the way we found it
+
+
+def main():
+    parser = ArgumentParser()
+    common.setup_global_opts(parser)
+    parser.add_argument(
+        "url",
+        nargs='?',
+        help=_(
+            'Base URL to mirror, can include the index signing key '
+            + 'using the query string: ?fingerprint='
+        ),
+    )
+    parser.add_argument(
+        "--all",
+        action='store_true',
+        default=False,
+        help=_("Mirror the full repo and archive, all file types."),
+    )
+    parser.add_argument(
+        "--archive",
+        action='store_true',
+        default=False,
+        help=_("Also mirror the full archive section"),
+    )
+    parser.add_argument(
+        "--build-logs",
+        action='store_true',
+        default=False,
+        help=_("Include the build logs in the mirror"),
+    )
+    parser.add_argument(
+        "--pgp-signatures",
+        action='store_true',
+        default=False,
+        help=_("Include the PGP signature .asc files in the mirror"),
+    )
+    parser.add_argument(
+        "--src-tarballs",
+        action='store_true',
+        default=False,
+        help=_("Include the source tarballs in the mirror"),
+    )
+    parser.add_argument(
+        "--output-dir", default=None, help=_("The directory to write the mirror to")
+    )
+    options = common.parse_args(parser)
+
+    common.set_console_logging(options.verbose, options.color)
+
+    if options.all:
+        options.archive = True
+        options.build_logs = True
+        options.pgp_signatures = True
+        options.src_tarballs = True
+
+    if options.url is None:
+        logging.error(_('A URL is required as an argument!') + '\n')
+        parser.print_help()
+        sys.exit(1)
+
+    scheme, hostname, path, params, query, fragment = urllib.parse.urlparse(options.url)
+    fingerprint = urllib.parse.parse_qs(query).get('fingerprint')
+
+    def _append_to_url_path(*args):
+        """Append the list of path components to URL, keeping the rest the same."""
+        newpath = posixpath.join(path, *args)
+        return urllib.parse.urlunparse(
+            (scheme, hostname, newpath, params, query, fragment)
+        )
+
+    if fingerprint:
+        config = common.read_config()
+        if not ('jarsigner' in config or 'apksigner' in config):
+            logging.error(
+                _('Java JDK not found! Install in standard location or set java_paths!')
+            )
+            sys.exit(1)
+
+        def _get_index(section, etag=None):
+            url = _append_to_url_path(section)
+            data, etag = index.download_repo_index(url, etag=etag)
+            return data, etag, _append_to_url_path(section, 'index-v1.jar')
+
+    else:
+
+        def _get_index(section, etag=None):
+            import io
+            import json
+            import zipfile
+
+            from . import net
+
+            url = _append_to_url_path(section, 'index-v1.jar')
+            content, etag = net.http_get(url)
+            with zipfile.ZipFile(io.BytesIO(content)) as zip:
+                jsoncontents = zip.open('index-v1.json').read()
+            data = json.loads(jsoncontents.decode('utf-8'))
+            return data, etag, None  # no verified index file to return
+
+    ip = None
+    try:
+        ip = ipaddress.ip_address(hostname)
+    except ValueError:
+        pass
+    if hostname == 'f-droid.org' or (
+        ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]
+    ):
+        logging.error(
+            _(
+                'This command should never be used to mirror f-droid.org! '
+                'A full copy requires more than 600GB.'
+            )
+        )
+        sys.exit(1)
+
+    path = path.rstrip('/')
+    if path.endswith('repo') or path.endswith('archive'):
+        logging.warning(
+            _('Do not include "{path}" in URL!').format(path=path.split('/')[-1])
+        )
+    elif not path.endswith('fdroid'):
+        logging.warning(
+            _('{url} does not end with "fdroid", check the URL path!').format(
+                url=options.url
+            )
+        )
+
+    icondirs = ['icons']
+    for density in update.screen_densities:
+        icondirs.append('icons-' + density)
+
+    if options.output_dir:
+        basedir = options.output_dir
+    else:
+        basedir = os.path.join(os.getcwd(), hostname, path.strip('/'))
+        os.makedirs(basedir, exist_ok=True)
+
+    if options.archive:
+        sections = ('repo', 'archive')
+    else:
+        sections = ('repo',)
+
+    for section in sections:
+        sectiondir = os.path.join(basedir, section)
+
+        urls = []
+        data, etag, index_url = _get_index(section)
+        if index_url:
+            urls.append(index_url)
+
+        os.makedirs(sectiondir, exist_ok=True)
+        os.chdir(sectiondir)
+        for icondir in icondirs:
+            os.makedirs(os.path.join(sectiondir, icondir), exist_ok=True)
+
+        for packageName, packageList in data['packages'].items():
+            for package in packageList:
+                to_fetch = []
+                keys = ['apkName']
+                if options.src_tarballs:
+                    keys.append('srcname')
+                for k in keys:
+                    if k in package:
+                        to_fetch.append(package[k])
+                    elif k == 'apkName':
+                        logging.error(
+                            _('{appid} is missing {name}').format(
+                                appid=package['packageName'], name=k
+                            )
+                        )
+                for f in to_fetch:
+                    if not os.path.exists(f) or (
+                        f.endswith('.apk') and os.path.getsize(f) != package['size']
+                    ):
+                        urls.append(_append_to_url_path(section, f))
+                        if options.pgp_signatures:
+                            urls.append(_append_to_url_path(section, f + '.asc'))
+                        if options.build_logs and f.endswith('.apk'):
+                            urls.append(
+                                _append_to_url_path(section, f[:-4] + '.log.gz')
+                            )
+
+        _run_wget(sectiondir, urls, options.verbose)
+
+        for app in data['apps']:
+            localized = app.get('localized')
+            if localized:
+                for locale, d in localized.items():
+                    urls = []
+                    components = (section, app['packageName'], locale)
+                    for k in update.GRAPHIC_NAMES:
+                        f = d.get(k)
+                        if f:
+                            filepath_tuple = components + (f,)
+                            urls.append(_append_to_url_path(*filepath_tuple))
+                    _run_wget(os.path.join(basedir, *components), urls, options.verbose)
+                    for k in update.SCREENSHOT_DIRS:
+                        urls = []
+                        filelist = d.get(k)
+                        if filelist:
+                            components = (section, app['packageName'], locale, k)
+                            for f in filelist:
+                                filepath_tuple = components + (f,)
+                                urls.append(_append_to_url_path(*filepath_tuple))
+                            _run_wget(
+                                os.path.join(basedir, *components),
+                                urls,
+                                options.verbose,
+                            )
+
+        urls = dict()
+        for app in data['apps']:
+            if 'icon' not in app:
+                logging.error(
+                    _('no "icon" in {appid}').format(appid=app['packageName'])
+                )
+                continue
+            icon = app['icon']
+            for icondir in icondirs:
+                url = _append_to_url_path(section, icondir, icon)
+                if icondir not in urls:
+                    urls[icondir] = []
+                urls[icondir].append(url)
+
+        for icondir in icondirs:
+            if icondir in urls:
+                _run_wget(
+                    os.path.join(basedir, section, icondir),
+                    urls[icondir],
+                    options.verbose,
+                )
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/net.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+#
+# net.py - part of the FDroid server tools
+# Copyright (C) 2015 Hans-Christoph Steiner <hans@eds.org>
+# Copyright (C) 2022 FC Stegerman <flx@obfusk.net>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import copy
+import logging
+import os
+import random
+import tempfile
+import time
+import urllib
+
+import requests
+import urllib3
+from requests.adapters import HTTPAdapter, Retry
+
+from . import _, common
+
+logger = logging.getLogger(__name__)
+
+HEADERS = {'User-Agent': 'F-Droid'}
+
+
+def download_file(url, local_filename=None, dldir='tmp', retries=3, backoff_factor=0.1):
+    """Try hard to download the file, including retrying on failures.
+
+    This has two retry cycles, one inside of the requests session, the
+    other provided by this function.  The requests retry logic applies
+    to failed DNS lookups, socket connections and connection timeouts,
+    never to requests where data has made it to the server. This
+    handles ChunkedEncodingError during transfer in its own retry
+    loop.  This can result in more retries than are specified in the
+    retries parameter.
+
+    """
+    filename = urllib.parse.urlparse(url).path.split('/')[-1]
+    if local_filename is None:
+        local_filename = os.path.join(dldir, filename)
+    for i in range(retries + 1):
+        if retries:
+            max_retries = Retry(total=retries - i, backoff_factor=backoff_factor)
+            adapter = HTTPAdapter(max_retries=max_retries)
+            session = requests.Session()
+            session.mount('http://', adapter)
+            session.mount('https://', adapter)
+        else:
+            session = requests
+        # the stream=True parameter keeps memory usage low
+        r = session.get(
+            url, stream=True, allow_redirects=True, headers=HEADERS, timeout=300
+        )
+        r.raise_for_status()
+        try:
+            with open(local_filename, 'wb') as f:
+                for chunk in r.iter_content(chunk_size=1024):
+                    if chunk:  # filter out keep-alive new chunks
+                        f.write(chunk)
+                        f.flush()
+            return local_filename
+        except requests.exceptions.ChunkedEncodingError as err:
+            if i == retries:
+                raise err
+            logger.warning('Download interrupted, retrying...')
+            time.sleep(backoff_factor * 2**i)
+    raise ValueError("retries must be >= 0")
+
+
+def download_using_mirrors(mirrors, local_filename=None):
+    """Try to download the file from any working mirror.
+
+    Download the file that all URLs in the mirrors list point to,
+    trying all the tricks, starting with the most private methods
+    first.  The list of mirrors is converted into a list of mirror
+    configurations to try, in order that the should be attempted.
+
+    This builds mirror_configs_to_try using all possible combos to
+    try.  If a mirror is marked with worksWithoutSNI: True, then this
+    logic will try it twice: first without SNI, then again with SNI.
+
+    """
+    mirrors = common.parse_list_of_dicts(mirrors)
+    mirror_configs_to_try = []
+    for mirror in mirrors:
+        mirror_configs_to_try.append(mirror)
+        if mirror.get('worksWithoutSNI'):
+            m = copy.deepcopy(mirror)
+            del m['worksWithoutSNI']
+            mirror_configs_to_try.append(m)
+
+    if not local_filename:
+        for mirror in mirrors:
+            filename = urllib.parse.urlparse(mirror['url']).path.split('/')[-1]
+            if filename:
+                break
+        if filename:
+            local_filename = os.path.join(common.get_cachedir(), filename)
+        else:
+            local_filename = tempfile.mkstemp(prefix='fdroid-')
+
+    timeouts = (2, 10, 100)
+    last_exception = None
+    for timeout in timeouts:
+        for mirror in mirror_configs_to_try:
+            last_exception = None
+            urllib3.util.ssl_.HAS_SNI = not mirror.get('worksWithoutSNI')
+            try:
+                # the stream=True parameter keeps memory usage low
+                r = requests.get(
+                    mirror['url'],
+                    stream=True,
+                    allow_redirects=False,
+                    headers=HEADERS,
+                    # add jitter to the timeout to be less predictable
+                    timeout=timeout + random.randint(0, timeout),  # nosec B311
+                )
+                if r.status_code != 200:
+                    raise requests.exceptions.HTTPError(r.status_code, response=r)
+                with open(local_filename, 'wb') as f:
+                    for chunk in r.iter_content(chunk_size=1024):
+                        if chunk:  # filter out keep-alive new chunks
+                            f.write(chunk)
+                            f.flush()
+                return local_filename
+            except (
+                ConnectionError,
+                requests.exceptions.ChunkedEncodingError,
+                requests.exceptions.ConnectionError,
+                requests.exceptions.ContentDecodingError,
+                requests.exceptions.HTTPError,
+                requests.exceptions.SSLError,
+                requests.exceptions.StreamConsumedError,
+                requests.exceptions.Timeout,
+                requests.exceptions.UnrewindableBodyError,
+            ) as e:
+                last_exception = e
+                logger.debug(_('Retrying failed download: %s') % str(e))
+    # if it hasn't succeeded by now, then give up and raise last exception
+    if last_exception:
+        raise last_exception
+
+
+def http_get(url, etag=None, timeout=600):
+    """Download the content from the given URL by making a GET request.
+
+    If an ETag is given, it will do a HEAD request first, to see if the content changed.
+
+    Parameters
+    ----------
+    url
+      The URL to download from.
+    etag
+      The last ETag to be used for the request (optional).
+
+    Returns
+    -------
+    A tuple consisting of:
+      - The raw content that was downloaded or None if it did not change
+      - The new eTag as returned by the HTTP request
+    """
+    # TODO disable TLS Session IDs and TLS Session Tickets
+    #      (plain text cookie visible to anyone who can see the network traffic)
+    if etag:
+        r = requests.head(url, headers=HEADERS, timeout=timeout)
+        r.raise_for_status()
+        if 'ETag' in r.headers and etag == r.headers['ETag']:
+            return None, etag
+
+    r = requests.get(url, headers=HEADERS, timeout=timeout)
+    r.raise_for_status()
+
+    new_etag = None
+    if 'ETag' in r.headers:
+        new_etag = r.headers['ETag']
+
+    return r.content, new_etag

fdroidserver/nightly.py

@@ -0,0 +1,612 @@
+#!/usr/bin/env python3
+"""Set up an app build for a nightly build repo."""
+#
+# nightly.py - part of the FDroid server tools
+# Copyright (C) 2017 Hans-Christoph Steiner <hans@eds.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import base64
+import datetime
+import hashlib
+import inspect
+import logging
+import os
+import platform
+import shutil
+import ssl
+import subprocess
+import sys
+import tempfile
+from argparse import ArgumentParser
+from typing import Optional
+from urllib.parse import urlparse
+
+import git
+import paramiko
+import yaml
+
+from . import _, common
+from .exception import VCSException
+
+# hard coded defaults for Android ~/.android/debug.keystore files
+# https://developers.google.com/android/guides/client-auth
+KEYSTORE_FILE = os.path.join(os.getenv('HOME'), '.android', 'debug.keystore')
+PASSWORD = 'android'  # nosec B105 standard hardcoded password for debug keystores
+KEY_ALIAS = 'androiddebugkey'
+DISTINGUISHED_NAME = 'CN=Android Debug,O=Android,C=US'
+
+# standard suffix for naming fdroid git repos
+NIGHTLY = '-nightly'
+
+
+def _get_keystore_secret_var(keystore: str) -> str:
+    """Get keystore secret as base64.
+
+    Parameters
+    ----------
+    keystore
+        The path of the keystore.
+
+    Returns
+    -------
+    base64_secret
+        The keystore secret as base64 string.
+    """
+    with open(keystore, 'rb') as fp:
+        return base64.standard_b64encode(fp.read()).decode('ascii')
+
+
+def _ssh_key_from_debug_keystore(keystore: Optional[str] = None) -> str:
+    """Convert a debug keystore to an SSH private key.
+
+    This leaves the original keystore file in place.
+
+    Parameters
+    ----------
+    keystore
+        The keystore to convert to a SSH private key.
+
+    Returns
+    -------
+    key_path
+        The SSH private key file path in the temporary directory.
+    """
+    if keystore is None:
+        # set this here so it can be overridden in the tests
+        # TODO convert this to a class to get rid of this nonsense
+        keystore = KEYSTORE_FILE
+    tmp_dir = tempfile.mkdtemp(prefix='.')
+    privkey = os.path.join(tmp_dir, '.privkey')
+    key_pem = os.path.join(tmp_dir, '.key.pem')
+    p12 = os.path.join(tmp_dir, '.keystore.p12')
+    _config = dict()
+    common.fill_config_defaults(_config)
+    subprocess.check_call(
+        [
+            _config['keytool'],
+            '-importkeystore',
+            '-srckeystore',
+            keystore,
+            '-srcalias',
+            KEY_ALIAS,
+            '-srcstorepass',
+            PASSWORD,
+            '-srckeypass',
+            PASSWORD,
+            '-destkeystore',
+            p12,
+            '-destalias',
+            KEY_ALIAS,
+            '-deststorepass',
+            PASSWORD,
+            '-destkeypass',
+            PASSWORD,
+            '-deststoretype',
+            'PKCS12',
+        ],
+        env={'LC_ALL': 'C.UTF-8'},
+    )
+    subprocess.check_call(
+        [
+            'openssl',
+            'pkcs12',
+            '-in',
+            p12,
+            '-out',
+            key_pem,
+            '-passin',
+            'pass:' + PASSWORD,
+            '-passout',
+            'pass:' + PASSWORD,
+        ],
+        env={'LC_ALL': 'C.UTF-8'},
+    )
+
+    # OpenSSL 3.0 changed the default output format from PKCS#1 to
+    # PKCS#8, which paramiko does not support.
+    # https://www.openssl.org/docs/man3.0/man1/openssl-rsa.html#traditional
+    # https://github.com/paramiko/paramiko/issues/1015
+    openssl_rsa_cmd = ['openssl', 'rsa']
+    if ssl.OPENSSL_VERSION_INFO[0] >= 3:
+        openssl_rsa_cmd += ['-traditional']
+    subprocess.check_call(
+        openssl_rsa_cmd
+        + [
+            '-in',
+            key_pem,
+            '-out',
+            privkey,
+            '-passin',
+            'pass:' + PASSWORD,
+        ],
+        env={'LC_ALL': 'C.UTF-8'},
+    )
+    os.remove(key_pem)
+    os.remove(p12)
+    os.chmod(privkey, 0o600)  # os.umask() should cover this, but just in case
+
+    rsakey = paramiko.RSAKey.from_private_key_file(privkey)
+    fingerprint = (
+        base64.b64encode(hashlib.sha256(rsakey.asbytes()).digest())
+        .decode('ascii')
+        .rstrip('=')
+    )
+    ssh_private_key_file = os.path.join(
+        tmp_dir, 'debug_keystore_' + fingerprint.replace('/', '_') + '_id_rsa'
+    )
+    shutil.move(privkey, ssh_private_key_file)
+
+    pub = rsakey.get_name() + ' ' + rsakey.get_base64() + ' ' + ssh_private_key_file
+    with open(ssh_private_key_file + '.pub', 'w') as fp:
+        fp.write(pub)
+
+    logging.info(_('\nSSH public key to be used as deploy key:') + '\n' + pub)
+
+    return ssh_private_key_file
+
+
+def get_repo_base_url(
+    clone_url: str, repo_git_base: str, force_type: Optional[str] = None
+) -> str:
+    """Generate the base URL for the F-Droid repository.
+
+    Parameters
+    ----------
+    clone_url
+        The URL to clone the Git repository.
+    repo_git_base
+        The project path of the Git repository at the Git forge.
+    force_type
+        The Git forge of the project.
+
+    Returns
+    -------
+    repo_base_url
+        The base URL of the F-Droid repository.
+    """
+    if force_type is None:
+        force_type = urlparse(clone_url).netloc
+    if force_type == 'gitlab.com':
+        return clone_url + '/-/raw/master/fdroid'
+    if force_type == 'github.com':
+        return 'https://raw.githubusercontent.com/%s/master/fdroid' % repo_git_base
+    print(_('ERROR: unsupported git host "%s", patches welcome!') % force_type)
+    sys.exit(1)
+
+
+def clone_git_repo(clone_url, git_mirror_path):
+    """Clone a git repo into the given path, failing if a password is required.
+
+    If GitPython's safe mode is present, this will use that.  Otherwise,
+    this includes a very limited version of the safe mode just to ensure
+    this won't hang on password prompts.
+
+    https://github.com/gitpython-developers/GitPython/pull/2029
+
+    """
+    logging.debug(_('cloning {url}').format(url=clone_url))
+    try:
+        sig = inspect.signature(git.Repo.clone_from)
+        if 'safe' in sig.parameters:
+            git.Repo.clone_from(clone_url, git_mirror_path, safe=True)
+        else:
+            git.Repo.clone_from(
+                clone_url,
+                git_mirror_path,
+                env={
+                    'GIT_ASKPASS': '/bin/true',
+                    'SSH_ASKPASS': '/bin/true',
+                    'GIT_USERNAME': 'u',
+                    'GIT_PASSWORD': 'p',
+                    'GIT_HTTP_USERNAME': 'u',
+                    'GIT_HTTP_PASSWORD': 'p',
+                    'GIT_SSH': '/bin/false',  # for git < 2.3
+                    'GIT_TERMINAL_PROMPT': '0',
+                },
+            )
+    except git.exc.GitCommandError as e:
+        logging.warning(_('WARNING: only public git repos are supported!'))
+        raise VCSException(f'git clone {clone_url} failed:', str(e)) from e
+
+
+def main():
+    """Deploy to F-Droid repository or generate SSH private key from keystore.
+
+    The behaviour of this function is influenced by the configuration file as
+    well as command line parameters.
+
+    Raises
+    ------
+    :exc:`~fdroidserver.exception.VCSException`
+        If the nightly Git repository could not be cloned during an attempt to
+        deploy.
+    """
+    parser = ArgumentParser()
+    common.setup_global_opts(parser)
+    parser.add_argument(
+        "--keystore",
+        default=KEYSTORE_FILE,
+        help=_("Specify which debug keystore file to use."),
+    )
+    parser.add_argument(
+        "--show-secret-var",
+        action="store_true",
+        default=False,
+        help=_("Print the secret variable to the terminal for easy copy/paste"),
+    )
+    parser.add_argument(
+        "--keep-private-keys",
+        action="store_true",
+        default=False,
+        help=_("Do not remove the private keys generated from the keystore"),
+    )
+    parser.add_argument(
+        "--no-deploy",
+        action="store_true",
+        default=False,
+        help=_("Do not deploy the new files to the repo"),
+    )
+    parser.add_argument(
+        "--file",
+        default='app/build/outputs/apk/*.apk',
+        help=_('The file to be included in the repo (path or glob)'),
+    )
+    parser.add_argument(
+        "--no-checksum",
+        action="store_true",
+        default=False,
+        help=_("Don't use rsync checksums"),
+    )
+    archive_older_unset = -1
+    parser.add_argument(
+        "--archive-older",
+        type=int,
+        default=archive_older_unset,
+        help=_("Set maximum releases in repo before older ones are archived"),
+    )
+    # TODO add --with-btlog
+    options = common.parse_args(parser)
+
+    # force a tighter umask since this writes private key material
+    umask = os.umask(0o077)
+
+    if 'CI' in os.environ:
+        v = os.getenv('DEBUG_KEYSTORE')
+        debug_keystore = None
+        if v:
+            debug_keystore = base64.b64decode(v)
+        if not debug_keystore:
+            logging.error(_('DEBUG_KEYSTORE is not set or the value is incomplete'))
+            sys.exit(1)
+        os.makedirs(os.path.dirname(KEYSTORE_FILE), exist_ok=True)
+        if os.path.exists(KEYSTORE_FILE):
+            logging.warning(_('overwriting existing {path}').format(path=KEYSTORE_FILE))
+        with open(KEYSTORE_FILE, 'wb') as fp:
+            fp.write(debug_keystore)
+
+        repo_basedir = os.path.join(os.getcwd(), 'fdroid')
+        repodir = os.path.join(repo_basedir, 'repo')
+        cibase = os.getcwd()
+        os.makedirs(repodir, exist_ok=True)
+
+        # the 'master' branch is hardcoded in fdroidserver/deploy.py
+        if 'CI_PROJECT_PATH' in os.environ and 'CI_PROJECT_URL' in os.environ:
+            # we are in GitLab CI
+            repo_git_base = os.getenv('CI_PROJECT_PATH') + NIGHTLY
+            clone_url = os.getenv('CI_PROJECT_URL') + NIGHTLY
+            repo_base = get_repo_base_url(
+                clone_url, repo_git_base, force_type='gitlab.com'
+            )
+            servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base
+            deploy_key_url = (
+                f'{clone_url}/-/settings/repository#js-deploy-keys-settings'
+            )
+            git_user_name = os.getenv('GITLAB_USER_NAME')
+            git_user_email = os.getenv('GITLAB_USER_EMAIL')
+        elif 'TRAVIS_REPO_SLUG' in os.environ:
+            # we are in Travis CI
+            repo_git_base = os.getenv('TRAVIS_REPO_SLUG') + NIGHTLY
+            clone_url = 'https://github.com/' + repo_git_base
+            repo_base = get_repo_base_url(
+                clone_url, repo_git_base, force_type='github.com'
+            )
+            servergitmirror = 'git@github.com:' + repo_git_base
+            deploy_key_url = (
+                f'https://github.com/{repo_git_base}/settings/keys'
+                + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
+            )
+            git_user_name = repo_git_base
+            git_user_email = os.getenv('USER') + '@' + platform.node()
+        elif (
+            'CIRCLE_REPOSITORY_URL' in os.environ
+            and 'CIRCLE_PROJECT_USERNAME' in os.environ
+            and 'CIRCLE_PROJECT_REPONAME' in os.environ
+        ):
+            # we are in Circle CI
+            repo_git_base = (
+                os.getenv('CIRCLE_PROJECT_USERNAME')
+                + '/'
+                + os.getenv('CIRCLE_PROJECT_REPONAME')
+                + NIGHTLY
+            )
+            clone_url = os.getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY
+            repo_base = get_repo_base_url(
+                clone_url, repo_git_base, force_type='github.com'
+            )
+            servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base
+            deploy_key_url = (
+                f'https://github.com/{repo_git_base}/settings/keys'
+                + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
+            )
+            git_user_name = os.getenv('CIRCLE_USERNAME')
+            git_user_email = git_user_name + '@' + platform.node()
+        elif 'GITHUB_ACTIONS' in os.environ:
+            # we are in Github actions
+            repo_git_base = os.getenv('GITHUB_REPOSITORY') + NIGHTLY
+            clone_url = os.getenv('GITHUB_SERVER_URL') + '/' + repo_git_base
+            repo_base = get_repo_base_url(
+                clone_url, repo_git_base, force_type='github.com'
+            )
+            servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base
+            deploy_key_url = (
+                f'https://github.com/{repo_git_base}/settings/keys'
+                + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
+            )
+            git_user_name = os.getenv('GITHUB_ACTOR')
+            git_user_email = git_user_name + '@' + platform.node()
+        else:
+            print(_('ERROR: unsupported CI type, patches welcome!'))
+            sys.exit(1)
+
+        repo_url = repo_base + '/repo'
+        git_mirror_path = os.path.join(repo_basedir, 'git-mirror')
+        git_mirror_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
+        git_mirror_repodir = os.path.join(git_mirror_fdroiddir, 'repo')
+        git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata')
+        if not os.path.isdir(git_mirror_repodir):
+            clone_git_repo(clone_url, git_mirror_path)
+        if not os.path.isdir(git_mirror_repodir):
+            os.makedirs(git_mirror_repodir, mode=0o755)
+        if os.path.exists('LICENSE'):
+            shutil.copy2('LICENSE', git_mirror_path)
+
+        mirror_git_repo = git.Repo.init(git_mirror_path)
+        writer = mirror_git_repo.config_writer()
+        writer.set_value('user', 'name', git_user_name)
+        writer.set_value('user', 'email', git_user_email)
+        writer.release()
+        for remote in mirror_git_repo.remotes:
+            mirror_git_repo.delete_remote(remote)
+
+        readme_path = os.path.join(git_mirror_path, 'README.md')
+        readme = '''
+# {repo_git_base}
+
+This is an app repository for nightly versions.
+You can use it with the [F-Droid](https://f-droid.org/) Android app.
+
+[![{repo_url}]({repo_url}/icons/icon.png)](https://fdroid.link/#{repo_url})
+
+Last updated: {date}'''.format(
+            repo_git_base=repo_git_base,
+            repo_url=repo_url,
+            date=datetime.datetime.now(datetime.timezone.utc).strftime(
+                '%Y-%m-%d %H:%M:%S UTC'
+            ),
+        )
+        with open(readme_path, 'w') as fp:
+            fp.write(readme)
+        mirror_git_repo.git.add(all=True)
+        mirror_git_repo.index.commit("update README")
+
+        mirror_git_repo.git.add(all=True)
+        mirror_git_repo.index.commit("update repo/website icon")
+
+        os.chdir(repo_basedir)
+        if os.path.isdir(git_mirror_repodir):
+            common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/')
+        if os.path.isdir(git_mirror_metadatadir):
+            common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/')
+
+        ssh_private_key_file = _ssh_key_from_debug_keystore()
+        # this is needed for GitPython to find the SSH key
+        ssh_dir = os.path.join(os.getenv('HOME'), '.ssh')
+        os.makedirs(ssh_dir, exist_ok=True)
+        ssh_config = os.path.join(ssh_dir, 'config')
+        logging.debug(_('adding IdentityFile to {path}').format(path=ssh_config))
+        with open(ssh_config, 'a') as fp:
+            fp.write('\n\nHost *\n\tIdentityFile %s\n' % ssh_private_key_file)
+
+        if options.archive_older == archive_older_unset:
+            fdroid_size = common.get_dir_size(git_mirror_fdroiddir)
+            max_size = common.GITLAB_COM_PAGES_MAX_SIZE
+            if fdroid_size < max_size:
+                options.archive_older = 20
+            else:
+                options.archive_older = 3
+                print(
+                    'WARNING: repo is %s over the GitLab Pages limit (%s)'
+                    % (fdroid_size - max_size, max_size)
+                )
+                print('Setting --archive-older to 3')
+
+        config = {
+            'identity_file': ssh_private_key_file,
+            'repo_name': repo_git_base,
+            'repo_url': repo_url,
+            'repo_description': 'Nightly builds from %s' % git_user_email,
+            'archive_name': repo_git_base + ' archive',
+            'archive_url': repo_base + '/archive',
+            'archive_description': 'Old nightly builds that have been archived.',
+            'archive_older': options.archive_older,
+            'servergitmirrors': [{"url": servergitmirror}],
+            'keystore': KEYSTORE_FILE,
+            'repo_keyalias': KEY_ALIAS,
+            'keystorepass': PASSWORD,
+            'keypass': PASSWORD,
+            'keydname': DISTINGUISHED_NAME,
+            'make_current_version_link': False,
+        }
+        with open(common.CONFIG_FILE, 'w', encoding='utf-8') as fp:
+            yaml.dump(config, fp, default_flow_style=False)
+        os.chmod(common.CONFIG_FILE, 0o600)
+        config = common.read_config()
+        common.assert_config_keystore(config)
+
+        logging.debug(
+            _(
+                'Run over {cibase} to find -debug.apk. and skip repo_basedir {repo_basedir}'
+            ).format(cibase=cibase, repo_basedir=repo_basedir)
+        )
+
+        for root, dirs, files in os.walk(cibase):
+            for d in ('.git', '.gradle'):
+                if d in dirs:
+                    dirs.remove(d)
+            if root == cibase and 'fdroid' in dirs:
+                dirs.remove('fdroid')
+
+            for f in files:
+                if f.endswith('-debug.apk'):
+                    apkfilename = os.path.join(root, f)
+                    logging.debug(
+                        _('Stripping mystery signature from {apkfilename}').format(
+                            apkfilename=apkfilename
+                        )
+                    )
+                    destapk = os.path.join(repodir, os.path.basename(f))
+                    os.chmod(apkfilename, 0o644)
+                    logging.debug(
+                        _(
+                            'Resigning {apkfilename} with provided debug.keystore'
+                        ).format(apkfilename=os.path.basename(apkfilename))
+                    )
+                    common.sign_apk(apkfilename, destapk, KEY_ALIAS)
+
+        if options.verbose:
+            logging.debug(_('attempting bare SSH connection to test deploy key:'))
+            try:
+                subprocess.check_call(
+                    [
+                        'ssh',
+                        '-Tvi',
+                        ssh_private_key_file,
+                        '-oIdentitiesOnly=yes',
+                        '-oStrictHostKeyChecking=no',
+                        servergitmirror.split(':')[0],
+                    ]
+                )
+            except subprocess.CalledProcessError:
+                pass
+
+        app_url = clone_url[: -len(NIGHTLY)]
+        template = dict()
+        template['AuthorName'] = clone_url.split('/')[4]
+        template['AuthorWebSite'] = '/'.join(clone_url.split('/')[:4])
+        template['Categories'] = ['nightly']
+        template['SourceCode'] = app_url
+        template['IssueTracker'] = app_url + '/issues'
+        template['Summary'] = 'Nightly build of ' + urlparse(app_url).path[1:]
+        template['Description'] = template['Summary']
+        with open('template.yml', 'w') as fp:
+            yaml.dump(template, fp)
+
+        subprocess.check_call(
+            ['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'],
+            cwd=repo_basedir,
+        )
+        common.local_rsync(
+            options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
+        )
+        mirror_git_repo.git.add(all=True)
+        mirror_git_repo.index.commit("update app metadata")
+
+        if not options.no_deploy:
+            try:
+                cmd = ['fdroid', 'deploy', '--verbose', '--no-keep-git-mirror-archive']
+                subprocess.check_call(cmd, cwd=repo_basedir)
+            except subprocess.CalledProcessError:
+                logging.error(
+                    _('cannot publish update, did you set the deploy key?')
+                    + '\n'
+                    + deploy_key_url
+                )
+                sys.exit(1)
+
+        if not options.keep_private_keys:
+            os.remove(KEYSTORE_FILE)
+            if shutil.rmtree.avoids_symlink_attacks:
+                shutil.rmtree(os.path.dirname(ssh_private_key_file))
+
+    else:
+        if not os.path.isfile(options.keystore):
+            androiddir = os.path.dirname(options.keystore)
+            if not os.path.exists(androiddir):
+                os.mkdir(androiddir)
+                logging.info(_('created {path}').format(path=androiddir))
+            logging.error(
+                _('{path} does not exist!  Create it by running:').format(
+                    path=options.keystore
+                )
+                + '\n    keytool -genkey -v -keystore '
+                + options.keystore
+                + ' -storepass android \\'
+                + '\n     -alias androiddebugkey -keypass android -keyalg RSA -keysize 2048 -validity 10000 \\'
+                + '\n     -dname "CN=Android Debug,O=Android,C=US"'
+            )
+            sys.exit(1)
+        ssh_dir = os.path.join(os.getenv('HOME'), '.ssh')
+        privkey = _ssh_key_from_debug_keystore(options.keystore)
+        if os.path.exists(ssh_dir):
+            ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey))
+            shutil.move(privkey, ssh_private_key_file)
+            shutil.move(privkey + '.pub', ssh_private_key_file + '.pub')
+        if shutil.rmtree.avoids_symlink_attacks:
+            shutil.rmtree(os.path.dirname(privkey))
+
+        if options.show_secret_var:
+            debug_keystore = _get_keystore_secret_var(options.keystore)
+            print(
+                _('\n{path} encoded for the DEBUG_KEYSTORE secret variable:').format(
+                    path=options.keystore
+                )
+            )
+            print(debug_keystore)
+
+    os.umask(umask)
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/publish.py

@@ -1,9 +1,9 @@
-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 #
 # publish.py - part of the FDroid server tools
 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
-# Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
+# Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
+# Copyright (C) 2021 Felix C. Stegerman <flx@obfusk.net>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,155 +18,461 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import sys
-import os
-import shutil
-import subprocess
-import md5
-import glob
-from optparse import OptionParser
+"""Sign APKs using keys or via reproducible builds signature copying.
 
-import common, metadata
-from common import BuildException
+This command takes unsigned APKs and signs them.  It looks for
+unsigned APKs in the unsigned/ directory and puts successfully signed
+APKs into the repo/ directory.  The default is to run in a kind of
+batch mode, where it will only quit on certain kinds of errors. It
+mostly reports success by moving an APK from unsigned/ to repo/
+
+"""
+
+import glob
+import hashlib
+import json
+import logging
+import os
+import re
+import shutil
+import sys
+import time
+import zipfile
+from argparse import ArgumentParser
+from collections import OrderedDict
+from gettext import ngettext
+
+from . import _, common, metadata
+from .common import FDroidPopen
+from .exception import BuildException, FDroidException
 
 config = None
-options = None
+start_timestamp = time.gmtime()
+
+
+def publish_source_tarball(apkfilename, unsigned_dir, output_dir):
+    """Move the source tarball into the output directory..."""
+    tarfilename = apkfilename[:-4] + '_src.tar.gz'
+    tarfile = os.path.join(unsigned_dir, tarfilename)
+    if os.path.exists(tarfile):
+        shutil.move(tarfile, os.path.join(output_dir, tarfilename))
+        logging.debug('...published %s', tarfilename)
+    else:
+        logging.debug('...no source tarball for %s', apkfilename)
+
+
+def key_alias(appid):
+    """No summary.
+
+    Get the alias which F-Droid uses to indentify the singing key
+    for this App in F-Droids keystore.
+    """
+    if config and 'keyaliases' in config and appid in config['keyaliases']:
+        # For this particular app, the key alias is overridden...
+        keyalias = config['keyaliases'][appid]
+        if keyalias.startswith('@'):
+            m = hashlib.md5()  # nosec just used to generate a keyalias
+            m.update(keyalias[1:].encode('utf-8'))
+            keyalias = m.hexdigest()[:8]
+        return keyalias
+    else:
+        m = hashlib.md5()  # nosec just used to generate a keyalias
+        m.update(appid.encode('utf-8'))
+        return m.hexdigest()[:8]
+
+
+def read_fingerprints_from_keystore():
+    """Obtain a dictionary containing all singning-key fingerprints which are managed by F-Droid, grouped by appid."""
+    env_vars = {'LC_ALL': 'C.UTF-8', 'FDROID_KEY_STORE_PASS': config['keystorepass']}
+    cmd = [
+        config['keytool'],
+        '-list',
+        '-v',
+        '-keystore',
+        config['keystore'],
+        '-storepass:env',
+        'FDROID_KEY_STORE_PASS',
+    ]
+    if config['keystore'] == 'NONE':
+        cmd += config['smartcardoptions']
+    p = FDroidPopen(cmd, envs=env_vars, output=False)
+    if p.returncode != 0:
+        raise FDroidException('could not read keystore {}'.format(config['keystore']))
+
+    realias = re.compile('Alias name: (?P<alias>.+)' + os.linesep)
+    resha256 = re.compile(r'\s+SHA256: (?P<sha256>[:0-9A-F]{95})' + os.linesep)
+    fps = {}
+    for block in p.output.split(('*' * 43) + os.linesep + '*' * 43):
+        s_alias = realias.search(block)
+        s_sha256 = resha256.search(block)
+        if s_alias and s_sha256:
+            sigfp = s_sha256.group('sha256').replace(':', '').lower()
+            fps[s_alias.group('alias')] = sigfp
+    return fps
+
+
+def sign_sig_key_fingerprint_list(jar_file):
+    """Sign the list of app-signing key fingerprints.
+
+    This is used primaryily by fdroid update to determine which APKs
+    where built and signed by F-Droid and which ones were
+    manually added by users.
+    """
+    cmd = [config['jarsigner']]
+    cmd += '-keystore', config['keystore']
+    cmd += '-storepass:env', 'FDROID_KEY_STORE_PASS'
+    cmd += '-digestalg', 'SHA1'
+    cmd += '-sigalg', 'SHA1withRSA'
+    cmd += jar_file, config['repo_keyalias']
+    if config['keystore'] == 'NONE':
+        cmd += config['smartcardoptions']
+    else:  # smardcards never use -keypass
+        cmd += '-keypass:env', 'FDROID_KEY_PASS'
+    env_vars = {
+        'FDROID_KEY_STORE_PASS': config['keystorepass'],
+        'FDROID_KEY_PASS': config.get('keypass', ""),
+    }
+    p = common.FDroidPopen(cmd, envs=env_vars)
+    if p.returncode != 0:
+        raise FDroidException("Failed to sign '{}'!".format(jar_file))
+
+
+def store_publish_signer_fingerprints(appids, indent=None):
+    """Store list of all signing-key fingerprints for given appids to HD.
+
+    This list will later on be needed by fdroid update.
+    """
+    if not os.path.exists('repo'):
+        os.makedirs('repo')
+    data = OrderedDict()
+    fps = read_fingerprints_from_keystore()
+    for appid in sorted(appids):
+        alias = key_alias(appid)
+        if alias in fps:
+            data[appid] = {'signer': fps[key_alias(appid)]}
+
+    jar_file = os.path.join('repo', 'signer-index.jar')
+    output = json.dumps(data, indent=indent)
+    with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar:
+        jar.writestr('signer-index.json', output)
+    with open(os.path.join('repo', 'signer-index.json'), 'w') as fp:
+        fp.write(output)
+    sign_sig_key_fingerprint_list(jar_file)
+
+
+def status_update_json(generatedKeys, signedApks):
+    """Output a JSON file with metadata about this run."""
+    logging.debug(_('Outputting JSON'))
+    output = common.setup_status_output(start_timestamp)
+    output['apksigner'] = shutil.which(config.get('apksigner', ''))
+    output['jarsigner'] = shutil.which(config.get('jarsigner', ''))
+    output['keytool'] = shutil.which(config.get('keytool', ''))
+    if generatedKeys:
+        output['generatedKeys'] = generatedKeys
+    if signedApks:
+        output['signedApks'] = signedApks
+    common.write_status_json(output)
+
+
+def check_for_key_collisions(allapps):
+    """Make sure there's no collision in keyaliases from apps.
+
+    It was suggested at
+    https://dev.guardianproject.info/projects/bazaar/wiki/FDroid_Audit
+    that a package could be crafted, such that it would use the same signing
+    key as an existing app. While it may be theoretically possible for such a
+    colliding package ID to be generated, it seems virtually impossible that
+    the colliding ID would be something that would be a) a valid package ID,
+    and b) a sane-looking ID that would make its way into the repo.
+    Nonetheless, to be sure, before publishing we check that there are no
+    collisions, and refuse to do any publishing if that's the case.
+
+    Parameters
+    ----------
+    allapps
+      a dict of all apps to process
+
+    Returns
+    -------
+    a list of all aliases corresponding to allapps
+    """
+    allaliases = []
+    for appid in allapps:
+        m = hashlib.md5()  # nosec just used to generate a keyalias
+        m.update(appid.encode('utf-8'))
+        keyalias = m.hexdigest()[:8]
+        if keyalias in allaliases:
+            logging.error(_("There is a keyalias collision - publishing halted"))
+            sys.exit(1)
+        allaliases.append(keyalias)
+    return allaliases
+
+
+def create_key_if_not_existing(keyalias):
+    """Ensure a signing key with the given keyalias exists.
+
+    Returns
+    -------
+    boolean
+      True if a new key was created, False otherwise
+    """
+    # See if we already have a key for this application, and
+    # if not generate one...
+    env_vars = {
+        'LC_ALL': 'C.UTF-8',
+        'FDROID_KEY_STORE_PASS': config['keystorepass'],
+        'FDROID_KEY_PASS': config.get('keypass', ""),
+    }
+    cmd = [
+        config['keytool'],
+        '-list',
+        '-alias',
+        keyalias,
+        '-keystore',
+        config['keystore'],
+        '-storepass:env',
+        'FDROID_KEY_STORE_PASS',
+    ]
+    if config['keystore'] == 'NONE':
+        cmd += config['smartcardoptions']
+    p = FDroidPopen(cmd, envs=env_vars)
+    if p.returncode != 0:
+        logging.info("Key does not exist - generating...")
+        cmd = [
+            config['keytool'],
+            '-genkey',
+            '-keystore',
+            config['keystore'],
+            '-alias',
+            keyalias,
+            '-keyalg',
+            'RSA',
+            '-keysize',
+            '2048',
+            '-validity',
+            '10000',
+            '-storepass:env',
+            'FDROID_KEY_STORE_PASS',
+            '-dname',
+            config['keydname'],
+        ]
+        if config['keystore'] == 'NONE':
+            cmd += config['smartcardoptions']
+        else:
+            cmd += '-keypass:env', 'FDROID_KEY_PASS'
+        p = FDroidPopen(cmd, envs=env_vars)
+        if p.returncode != 0:
+            raise BuildException("Failed to generate key", p.output)
+        return True
+    else:
+        return False
+
 
 def main():
-
-    global config, options
+    global config
 
     # Parse command line...
-    parser = OptionParser(usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
-    parser.add_option("-v", "--verbose", action="store_true", default=False,
-                      help="Spew out even more information than normal")
-    (options, args) = parser.parse_args()
+    parser = ArgumentParser(
+        usage="%(prog)s [options] " "[APPID[:VERCODE] [APPID[:VERCODE] ...]]"
+    )
+    common.setup_global_opts(parser)
+    parser.add_argument(
+        "-e",
+        "--error-on-failed",
+        action="store_true",
+        default=False,
+        help=_("When signing or verifying fails, exit with an error code."),
+    )
+    parser.add_argument(
+        "appid",
+        nargs='*',
+        help=_("application ID with optional versionCode in the form APPID[:VERCODE]"),
+    )
+    metadata.add_metadata_arguments(parser)
+    options = common.parse_args(parser)
+    metadata.warnings_action = options.W
 
-    config = common.read_config(options)
+    config = common.read_config()
+
+    if not ('jarsigner' in config and 'keytool' in config):
+        logging.critical(
+            _('Java JDK not found! Install in standard location or set java_paths!')
+        )
+        sys.exit(1)
+
+    common.assert_config_keystore(config)
 
     log_dir = 'logs'
     if not os.path.isdir(log_dir):
-        print "Creating log directory"
+        logging.info(_("Creating log directory"))
         os.makedirs(log_dir)
 
     tmp_dir = 'tmp'
     if not os.path.isdir(tmp_dir):
-        print "Creating temporary directory"
+        logging.info(_("Creating temporary directory"))
         os.makedirs(tmp_dir)
 
     output_dir = 'repo'
     if not os.path.isdir(output_dir):
-        print "Creating output directory"
+        logging.info(_("Creating output directory"))
         os.makedirs(output_dir)
 
     unsigned_dir = 'unsigned'
     if not os.path.isdir(unsigned_dir):
-        print "No unsigned directory - nothing to do"
-        sys.exit(0)
+        logging.warning(_("No unsigned directory - nothing to do"))
+        sys.exit(1)
+    binaries_dir = os.path.join(unsigned_dir, 'binaries')
+
+    if not config['keystore'] == "NONE" and not os.path.exists(config['keystore']):
+        logging.error("Config error - missing '{0}'".format(config['keystore']))
+        sys.exit(1)
 
-    # It was suggested at https://dev.guardianproject.info/projects/bazaar/wiki/FDroid_Audit
-    # that a package could be crafted, such that it would use the same signing
-    # key as an existing app. While it may be theoretically possible for such a
-    # colliding package ID to be generated, it seems virtually impossible that
-    # the colliding ID would be something that would be a) a valid package ID,
-    # and b) a sane-looking ID that would make its way into the repo.
-    # Nonetheless, to be sure, before publishing we check that there are no
-    # collisions, and refuse to do any publishing if that's the case...
     allapps = metadata.read_metadata()
-    vercodes = common.read_pkg_args(args, True)
-    allaliases = []
-    for app in allapps:
-        m = md5.new()
-        m.update(app['id'])
-        keyalias = m.hexdigest()[:8]
-        if keyalias in allaliases:
-            print "There is a keyalias collision - publishing halted"
-            sys.exit(1)
-        allaliases.append(keyalias)
-    if options.verbose:
-        print "{0} apps, {0} key aliases".format(len(allapps), len(allaliases))
+    vercodes = common.read_pkg_args(options.appid, True)
+    common.get_metadata_files(vercodes)  # only check appids
+    signed_apks = dict()
+    generated_keys = dict()
+    allaliases = check_for_key_collisions(allapps)
+    logging.info(
+        ngettext(
+            '{0} app, {1} key aliases', '{0} apps, {1} key aliases', len(allapps)
+        ).format(len(allapps), len(allaliases))
+    )
 
-    # Process any apks that are waiting to be signed...
-    for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
-
-        appid, vercode = common.apknameinfo(apkfile)
+    failed = 0
+    # Process any APKs or ZIPs that are waiting to be signed...
+    for apkfile in sorted(
+        glob.glob(os.path.join(unsigned_dir, '*.apk'))
+        + glob.glob(os.path.join(unsigned_dir, '*.zip'))
+    ):
+        appid, vercode = common.publishednameinfo(apkfile)
         apkfilename = os.path.basename(apkfile)
         if vercodes and appid not in vercodes:
             continue
         if appid in vercodes and vercodes[appid]:
             if vercode not in vercodes[appid]:
                 continue
-        print "Processing " + apkfile
+        logging.info(_("Processing {apkfilename}").format(apkfilename=apkfile))
+
+        # There ought to be valid metadata for this app, otherwise why are we
+        # trying to publish it?
+        if appid not in allapps:
+            logging.error(
+                "Unexpected {0} found in unsigned directory".format(apkfilename)
+            )
+            sys.exit(1)
+        app = allapps[appid]
+
+        build = None
+        for b in app.get("Builds", ()):
+            if b.get("versionCode") == vercode:
+                build = b
+        if app.Binaries or (build and build.binary):
+            # It's an app where we build from source, and verify the apk
+            # contents against a developer's binary, and then publish their
+            # version if everything checks out.
+            # The binary should already have been retrieved during the build
+            # process.
+
+            srcapk = re.sub(r'\.apk$', '.binary.apk', apkfile)
+            srcapk = srcapk.replace(unsigned_dir, binaries_dir)
+
+            if not os.path.isfile(srcapk):
+                logging.error(
+                    "...reference binary missing - publish skipped: '{refpath}'".format(
+                        refpath=srcapk
+                    )
+                )
+                failed += 1
+            else:
+                # Compare our unsigned one with the downloaded one...
+                compare_result = common.verify_apks(srcapk, apkfile, tmp_dir)
+                if compare_result:
+                    logging.error(
+                        "...verification failed - publish skipped : {result}".format(
+                            result=compare_result
+                        )
+                    )
+                    failed += 1
+                else:
+                    # Success! So move the downloaded file to the repo, and remove
+                    # our built version.
+                    shutil.move(srcapk, os.path.join(output_dir, apkfilename))
+                    os.remove(apkfile)
+
+                    publish_source_tarball(apkfilename, unsigned_dir, output_dir)
+                    logging.info('Published ' + apkfilename)
+
+        elif apkfile.endswith('.zip'):
+            # OTA ZIPs built by fdroid do not need to be signed by jarsigner,
+            # just to be moved into place in the repo
+            shutil.move(apkfile, os.path.join(output_dir, apkfilename))
+            publish_source_tarball(apkfilename, unsigned_dir, output_dir)
+            logging.info('Published ' + apkfilename)
 
-        # Figure out the key alias name we'll use. Only the first 8
-        # characters are significant, so we'll use the first 8 from
-        # the MD5 of the app's ID and hope there are no collisions.
-        # If a collision does occur later, we're going to have to
-        # come up with a new alogrithm, AND rename all existing keys
-        # in the keystore!
-        if appid in config['keyaliases']:
-            # For this particular app, the key alias is overridden...
-            keyalias = config['keyaliases'][appid]
-            if keyalias.startswith('@'):
-                m = md5.new()
-                m.update(keyalias[1:])
-                keyalias = m.hexdigest()[:8]
         else:
-            m = md5.new()
-            m.update(appid)
-            keyalias = m.hexdigest()[:8]
-        print "Key alias: " + keyalias
+            # It's a 'normal' app, i.e. we sign and publish it...
+            skipsigning = False
 
-        # See if we already have a key for this application, and
-        # if not generate one...
-        p = subprocess.Popen(['keytool', '-list',
-            '-alias', keyalias, '-keystore', config['keystore'],
-            '-storepass', config['keystorepass']], stdout=subprocess.PIPE)
-        output = p.communicate()[0]
-        if p.returncode !=0:
-            print "Key does not exist - generating..."
-            p = subprocess.Popen(['keytool', '-genkey',
-                '-keystore', config['keystore'], '-alias', keyalias,
-                '-keyalg', 'RSA', '-keysize', '2048',
-                '-validity', '10000',
-                '-storepass', config['keystorepass'],
-                '-keypass', config['keypass'],
-                '-dname', config['keydname']], stdout=subprocess.PIPE)
-            output = p.communicate()[0]
-            print output
-            if p.returncode != 0:
-                raise BuildException("Failed to generate key")
+            # First we handle signatures for this app from local metadata
+            signingfiles = common.metadata_find_developer_signing_files(appid, vercode)
+            if signingfiles:
+                # There's a signature of the app developer present in our
+                # metadata. This means we're going to prepare both a locally
+                # signed APK and a version signed with the developers key.
 
-        # Sign the application...
-        p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
-            '-storepass', config['keystorepass'],
-            '-keypass', config['keypass'], '-sigalg',
-            'MD5withRSA', '-digestalg', 'SHA1',
-                apkfile, keyalias], stdout=subprocess.PIPE)
-        output = p.communicate()[0]
-        print output
-        if p.returncode != 0:
-            raise BuildException("Failed to sign application")
+                signature_file, _ignored, manifest, v2_files = signingfiles
 
-        # Zipalign it...
-        p = subprocess.Popen([os.path.join(config['sdk_path'],'tools','zipalign'),
-                            '-v', '4', apkfile,
-                            os.path.join(output_dir, apkfilename)],
-                            stdout=subprocess.PIPE)
-        output = p.communicate()[0]
-        print output
-        if p.returncode != 0:
-            raise BuildException("Failed to align application")
-        os.remove(apkfile)
+                with open(signature_file, 'rb') as f:
+                    devfp = common.signer_fingerprint_short(
+                        common.get_certificate(f.read())
+                    )
+                devsigned = '{}_{}_{}.apk'.format(appid, vercode, devfp)
+                devsignedtmp = os.path.join(tmp_dir, devsigned)
 
-        # Move the source tarball into the output directory...
-        tarfilename = apkfilename[:-4] + '_src.tar.gz'
-        shutil.move(os.path.join(unsigned_dir, tarfilename),
-                os.path.join(output_dir, tarfilename))
+                common.apk_implant_signatures(apkfile, devsignedtmp, manifest=manifest)
+                if common.verify_apk_signature(devsignedtmp):
+                    shutil.move(devsignedtmp, os.path.join(output_dir, devsigned))
+                else:
+                    os.remove(devsignedtmp)
+                    logging.error('...verification failed - skipping: %s', devsigned)
+                    skipsigning = True
+                    failed += 1
 
-        print 'Published ' + apkfilename
+            # Now we sign with the F-Droid key.
+            if not skipsigning:
+                keyalias = key_alias(appid)
+                logging.info("Key alias: " + keyalias)
+
+                if create_key_if_not_existing(keyalias):
+                    generated_keys[appid] = keyalias
+
+                signed_apk_path = os.path.join(output_dir, apkfilename)
+                if os.path.exists(signed_apk_path):
+                    raise BuildException(
+                        _(
+                            "Refusing to sign '{path}', file exists in both {dir1} and {dir2} folder."
+                        ).format(path=apkfilename, dir1=unsigned_dir, dir2=output_dir)
+                    )
+
+                # Sign the application...
+                common.sign_apk(apkfile, signed_apk_path, keyalias)
+                if appid not in signed_apks:
+                    signed_apks[appid] = []
+                signed_apks[appid].append({"keyalias": keyalias, "filename": apkfile})
+
+                publish_source_tarball(apkfilename, unsigned_dir, output_dir)
+                logging.info('Published ' + apkfilename)
+
+    store_publish_signer_fingerprints(allapps.keys())
+    status_update_json(generated_keys, signed_apks)
+    logging.info('published list signing-key fingerprints')
+
+    if failed:
+        logging.error(_('%d APKs failed to be signed or verified!') % failed)
+        if options.error_on_failed:
+            sys.exit(failed)
 
 
 if __name__ == "__main__":
     main()
-

fdroidserver/readmeta.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+#
+# readmeta.py - part of the FDroid server tools
+# Copyright (C) 2014 Daniel Martí <mvdan@mvdan.cc>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from argparse import ArgumentParser
+
+from . import common, metadata
+
+
+def main():
+    parser = ArgumentParser()
+    common.setup_global_opts(parser)
+    metadata.add_metadata_arguments(parser)
+    options = parser.parse_args()
+    metadata.warnings_action = options.W
+    common.read_config()
+
+    metadata.read_metadata()
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/rewritemeta.py

@@ -1,7 +1,7 @@
-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 #
 # rewritemeta.py - part of the FDroid server tools
+# This cleans up the original .yml metadata file format.
 # Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
 #
 # This program is free software: you can redistribute it and/or modify
@@ -17,35 +17,97 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import os
-from optparse import OptionParser
-import common, metadata
+import io
+import logging
+import shutil
+import tempfile
+from argparse import ArgumentParser
+from pathlib import Path
+
+from . import _, common, metadata
 
 config = None
-options = None
+
+
+def proper_format(app):
+    s = io.StringIO()
+    # TODO: currently reading entire file again, should reuse first
+    # read in metadata.py
+    cur_content = Path(app.metadatapath).read_text(encoding='utf-8')
+    if Path(app.metadatapath).suffix == '.yml':
+        metadata.write_yaml(s, app)
+    content = s.getvalue()
+    s.close()
+    return content == cur_content
+
+
+def remove_blank_flags_from_builds(builds):
+    """Remove unset entries from Builds so they are not written out."""
+    if not builds:
+        return list()
+    newbuilds = list()
+    for build in builds:
+        new = dict()
+        for k in metadata.build_flags:
+            v = build.get(k)
+            # 0 is valid value, it should not be stripped
+            if v is None or v is False or v == '' or v == dict() or v == list():
+                continue
+            new[k] = v
+        newbuilds.append(new)
+    return newbuilds
+
 
 def main():
+    global config
 
-    global config, options
+    parser = ArgumentParser()
+    common.setup_global_opts(parser)
+    parser.add_argument(
+        "-l",
+        "--list",
+        action="store_true",
+        default=False,
+        help=_("List files that would be reformatted (dry run)"),
+    )
+    parser.add_argument(
+        "appid", nargs='*', help=_("application ID of file to operate on")
+    )
+    metadata.add_metadata_arguments(parser)
+    options = common.parse_args(parser)
+    metadata.warnings_action = options.W
 
-    # Parse command line...
-    parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]")
-    parser.add_option("-v", "--verbose", action="store_true", default=False,
-                      help="Spew out even more information than normal")
-    (options, args) = parser.parse_args()
+    config = common.read_config()
 
-    config = common.read_config(options)
+    apps = common.read_app_args(options.appid)
 
-    # Get all apps...
-    allapps = metadata.read_metadata(xref=False)
-    apps = common.read_app_args(args, allapps, False)
+    for appid, app in apps.items():
+        path = Path(app.metadatapath)
+        if path.suffix == '.yml':
+            logging.info(_("Rewriting '{appid}'").format(appid=appid))
+        else:
+            logging.warning(_('Cannot rewrite "{path}"').format(path=path))
+            continue
 
-    for app in apps:
-        print "Writing " + app['id']
-        metadata.write_metadata(os.path.join('metadata', app['id'])+'.txt', app)
+        if options.list:
+            if not proper_format(app):
+                print(path)
+            continue
+
+        # TODO these should be moved to metadata.write_yaml()
+        builds = remove_blank_flags_from_builds(app.get('Builds'))
+        if builds:
+            app['Builds'] = builds
+
+        # rewrite to temporary file before overwriting existing
+        # file in case there's a bug in write_metadata
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmp_path = Path(tmpdir) / path.name
+            metadata.write_metadata(tmp_path, app)
+            shutil.move(tmp_path, path)
+
+    logging.debug(_("Finished"))
 
-    print "Finished."
 
 if __name__ == "__main__":
     main()
-

fdroidserver/server.py

@@ -1,92 +0,0 @@
-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
-#
-# server.py - part of the FDroid server tools
-# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import sys
-import os
-import subprocess
-from optparse import OptionParser
-import common
-
-config = None
-options = None
-
-
-def main():
-
-    global config, options
-
-    # Parse command line...
-    parser = OptionParser()
-    parser.add_option("-v", "--verbose", action="store_true", default=False,
-                      help="Spew out even more information than normal")
-    (options, args) = parser.parse_args()
-
-    config = common.read_config(options)
-
-    if len(args) != 1:
-        print "Specify a single command"
-        sys.exit(1)
-
-    if args[0] != 'init' and args[0] != 'update':
-        print "The only commands currently supported are 'init' and 'update'"
-        sys.exit(1)
-
-    serverwebroot = config['serverwebroot'].rstrip('/').replace('//', '/')
-    host, fdroiddir = serverwebroot.split(':')
-    serverrepobase = os.path.basename(fdroiddir)
-    if 'nonstandardwebroot' in config and config['nonstandardwebroot'] == True:
-        standardwebroot = False
-    else:
-        standardwebroot = True
-    if serverrepobase != 'fdroid' and standardwebroot:
-        print('ERROR: serverwebroot does not end with "fdroid", '
-              + 'perhaps you meant one of these:\n\t'
-              + serverwebroot.rstrip('/') + '/fdroid\n\t'
-              + serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid')
-        sys.exit(1)
-
-    repodirs = ['repo']
-    if config['archive_older'] != 0:
-        repodirs.append('archive')
-
-    for repodir in repodirs:
-        if args[0] == 'init':
-            if subprocess.call(['ssh', '-v', host,
-                                'mkdir -p', fdroiddir + '/' + repodir]) != 0:
-                sys.exit(1)
-        elif args[0] == 'update':
-            index = os.path.join(repodir, 'index.xml')
-            indexjar = os.path.join(repodir, 'index.jar')
-            if subprocess.call(['rsync', '-u', '-v', '-r', '--delete',
-                                '--exclude', index, '--exclude', indexjar,
-                                repodir, config['serverwebroot']]) != 0:
-                sys.exit(1)
-            if subprocess.call(['rsync', '-u', '-v', '-r', '--delete',
-                                index,
-                                config['serverwebroot'] + '/' + repodir]) != 0:
-                sys.exit(1)
-            if subprocess.call(['rsync', '-u', '-v', '-r', '--delete',
-                                indexjar,
-                                config['serverwebroot'] + '/' + repodir]) != 0:
-                sys.exit(1)
-
-    sys.exit(0)
-
-if __name__ == "__main__":
-    main()

fdroidserver/signatures.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2017, Michael Poehn <michael.poehn@fsfe.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+import os
+import re
+import sys
+from argparse import ArgumentParser
+
+from . import _, common
+from .exception import FDroidException
+
+
+def extract_signature(apkpath):
+    if not os.path.exists(apkpath):
+        raise FDroidException("file APK does not exists '{}'".format(apkpath))
+    if not common.verify_apk_signature(apkpath):
+        raise FDroidException("no valid signature in '{}'".format(apkpath))
+    logging.debug('signature okay: %s', apkpath)
+
+    appid, vercode, _ignored = common.get_apk_id(apkpath)
+    sigdir = common.metadata_get_sigdir(appid, vercode)
+    if not os.path.exists(sigdir):
+        os.makedirs(sigdir)
+    common.apk_extract_signatures(apkpath, sigdir)
+
+    return sigdir
+
+
+def extract(options):
+    # Create tmp dir if missing…
+    tmp_dir = 'tmp'
+    if not os.path.exists(tmp_dir):
+        os.mkdir(tmp_dir)
+
+    if not options.APK or len(options.APK) <= 0:
+        logging.critical(_('no APK supplied'))
+        sys.exit(1)
+
+    # iterate over supplied APKs downlaod and extract them…
+    httpre = re.compile(r'https?:\/\/')
+    for apk in options.APK:
+        try:
+            if os.path.isfile(apk):
+                sigdir = extract_signature(apk)
+                logging.info(
+                    _("Fetched signatures for '{apkfilename}' -> '{sigdir}'").format(
+                        apkfilename=apk, sigdir=sigdir
+                    )
+                )
+            elif httpre.match(apk):
+                if apk.startswith('https') or options.no_check_https:
+                    try:
+                        from . import net
+
+                        tmp_apk = os.path.join(tmp_dir, 'signed.apk')
+                        net.download_file(apk, tmp_apk)
+                        sigdir = extract_signature(tmp_apk)
+                        logging.info(
+                            _(
+                                "Fetched signatures for '{apkfilename}' -> '{sigdir}'"
+                            ).format(apkfilename=apk, sigdir=sigdir)
+                        )
+                    finally:
+                        if tmp_apk and os.path.exists(tmp_apk):
+                            os.remove(tmp_apk)
+                else:
+                    logging.warning(
+                        _(
+                            'refuse downloading via insecure HTTP connection '
+                            '(use HTTPS or specify --no-https-check): {apkfilename}'
+                        ).format(apkfilename=apk)
+                    )
+        except FDroidException as e:
+            logging.warning(
+                _("Failed fetching signatures for '{apkfilename}': {error}").format(
+                    apkfilename=apk, error=e
+                )
+            )
+            if e.detail:
+                logging.debug(e.detail)
+
+
+def main():
+    parser = ArgumentParser()
+    common.setup_global_opts(parser)
+    parser.add_argument(
+        "APK", nargs='*', help=_("signed APK, either a file-path or HTTPS URL.")
+    )
+    parser.add_argument("--no-check-https", action="store_true", default=False)
+    options = common.parse_args(parser)
+    common.set_console_logging(options.verbose, options.color)
+    common.read_config()
+
+    extract(options)

fdroidserver/signindex.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+#
+# gpgsign.py - part of the FDroid server tools
+# Copyright (C) 2015, Ciaran Gultnieks, ciaran@ciarang.com
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import logging
+import os
+import time
+import zipfile
+from argparse import ArgumentParser
+
+from . import _, common, metadata
+from .exception import FDroidException
+
+config = None
+start_timestamp = time.gmtime()
+
+
+def sign_jar(jar, use_old_algs=False):
+    """Sign a JAR file with the best available algorithm.
+
+    The current signing method uses apksigner to sign the JAR so that
+    it will automatically select algorithms that are compatible with
+    Android SDK 23, which added the most recent algorithms:
+    https://developer.android.com/reference/java/security/Signature
+
+    This signing method uses then inherits the default signing
+    algothim settings, since Java and Android both maintain those.
+    That helps avoid a repeat of being stuck on an old signing
+    algorithm.  That means specifically that this call to apksigner
+    does not specify any of the algorithms.
+
+    The old indexes must be signed by SHA1withRSA otherwise they will
+    no longer be compatible with old Androids.
+
+    This method requires a properly initialized config object.
+
+    """
+    if use_old_algs:
+        # This does use old hashing algorithms, i.e. SHA1, but that's not
+        # broken yet for file verification.  This could be set to SHA256,
+        # but then Android < 4.3 would not be able to verify it.
+        # https://code.google.com/p/android/issues/detail?id=38321
+        args = [
+            config['jarsigner'],
+            '-keystore',
+            config['keystore'],
+            '-storepass:env',
+            'FDROID_KEY_STORE_PASS',
+            '-digestalg',
+            'SHA1',
+            '-sigalg',
+            'SHA1withRSA',
+            jar,
+            config['repo_keyalias'],
+        ]
+        if config['keystore'] == 'NONE':
+            args += config['smartcardoptions']
+        else:  # smardcards never use -keypass
+            args += ['-keypass:env', 'FDROID_KEY_PASS']
+    else:
+        # https://developer.android.com/studio/command-line/apksigner
+        args = [
+            config['apksigner'],
+            'sign',
+            '--min-sdk-version',
+            '23',  # enable all current algorithms
+            '--max-sdk-version',
+            '24',  # avoid future incompatible algorithms
+            # disable all APK signature types, only use JAR sigs aka v1
+            '--v1-signing-enabled',
+            'true',
+            '--v2-signing-enabled',
+            'false',
+            '--v3-signing-enabled',
+            'false',
+            '--v4-signing-enabled',
+            'false',
+            '--ks',
+            config['keystore'],
+            '--ks-pass',
+            'env:FDROID_KEY_STORE_PASS',
+            '--ks-key-alias',
+            config['repo_keyalias'],
+        ]
+        if config['keystore'] == 'NONE':
+            args += common.get_apksigner_smartcardoptions(config['smartcardoptions'])
+        else:  # smardcards never use --key-pass
+            args += ['--key-pass', 'env:FDROID_KEY_PASS']
+        args += [jar]
+    env_vars = {
+        'FDROID_KEY_STORE_PASS': config['keystorepass'],
+        'FDROID_KEY_PASS': config.get('keypass', ""),
+    }
+    p = common.FDroidPopen(args, envs=env_vars)
+    if not use_old_algs and p.returncode != 0:
+        # workaround for apksigner v30 on f-droid.org publish server
+        v4 = args.index("--v4-signing-enabled")
+        del args[v4 + 1]
+        del args[v4]
+        p = common.FDroidPopen(args, envs=env_vars)
+        if p.returncode != 0:
+            raise FDroidException("Failed to sign %s: %s" % (jar, p.output))
+
+
+def sign_index(repodir, json_name):
+    """Sign data file like entry.json to make a signed JAR like entry.jar.
+
+    The data file like index-v1.json means that there is unsigned
+    data.  That file is then stuck into a jar and signed by the
+    signing process. This is a bit different than sign_jar, which is
+    used for index.jar: that creates index.xml then puts that in a
+    index_unsigned.jar, then that file is signed.
+
+    This also checks to make sure that the JSON files are intact
+    before signing them.  Broken JSON files should never be signed, so
+    taking some extra time and failing hard is the preferred
+    option. This signing process can happen on an entirely separate
+    machine and file tree, so this ensures that nothing got broken
+    during transfer.
+
+    """
+    json_file = os.path.join(repodir, json_name)
+    with open(json_file, encoding="utf-8") as fp:
+        data = json.load(fp)
+        if json_name == 'entry.json':
+            index_file = os.path.join(repodir, data['index']['name'].lstrip('/'))
+            sha256 = common.sha256sum(index_file)
+            if sha256 != data['index']['sha256']:
+                raise FDroidException(
+                    _('%s has bad SHA-256: %s') % (index_file, sha256)
+                )
+            with open(index_file) as fp:
+                index = json.load(fp)
+            if not isinstance(index, dict):
+                raise FDroidException(_('%s did not produce a dict!') % index_file)
+        elif json_name == 'index-v1.json':
+            [metadata.App(app) for app in data["apps"]]
+
+    name, ext = common.get_extension(json_name)
+    jar_file = os.path.join(repodir, name + '.jar')
+    with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar:
+        jar.write(json_file, json_name)
+
+    if json_name in ('index.xml', 'index-v1.json'):
+        sign_jar(jar_file, use_old_algs=True)
+    else:
+        sign_jar(jar_file)
+
+
+def status_update_json(signed):
+    """Output a JSON file with metadata about this run."""
+    logging.debug(_('Outputting JSON'))
+    output = common.setup_status_output(start_timestamp)
+    if signed:
+        output['signed'] = signed
+    common.write_status_json(output)
+
+
+def main():
+    global config
+
+    parser = ArgumentParser()
+    common.setup_global_opts(parser)
+    common.parse_args(parser)
+
+    config = common.read_config()
+
+    if 'jarsigner' not in config:
+        raise FDroidException(
+            _(
+                'Java jarsigner not found! Install in standard location or set java_paths!'
+            )
+        )
+
+    repodirs = ['repo']
+    if config['archive_older'] != 0:
+        repodirs.append('archive')
+
+    signed = []
+    for output_dir in repodirs:
+        if not os.path.isdir(output_dir):
+            raise FDroidException("Missing output directory '" + output_dir + "'")
+
+        unsigned = os.path.join(output_dir, 'index_unsigned.jar')
+        if os.path.exists(unsigned):
+            sign_jar(unsigned)
+            index_jar = os.path.join(output_dir, 'index.jar')
+            os.rename(unsigned, index_jar)
+            logging.info('Signed index in ' + output_dir)
+            signed.append(index_jar)
+
+        json_name = 'index-v1.json'
+        index_file = os.path.join(output_dir, json_name)
+        if os.path.exists(index_file):
+            sign_index(output_dir, json_name)
+            logging.info('Signed ' + index_file)
+            signed.append(index_file)
+
+        json_name = 'entry.json'
+        index_file = os.path.join(output_dir, json_name)
+        if os.path.exists(index_file):
+            sign_index(output_dir, json_name)
+            logging.info('Signed ' + index_file)
+            signed.append(index_file)
+
+    if not signed:
+        logging.info(_("Nothing to do"))
+    status_update_json(signed)
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/stats.py

@@ -1,259 +0,0 @@
-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
-#
-# stats.py - part of the FDroid server tools
-# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import sys
-import os
-import re
-import time
-import traceback
-import glob
-from optparse import OptionParser
-import paramiko
-import common, metadata
-import socket
-import subprocess
-
-def carbon_send(key, value):
-    s = socket.socket()
-    s.connect((config['carbon_host'], config['carbon_port']))
-    msg = '%s %d %d\n' % (key, value, int(time.time()))
-    s.sendall(msg)
-    s.close()
-
-options = None
-config = None
-
-def main():
-
-    global options, config
-
-    # Parse command line...
-    parser = OptionParser()
-    parser.add_option("-v", "--verbose", action="store_true", default=False,
-                      help="Spew out even more information than normal")
-    parser.add_option("-d", "--download", action="store_true", default=False,
-                      help="Download logs we don't have")
-    parser.add_option("--nologs", action="store_true", default=False,
-                      help="Don't do anything logs-related")
-    (options, args) = parser.parse_args()
-
-    config = common.read_config(options)
-
-    if not config['update_stats']:
-        print "Stats are disabled - check your configuration"
-        sys.exit(1)
-
-    # Get all metadata-defined apps...
-    metaapps = metadata.read_metadata(options.verbose)
-
-    statsdir = 'stats'
-    logsdir = os.path.join(statsdir, 'logs')
-    datadir = os.path.join(statsdir, 'data')
-    if not os.path.exists(statsdir):
-        os.mkdir(statsdir)
-    if not os.path.exists(logsdir):
-        os.mkdir(logsdir)
-    if not os.path.exists(datadir):
-        os.mkdir(datadir)
-
-    if options.download:
-        # Get any access logs we don't have...
-        ssh = None
-        ftp = None
-        try:
-            print 'Retrieving logs'
-            ssh = paramiko.SSHClient()
-            ssh.load_system_host_keys()
-            ssh.connect('f-droid.org', username='fdroid', timeout=10,
-                    key_filename=config['webserver_keyfile'])
-            ftp = ssh.open_sftp()
-            ftp.get_channel().settimeout(60)
-            print "...connected"
-
-            ftp.chdir('logs')
-            files = ftp.listdir()
-            for f in files:
-                if f.startswith('access-') and f.endswith('.log.gz'):
-
-                    destpath = os.path.join(logsdir, f)
-                    destsize = ftp.stat(f).st_size
-                    if (not os.path.exists(destpath) or
-                            os.path.getsize(destpath) != destsize):
-                        print "...retrieving " + f
-                        ftp.get(f, destpath)
-        except Exception:
-            traceback.print_exc()
-            sys.exit(1)
-        finally:
-            #Disconnect
-            if ftp is not None:
-                ftp.close()
-            if ssh is not None:
-                ssh.close()
-
-    knownapks = common.KnownApks()
-    unknownapks = []
-
-    if not options.nologs:
-        # Process logs
-        if options.verbose:
-            print 'Processing logs...'
-        apps = {}
-        logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
-        logsearch = re.compile(logexpr).search
-        for logfile in glob.glob(os.path.join(logsdir,'access-*.log.gz')):
-            if options.verbose:
-                print '...' + logfile
-            p = subprocess.Popen(["zcat", logfile], stdout = subprocess.PIPE)
-            matches = (logsearch(line) for line in p.stdout)
-            for match in matches:
-                if match and match.group('statuscode') == '200':
-                    uri = match.group('uri')
-                    if uri.endswith('.apk'):
-                        _, apkname = os.path.split(uri)
-                        app = knownapks.getapp(apkname)
-                        if app:
-                            appid, _ = app
-                            if appid in apps:
-                                apps[appid] += 1
-                            else:
-                                apps[appid] = 1
-                        else:
-                            if not apkname in unknownapks:
-                                unknownapks.append(apkname)
-
-        # Calculate and write stats for total downloads...
-        lst = []
-        alldownloads = 0
-        for app, count in apps.iteritems():
-            lst.append(app + " " + str(count))
-            if config['stats_to_carbon']:
-                carbon_send('fdroid.download.' + app.replace('.', '_'), count)
-            alldownloads += count
-        lst.append("ALL " + str(alldownloads))
-        f = open('stats/total_downloads_app.txt', 'w')
-        f.write('# Total downloads by application, since October 2011\n')
-        for line in sorted(lst):
-            f.write(line + '\n')
-        f.close()
-
-    # Calculate and write stats for repo types...
-    if options.verbose:
-        print "Processing repo types..."
-    repotypes = {}
-    for app in metaapps:
-        if len(app['Repo Type']) == 0:
-            rtype = 'none'
-        else:
-            if app['Repo Type'] == 'srclib':
-                rtype = common.getsrclibvcs(app['Repo'])
-            else:
-                rtype = app['Repo Type']
-        if rtype in repotypes:
-            repotypes[rtype] += 1;
-        else:
-            repotypes[rtype] = 1
-    f = open('stats/repotypes.txt', 'w')
-    for rtype, count in repotypes.iteritems():
-        f.write(rtype + ' ' + str(count) + '\n')
-    f.close()
-
-    # Calculate and write stats for update check modes...
-    if options.verbose:
-        print "Processing update check modes..."
-    ucms = {}
-    for app in metaapps:
-        checkmode = app['Update Check Mode'].split('/')[0]
-        if checkmode in ucms:
-            ucms[checkmode] += 1;
-        else:
-            ucms[checkmode] = 1
-    f = open('stats/update_check_modes.txt', 'w')
-    for checkmode, count in ucms.iteritems():
-        f.write(checkmode + ' ' + str(count) + '\n')
-    f.close()
-
-    if options.verbose:
-        print "Processing categories..."
-    ctgs = {}
-    for app in metaapps:
-        if app['Categories'] is None:
-            continue
-        categories = [c.strip() for c in app['Categories'].split(',')]
-        for category in categories:
-            if category in ctgs:
-                ctgs[category] += 1;
-            else:
-                ctgs[category] = 1
-    f = open('stats/categories.txt', 'w')
-    for category, count in ctgs.iteritems():
-        f.write(category + ' ' + str(count) + '\n')
-    f.close()
-
-    if options.verbose:
-        print "Processing antifeatures..."
-    afs = {}
-    for app in metaapps:
-        if app['AntiFeatures'] is None:
-            continue
-        antifeatures = [a.strip() for a in app['AntiFeatures'].split(',')]
-        for antifeature in antifeatures:
-            if antifeature in afs:
-                afs[antifeature] += 1;
-            else:
-                afs[antifeature] = 1
-    f = open('stats/antifeatures.txt', 'w')
-    for antifeature, count in afs.iteritems():
-        f.write(antifeature + ' ' + str(count) + '\n')
-    f.close()
-
-    # Calculate and write stats for licenses...
-    if options.verbose:
-        print "Processing licenses..."
-    licenses = {}
-    for app in metaapps:
-        license = app['License']
-        if license in licenses:
-            licenses[license] += 1;
-        else:
-            licenses[license] = 1
-    f = open('stats/licenses.txt', 'w')
-    for license, count in licenses.iteritems():
-        f.write(license + ' ' + str(count) + '\n')
-    f.close()
-
-    # Write list of latest apps added to the repo...
-    if options.verbose:
-        print "Processing latest apps..."
-    latest = knownapks.getlatest(10)
-    f = open('stats/latestapps.txt', 'w')
-    for app in latest:
-        f.write(app + '\n')
-    f.close()
-
-    if unknownapks:
-        print '\nUnknown apks:'
-        for apk in unknownapks:
-            print apk
-
-    print "Finished."
-
-if __name__ == "__main__":
-    main()
-

fdroidserver/tail.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+
+"""Python-Tail - Unix tail follow implementation in Python.
+
+python-tail can be used to monitor changes to a file.
+
+Example
+-------
+>>> import tail
+>>>
+>>> # Create a tail instance
+>>> t = tail.Tail('file-to-be-followed')
+>>>
+>>> # Register a callback function to be called when a new line is found in the followed file.
+>>> # If no callback function is registerd, new lines would be printed to standard out.
+>>> t.register_callback(callback_function)
+>>>
+>>> # Follow the file with 5 seconds as sleep time between iterations.
+>>> # If sleep time is not provided 1 second is used as the default time.
+>>> t.follow(s=5)
+"""
+
+# Author - Kasun Herath <kasunh01 at gmail.com>
+# Source - https://github.com/kasun/python-tail
+
+# modified by Hans-Christoph Steiner <hans@eds.org> to add the
+# background thread and support reading multiple lines per read cycle
+
+import os
+import sys
+import threading
+import time
+
+
+class Tail(object):
+    """Represents a tail command."""
+
+    def __init__(self, tailed_file):
+        """Initialize a Tail instance.
+
+        Check for file validity, assigns callback function to standard out.
+
+        Parameters
+        ----------
+        tailed_file
+          File to be followed.
+        """
+        self.check_file_validity(tailed_file)
+        self.tailed_file = tailed_file
+        self.callback = sys.stdout.write
+        self.t_stop = threading.Event()
+
+    def start(self, s=1):
+        """Start tailing a file in a background thread.
+
+        Parameters
+        ----------
+        s
+          Number of seconds to wait between each iteration; Defaults to 3.
+        """
+        t = threading.Thread(target=self.follow, args=(s,))
+        t.start()
+
+    def stop(self):
+        """Stop a background tail."""
+        self.t_stop.set()
+
+    def follow(self, s=1):
+        """Do a tail follow.
+
+        If a callback function is registered it is called with every new line.
+        Else printed to standard out.
+
+        Parameters
+        ----------
+        s
+          Number of seconds to wait between each iteration; Defaults to 1.
+        """
+        with open(self.tailed_file) as file_:
+            # Go to the end of file
+            file_.seek(0, 2)
+            while not self.t_stop.is_set():
+                curr_position = file_.tell()
+                lines = file_.readlines()
+                if len(lines) == 0:
+                    file_.seek(curr_position)
+                else:
+                    for line in lines:
+                        self.callback(line)
+                time.sleep(s)
+
+    def register_callback(self, func):
+        """Override default callback function to provided function."""
+        self.callback = func
+
+    def check_file_validity(self, file_):
+        """Check whether the a given file exists, readable and is a file."""
+        if not os.access(file_, os.F_OK):
+            raise TailError("File '%s' does not exist" % (file_))
+        if not os.access(file_, os.R_OK):
+            raise TailError("File '%s' not readable" % (file_))
+        if os.path.isdir(file_):
+            raise TailError("File '%s' is a directory" % (file_))
+
+
+class TailError(Exception):
+    def __init__(self, msg):
+        super().__init__()
+        self.message = msg
+
+    def __str__(self):
+        return self.message