diff --git a/.bandit b/.bandit deleted file mode 100644 index dc28620f..00000000 --- a/.bandit +++ /dev/null @@ -1,3 +0,0 @@ -[bandit] -skips: B110,B404,B408,B603,B607,B322 -targets: . diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index 43412092..00000000 --- a/.editorconfig +++ /dev/null @@ -1,15 +0,0 @@ -root = true - -[*] -charset = utf-8 -end_of_line = lf -insert_final_newline = true -trim_trailing_whitespace = true - -[**.py] -indent_style = space -indent_size = 4 - -[.gitlab-ci.yml] -indent_style = space -indent_size = 2 diff --git a/.gitignore b/.gitignore index ce3a0e9a..223807b5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,14 +4,13 @@ *.box TAGS .idea -.ropeproject/ # files generated by build -/build/ -/dist/ +build/ +dist/ env/ ENV/ -/fdroidserver.egg-info/ +fdroidserver.egg-info/ pylint.parseable /.testfiles/ README.rst @@ -19,7 +18,6 @@ README.rst # editor tmp files .*.swp -.ropeproject/ # files generated by tests tmp/ @@ -27,6 +25,7 @@ tmp/ /tests/repo/status # files used in manual testing +/config.py /config.yml /tmp/ /logs/ @@ -41,25 +40,14 @@ makebuildserver.config.py /tests/OBBMainPatchCurrent.apk /tests/OBBMainTwoVersions.apk /tests/archive/categories.txt -/tests/archive/diff/[1-9]*.json -/tests/archive/entry.jar -/tests/archive/entry.json /tests/archive/icons* +/tests/archive/index.jar +/tests/archive/index_unsigned.jar +/tests/archive/index.xml /tests/archive/index-v1.jar /tests/archive/index-v1.json -/tests/archive/index-v2.json -/tests/archive/index.css -/tests/archive/index.html -/tests/archive/index.jar -/tests/archive/index.png -/tests/archive/index.xml -/tests/archive/index_unsigned.jar /tests/metadata/org.videolan.vlc/en-US/icon*.png -/tests/repo/diff/[1-9]*.json -/tests/repo/index.css -/tests/repo/index.html /tests/repo/index.jar -/tests/repo/index.png /tests/repo/index_unsigned.jar /tests/repo/index-v1.jar /tests/repo/info.guardianproject.urzip/ @@ -74,6 +62,3 @@ makebuildserver.config.py # generated by gettext locale/*/LC_MESSAGES/fdroidserver.mo - -# sphinx -public/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 65510c45..52aafa2e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,52 +1,18 @@ ---- - -# Use merge request pipelines when a merge request is open for the branch. -# Use branch pipelines when a merge request is not open for the branch. -# https://docs.gitlab.com/ci/yaml/workflow/#switch-between-branch-pipelines-and-merge-request-pipelines -workflow: - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS - when: never - - if: $CI_COMMIT_BRANCH - - -stages: - - lint - - test # default for jobs that do not specify stage: - - deploy - variables: pip: pip3 --timeout 100 --retries 10 - # speed up git checkout phase +# speed up git checkout phase GIT_DEPTH: 1 -# Run the whole test suite in an environment that is like the -# buildserver guest VM. This installs python3-babel because that is -# only used by the test suite, and not needed in the buildserver. -# -# Some extra packages are required for this test run that are not -# provided by the buildserver since they are not needed there: -# * python3-babel for compiling localization files -# * gnupg-agent for the full signing setup -# * python3-clint for fancy progress bars for users -# * python3-pycountry for linting config/mirrors.yml -buildserver run-tests: - image: registry.gitlab.com/fdroid/fdroidserver:buildserver +test: + image: registry.gitlab.com/fdroid/ci-images-base script: - - apt-get update - - apt-get install gnupg-agent python3-babel python3-biplist python3-clint python3-pycountry - - ./tests/run-tests - # make sure that translations do not cause stacktraces - - cd $CI_PROJECT_DIR/locale - - for locale in *; do - test -d $locale || continue; - for cmd in `sed -n 's/.*("\(.*\)", *_.*/\1/p' $CI_PROJECT_DIR/fdroid`; do - LANGUAGE=$locale $CI_PROJECT_DIR/fdroid $cmd --help > /dev/null; - done - done + - $pip install -e .[test] + # the `fdroid build` test in tests/run-tests needs android-23 + - echo y | $ANDROID_HOME/tools/bin/sdkmanager "platforms;android-23" + - cd tests + - ./complete-ci-tests # Test that the parsing of the .yml metadata format didn't change from last # released version. This uses the commit ID of the release tags, @@ -56,18 +22,17 @@ buildserver run-tests: # The COMMIT_ID should be bumped after each release, so that the list # of sed hacks needed does not continuously grow. metadata_v0: - image: registry.gitlab.com/fdroid/fdroidserver:buildserver + image: registry.gitlab.com/fdroid/ci-images-base variables: GIT_DEPTH: 1000 - RELEASE_COMMIT_ID: 50aa35772b058e76b950c01e16019c072c191b73 # after switching to `git rev-parse` + RELEASE_COMMIT_ID: 37f37ebd88e79ebe93239b72ed5503d5bde13f4b # 2.0a~ script: - git fetch https://gitlab.com/fdroid/fdroidserver.git $RELEASE_COMMIT_ID - cd tests - - export GITCOMMIT=$(git rev-parse HEAD) + - export GITCOMMIT=`git describe` - git checkout $RELEASE_COMMIT_ID - cd .. - git clone --depth 1 https://gitlab.com/fdroid/fdroiddata.git - - rm -f fdroiddata/config.yml # ignore config for this test - cd fdroiddata - ../tests/dump_internal_metadata_format.py - cd .. @@ -76,9 +41,8 @@ metadata_v0: - cd fdroiddata - ../tests/dump_internal_metadata_format.py - sed -i - -e '/ArchivePolicy:/d' - -e '/FlattrID:/d' - -e '/RequiresRoot:/d' + -e '/Liberapay:/d' + -e '/OpenCollective/d' metadata/dump_*/*.yaml - diff -uw metadata/dump_* @@ -90,330 +54,132 @@ metadata_v0: - echo Etc/UTC > /etc/timezone - echo 'APT::Install-Recommends "0";' 'APT::Install-Suggests "0";' + 'APT::Acquire::Retries "20";' 'APT::Get::Assume-Yes "true";' - 'Acquire::Retries "20";' 'Dpkg::Use-Pty "0";' 'quiet "1";' >> /etc/apt/apt.conf.d/99gitlab - # Ubuntu and other distros often lack https:// support - - grep Debian /etc/issue.net - && { find /etc/apt/sources.list* -type f | xargs sed -i s,http:,https:, ; } - # The official Debian docker images ship without ca-certificates, - # TLS certificates cannot be verified until that is installed. The - # following code turns off TLS verification, and enables HTTPS, so - # at least unverified TLS is used for apt-get instead of plain - # HTTP. Once ca-certificates is installed, the CA verification is - # enabled by removing this config. This set up makes the initial - # `apt-get update` and `apt-get install` look the same as verified - # TLS to the network observer and hides the metadata. - - echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/99nocacertificates - apt-get update - - apt-get install ca-certificates - - rm /etc/apt/apt.conf.d/99nocacertificates - apt-get dist-upgrade -# For jobs that only need to run when there are changes to Python files. -.python-rules-changes: &python-rules-changes - rules: - - changes: - - .gitlab-ci.yml - - fdroid - - makebuildserver - - setup.py - - fdroidserver/*.py - - tests/*.py - - -# Since F-Droid uses Debian as its default platform, from production -# servers to CI to contributor machines, it is important to know when -# changes in Debian break our stuff. This tests against the latest -# dependencies as they are included in Debian. debian_testing: image: debian:testing <<: *apt-template - rules: - - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver" + only: + - master@fdroid/fdroidserver script: - apt-get install aapt androguard apksigner - dexdump fdroidserver git gnupg - ipfs-cid - python3-biplist python3-defusedxml - python3-libcloud - python3-pycountry python3-setuptools - sdkmanager + zipalign - python3 -c 'import fdroidserver' - python3 -c 'import androguard' - - python3 -c 'import sdkmanager' - cd tests - ./run-tests - # Test using latest LTS set up with the PPA, including Recommends. +# bionic's apksigner, which comes from Recommends:, requires binfmt +# support in the kernel. ubuntu_lts_ppa: image: ubuntu:latest <<: *apt-template - rules: - - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver" + only: + - master@fdroid/fdroidserver script: - export ANDROID_HOME=/usr/lib/android-sdk - apt-get install gnupg - - while ! apt-key adv --keyserver keyserver.ubuntu.com --recv-key 9AAC253193B65D4DF1D0A13EEC4632C79C5E0151; do sleep 15; done - - export RELEASE=$(sed -n 's,^Suites\x3a \([a-z]*\).*,\1,p' /etc/apt/sources.list.d/*.sources | head -1) + - while ! apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key 9AAC253193B65D4DF1D0A13EEC4632C79C5E0151; do sleep 15; done + - export RELEASE=`sed -n 's,^deb [^ ][^ ]* \([a-z]*\).*,\1,p' /etc/apt/sources.list | head -1` - echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list - apt-get update - apt-get dist-upgrade - - apt-get install --install-recommends - dexdump - fdroidserver - git - python3-biplist - python3-pycountry - python3-setuptools - sdkmanager - - # Test things work with a default branch other than 'master' - - git config --global init.defaultBranch thisisnotmasterormain - + - mount | grep binfmt_misc || mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc + - apt-get install --install-recommends binfmt-support fdroidserver git python3-defusedxml python3-setuptools + - ls -l /proc/sys/fs/binfmt_misc || true + - test -e /proc/sys/fs/binfmt_misc/jarwrapper || apt -qy purge apksigner - cd tests - ./run-tests - -# Test to see how rclone works with S3 -test_deploy_to_s3_with_rclone: - image: debian:bookworm-slim - <<: *apt-template - tags: - - saas-linux-small-amd64 # the shared runners are known to support Docker. - services: - - name: docker:dind - command: ["--tls=false"] - variables: - DOCKER_HOST: "tcp://docker:2375" - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - before_script: - # ensure minio is up before executing tests - - apt-get update - - apt-get install -y - androguard - apksigner - curl - docker.io - git - python3-venv - rclone - # This job requires working docker but will silently fail if docker is not available - - docker info - - python3 -m venv --system-site-packages test-venv - - . test-venv/bin/activate - - pip install testcontainers[minio] - - pip install . - script: - - python3 -m unittest -k test_update_remote_storage_with_rclone --verbose - rules: - - changes: - - .gitlab-ci.yml - - fdroidserver/deploy.py - - tests/test_deploy.py - - tests/test_integration.py - - -# Test using Ubuntu/jammy LTS (supported til April, 2027) with depends -# from pypi and sdkmanager. The venv is used to isolate the dist -# tarball generation environment from the clean install environment. -ubuntu_jammy_pip: - image: ubuntu:jammy +# Test using Xenial LTS with all depends from pypi. The venv is used +# to isolate the dist tarball generation environment from the clean +# install environment. Xenial's pip is too old to install all the +# dependencies, so this has to uppgrade pip and setuptools in order to +# run the install. +ubuntu_xenial_pip: + image: ubuntu:xenial <<: *apt-template script: - - apt-get install git default-jdk-headless python3-pip python3-venv rsync - + - apt-get install git default-jdk-headless python3-pip python3-venv rsync zipalign libarchive13 + - rm -rf env + - pyvenv env + - . env/bin/activate + - $pip install --upgrade babel pip setuptools # setup venv to act as release build machine - - python3 -m venv sdist-env + - python -m venv sdist-env - . sdist-env/bin/activate - - ./setup.py sdist + - ./setup.py compile_catalog sdist - deactivate - - tar tzf dist/fdroidserver-*.tar.gz - + - tar tzf dist/fdroidserver-*.tar.gz | grep locale/de/LC_MESSAGES/fdroidserver.mo # back to bare machine to act as user's install machine - - export ANDROID_HOME=/opt/android-sdk - - $pip install sdkmanager - - sdkmanager 'build-tools;35.0.0' - - # Install extras_require.optional from setup.py - - $pip install biplist pycountry - + - $pip install --upgrade pip setuptools - $pip install dist/fdroidserver-*.tar.gz - - tar xzf dist/fdroidserver-*.tar.gz - - cd fdroidserver-* - - export PATH=$PATH:$ANDROID_HOME/build-tools/35.0.0 - - fdroid=`which fdroid` ./tests/run-tests + - test -e /usr/share/locale/de/LC_MESSAGES/fdroidserver.mo + - ./tests/run-tests - # check localization was properly installed - - LANGUAGE='de' fdroid --help | grep 'Gültige Befehle sind' +# test install process on a bleeding edge distro with pip +arch_pip_install: + image: archlinux/base + only: + - master@fdroid/fdroidserver + script: + - pacman --sync --sysupgrade --refresh --noconfirm git grep python-pip python-virtualenv tar + - pip install -e . + - fdroid + - fdroid readmeta + - fdroid update --help - -# Run all the various linters and static analysis tools. -hooks/pre-commit: - stage: lint - image: debian:bookworm-slim +lint_format_safety_bandit_checks: + image: alpine:3.10 # cannot upgrade until bandit supports Python 3.8 variables: LANG: C.UTF-8 script: - - apt-get update - - apt-get -y install --no-install-recommends - bash - ca-certificates - dash - gcc - git - make - pycodestyle - pyflakes3 - python3-dev - python3-git - python3-nose - python3-pip - python3-yaml - - ./hooks/pre-commit - -bandit: - image: debian:bookworm-slim - <<: *python-rules-changes - <<: *apt-template - script: - - apt-get install python3-pip - - $pip install --break-system-packages bandit - - bandit -r -ii --ini .bandit - -pylint: - stage: lint - image: debian:bookworm-slim - <<: *python-rules-changes - <<: *apt-template - script: - - apt-get install pylint python3-pip - - $pip install --break-system-packages pylint-gitlab - - pylint --output-format=colorized,pylint_gitlab.GitlabCodeClimateReporter:pylint-report.json + - apk add --no-cache bash build-base dash ca-certificates gcc python3 python3-dev + - python3 -m ensurepip + - $pip install Babel 'bandit<1.6.0' pycodestyle pyflakes pylint safety + - export EXITVALUE=0 + - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; } + - ./hooks/pre-commit || set_error + - ./tests/test-gradlew-fdroid || set_error + - bandit + -ii + -s B110,B322,B404,B408,B410,B603,B607 + -r $CI_PROJECT_DIR fdroid + || set_error + - safety check --full-report || set_error + - pylint --rcfile=.pylint-rcfile --output-format=colorized --reports=n fdroid makebuildserver setup.py fdroidserver/*.py tests/*.py - artifacts: - reports: - codequality: pylint-report.json - when: always - - -shellcheck: - stage: lint - image: debian:bookworm-slim - rules: - - changes: - - .gitlab-ci.yml - - hooks/install-hooks.sh - - hooks/pre-commit - - tests/run-tests - <<: *apt-template - script: - - apt-get install shellcheck - # TODO GitLab Code Quality report https://github.com/koalaman/shellcheck/issues/3155 - - shellcheck --exclude SC2046,SC2090 --severity=warning --color - hooks/install-hooks.sh - hooks/pre-commit - tests/run-tests - -# Check all the dependencies in Debian to mirror production. CVEs are -# generally fixed in the latest versions in pip/pypi.org, so it isn't -# so important to scan that kind of install in CI. -# https://docs.safetycli.com/safety-docs/installation/gitlab -safety: - image: debian:bookworm-slim - rules: - - if: $SAFETY_API_KEY - changes: - - .gitlab-ci.yml - - .safety-policy.yml - - pyproject.toml - - setup.py - <<: *apt-template - variables: - LANG: C.UTF-8 - script: - - apt-get install - fdroidserver - python3-biplist - python3-pip - python3-pycountry - - $pip install --break-system-packages . - - - $pip install --break-system-packages safety - - python3 -m safety --key "$SAFETY_API_KEY" --stage cicd scan - - -# TODO tests/*/*/*.yaml are not covered -yamllint: - stage: lint - image: debian:bookworm-slim - rules: - - changes: - - .gitlab-ci.yml - - .safety-policy.yml - - .yamllint - - tests/*.yml - - tests/*/*.yml - - tests/*/*/.*.yml - <<: *apt-template - variables: - LANG: C.UTF-8 - script: - - apt-get install yamllint - - yamllint - .gitlab-ci.yml - .safety-policy.yml - .yamllint - tests/*.yml - tests/*/*.yml - tests/*/*/.*.yml - - -locales: - stage: lint - image: debian:bookworm-slim - variables: - LANG: C.UTF-8 - script: - - apt-get update - - apt-get -y install --no-install-recommends - gettext - make - python3-babel - - export EXITVALUE=0 - - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; } + tests/*.TestCase + || set_error + - apk add --no-cache gettext make - make -C locale compile || set_error - rm -f locale/*/*/*.mo - - pybabel compile --domain=fdroidserver --directory locale 2>&1 | { grep -F "error:" && exit 1; } || true + - pybabel compile --domain=fdroidserver --directory locale 2>&1 | (grep -F "error:" && exit 1) || true - exit $EXITVALUE - -black: - stage: lint - image: debian:bookworm-slim - <<: *apt-template - script: - - apt-get install black - - black --check --diff --color $CI_PROJECT_DIR - fedora_latest: - image: fedora:39 # support ends on 2024-11-12 + image: fedora:latest + only: + - master@fdroid/fdroidserver script: # tricks to hopefully make runs more reliable - echo "timeout=600" >> /etc/dnf/dnf.conf @@ -426,429 +192,102 @@ fedora_latest: findutils git gnupg - java-17-openjdk-devel - openssl + java-1.8.0-openjdk-devel python3 python3-babel python3-matplotlib python3-pip - python3-pycountry rsync + unzip + wget which - - $pip install sdkmanager - - ./setup.py sdist + - ./setup.py compile_catalog sdist - useradd -m -c "test account" --password "fakepassword" testuser - su testuser --login --command "cd `pwd`; $pip install --user dist/fdroidserver-*.tar.gz" - test -e ~testuser/.local/share/locale/de/LC_MESSAGES/fdroidserver.mo + - wget --no-verbose -O tools.zip https://dl.google.com/android/repository/tools_r25.2.5-linux.zip + - unzip -q tools.zip + - rm tools.zip - export BUILD_TOOLS_VERSION=`sed -n "s,^MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py` + - export JAVA_HOME=/etc/alternatives/jre - export ANDROID_HOME=`pwd`/android-sdk + - mkdir $ANDROID_HOME + - mv tools $ANDROID_HOME/ - mkdir -p $ANDROID_HOME/licenses/ - printf "\n8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e\n24333f8a63b6825ea9c5514f83c2829b004d1fee" > $ANDROID_HOME/licenses/android-sdk-license - printf "\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license - printf "\n79120722343a6f314e0719f863036c702b0e6b2a\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license-old - mkdir ~/.android - touch ~/.android/repositories.cfg - - sdkmanager "platform-tools" "build-tools;$BUILD_TOOLS_VERSION" + - echo y | $ANDROID_HOME/tools/bin/sdkmanager "platform-tools" + - echo y | $ANDROID_HOME/tools/bin/sdkmanager "build-tools;$BUILD_TOOLS_VERSION" - chown -R testuser . - cd tests - su testuser --login --command - "cd `pwd`; export CI=$CI ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests" - - -macOS: - tags: - - saas-macos-medium-m1 - rules: - - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver" - script: - - export HOMEBREW_CURL_RETRIES=10 - - brew update > /dev/null - - brew upgrade - - brew install fdroidserver - - # Android SDK and Java JDK - - brew install --cask android-commandlinetools temurin # temurin is a JDK - - # test suite dependencies - - brew install bash coreutils gnu-sed - # TODO port tests/run-tests to POSIX and gsed, it has a couple GNU-isms like du --bytes - - export PATH="$(brew --prefix fdroidserver)/libexec/bin:$(brew --prefix coreutils)/libexec/gnubin:$PATH" - - - brew autoremove - - brew info fdroidserver - - - export BUILD_TOOLS_VERSION=`gsed -n "s,^MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py` - - export ANDROID_HOME="$(brew --prefix)/share/android-commandlinetools" - - mkdir -p "$ANDROID_HOME/licenses" - - echo -e "\n8933bad161af4178b1185d1a37fbf41ea5269c55" > "$ANDROID_HOME/licenses/android-sdk-license" - - echo -e "\nd56f5187479451eabf01fb78af6dfcb131a6481e" >> "$ANDROID_HOME/licenses/android-sdk-license" - - echo -e "\n24333f8a63b6825ea9c5514f83c2829b004d1fee" >> "$ANDROID_HOME/licenses/android-sdk-license" - - $(brew --prefix)/bin/sdkmanager "build-tools;$BUILD_TOOLS_VERSION" - - - echo "macOS sticks with bash 3.x because of licenses, so avoid new bash syntax" - - /bin/bash --version - - /bin/bash -n tests/run-tests - - # test fdroidserver from git with current package's dependencies - - fdroid="$(brew --prefix fdroidserver)/libexec/bin/python3 $PWD/fdroid" ./tests/run-tests - + "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests" gradle: - image: debian:trixie-slim - <<: *apt-template - rules: - - changes: - - .gitlab-ci.yml - - makebuildserver + image: alpine:3.7 + variables: + LANG: C.UTF-8 script: - - apt-get install - ca-certificates - git - python3-colorama - python3-packaging - python3-requests + - apk add --no-cache ca-certificates git python3 + # if this is a merge request fork, then only check if makebuildserver changed + - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then + git fetch https://gitlab.com/fdroid/fdroidserver.git; + for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do + test "$f" == "makebuildserver" && export CHANGED="yes"; + done; + test -z "$CHANGED" && exit; + fi + - python3 -m ensurepip + - $pip install beautifulsoup4 requests - ./tests/gradle-release-checksums.py - -# Run an actual build in a simple, faked version of the buildserver guest VM. fdroid build: - image: registry.gitlab.com/fdroid/fdroidserver:buildserver - rules: - - changes: - - .gitlab-ci.yml - - fdroidserver/build.py - - fdroidserver/common.py - - fdroidserver/exception.py - - fdroidserver/metadata.py - - fdroidserver/net.py - - fdroidserver/scanner.py - - fdroidserver/vmtools.py - # for the docker: job which depends on this one - - makebuildserver - - buildserver/* + image: registry.gitlab.com/fdroid/ci-images-client + only: + refs: + - branches + - pipelines + changes: + - .gitlab-ci.yml + - buildserver/provision-apt-get-install + - fdroidserver/build.py + - fdroidserver/common.py + - fdroidserver/exception.py + - fdroidserver/metadata.py + - fdroidserver/net.py + - fdroidserver/scanner.py + - fdroidserver/vmtools.py cache: key: "$CI_JOB_NAME" paths: - .gradle script: - - apt-get update + - bash buildserver/provision-apt-get-install http://deb.debian.org/debian - apt-get dist-upgrade - - apt-get clean - - - test -n "$fdroidserver" || source /etc/profile.d/bsenv.sh - - - ln -fsv "$CI_PROJECT_DIR" "$fdroidserver" - - # TODO remove sdkmanager install once it is included in the buildserver image - - apt-get install sdkmanager - - rm -rf "$ANDROID_HOME/tools" # TODO remove once sdkmanager can upgrade installed packages - - sdkmanager "tools" "platform-tools" "build-tools;31.0.0" - - - git ls-remote https://gitlab.com/fdroid/fdroiddata.git master - - git clone --depth 1 https://gitlab.com/fdroid/fdroiddata.git - - cd fdroiddata - - for d in build logs repo tmp unsigned $home_vagrant/.android; do - test -d $d || mkdir $d; - chown -R vagrant $d; - done - - - export GRADLE_USER_HOME=$home_vagrant/.gradle - - export fdroid="sudo --preserve-env --user vagrant - env PATH=$fdroidserver:$PATH - env PYTHONPATH=$fdroidserver:$fdroidserver/examples - env PYTHONUNBUFFERED=true - env TERM=$TERM - env HOME=$home_vagrant - fdroid" - - - git -C $home_vagrant/gradlew-fdroid pull - - - chown -R vagrant $home_vagrant - - chown -R vagrant $fdroidserver/.git - - chown vagrant $fdroidserver/ - - chown -R vagrant .git - - chown vagrant . - - # try user build - - $fdroid build --verbose --latest org.fdroid.fdroid.privileged - - # try on-server build - - $fdroid build --verbose --on-server --no-tarball --latest org.fdroid.fdroid - - # each `fdroid build --on-server` run expects sudo, then uninstalls it - - if dpkg --list sudo; then echo "sudo should not be still there"; exit 1; fi - - 'if [ ! -f repo/status/running.json ]; then echo "ERROR: running.json does not exist!"; exit 1; fi' - - 'if [ ! -f repo/status/build.json ]; then echo "ERROR: build.json does not exist!"; exit 1; fi' - - -# test the plugin API and specifically the fetchsrclibs plugin, which -# is used by the `fdroid build` job. This uses a fixed commit from -# fdroiddata because that one is known to work, and this is a CI job, -# so it should be isolated from the normal churn of fdroiddata. -plugin_fetchsrclibs: - image: debian:bookworm-slim - <<: *apt-template - rules: - - changes: - - .gitlab-ci.yml - - examples/fdroid_fetchsrclibs.py - - fdroidserver/__main__.py - script: - - apt-get install - curl - git - python3-cffi - python3-matplotlib - python3-nacl - python3-paramiko - python3-pil - python3-pip - python3-pycparser - python3-venv - - python3 -m venv --system-site-packages env + - apt-get install -t stretch-backports + python3-asn1crypto + python3-pip + python3-ruamel.yaml + python3-setuptools + python3-venv + - apt-get purge fdroidserver + - pyvenv env --system-site-packages - . env/bin/activate + - $pip install -e . - export PATH="$CI_PROJECT_DIR:$PATH" - - export PYTHONPATH="$CI_PROJECT_DIR/examples" - # workaround https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1003252 - - export SETUPTOOLS_USE_DISTUTILS=stdlib - - $pip install -e . - - fdroid | grep fetchsrclibs + - export PYTHONPATH=$CI_PROJECT_DIR + - export PYTHONUNBUFFERED=true - - mkdir fdroiddata - - commitid=b9e9a077d720c86ff6fff4dbb341254cc4370b1a - - curl https://gitlab.com/fdroid/fdroiddata/-/archive/${commitid}/fdroiddata-${commitid}.tar.gz - | tar -xz --directory=fdroiddata --strip-components=1 + - git clone https://gitlab.com/fdroid/fdroiddata.git --depth 1 - cd fdroiddata - - fdroid fetchsrclibs freemap.opentrail:4 --verbose - - test -d build/freemap.opentrail/.git - - test -d build/srclib/andromaps/.git - - test -d build/srclib/freemaplib/.git - - test -d build/srclib/freemaplibProj/.git - - test -d build/srclib/JCoord/.git - - test -d build/srclib/javaproj/.git + - test -d build || mkdir build - -# test a full update and deploy cycle to gitlab.com -servergitmirrors: - image: debian:bookworm-slim - <<: *apt-template - rules: - - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver" - script: - - apt-get install - default-jdk-headless - git - openssh-client - openssl - python3-cffi - python3-cryptography - python3-matplotlib - python3-nacl - python3-pil - python3-pip - python3-pycparser - python3-setuptools - python3-venv - rsync - wget - - apt-get install apksigner - - python3 -m venv --system-site-packages env - - . env/bin/activate - - export PYTHONPATH=`pwd` - - export SETUPTOOLS_USE_DISTUTILS=stdlib # https://github.com/pypa/setuptools/issues/2956 - - $pip install -e . - - mkdir /root/.ssh/ - - ./tests/key-tricks.py - - ssh-keyscan gitlab.com >> /root/.ssh/known_hosts - - test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo - - cp tests/config.yml tests/keystore.jks /tmp/fdroid/ - - cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/ - - cd /tmp/fdroid - - touch fdroid-icon.png - - printf "\nservergitmirrors\x3a 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.yml - - $PYTHONPATH/fdroid update --verbose --create-metadata - - $PYTHONPATH/fdroid deploy --verbose - - export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json` - - echo $DLURL - - wget $DLURL/index-v1.jar - - diff repo/index-v1.jar index-v1.jar - -Build documentation: - image: debian:bookworm-slim - <<: *python-rules-changes - <<: *apt-template - script: - - apt-get install make python3-sphinx python3-numpydoc python3-pydata-sphinx-theme pydocstyle fdroidserver - - apt purge fdroidserver - # ignore vendored files - - pydocstyle --verbose --match='(?!apksigcopier|looseversion|setup|test_).*\.py' fdroidserver - - cd docs - - sphinx-apidoc -o ./source ../fdroidserver -M -e - - PYTHONPATH=.. sphinx-autogen -o generated source/*.rst - - PYTHONPATH=.. make html - artifacts: - paths: - - docs/build/html/ - - -# this job will only run in branches called "windows" until the Windows port is complete -Windows: - tags: - - windows - rules: - - if: $CI_COMMIT_BRANCH == "windows" - script: - - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" - - choco install --no-progress -y git --force --params "/GitAndUnixToolsOnPath" - - choco install --no-progress -y python3 --version=3.10 - - choco install --no-progress -y jdk8 - - choco install --no-progress -y rsync - - refreshenv - - python -m pip install --upgrade babel pip setuptools - - python -m pip install -e . - - - $files = @(Get-ChildItem tests\test_*.py) - - foreach ($f in $files) { - write-output $f; - python -m unittest $f; - if( $LASTEXITCODE -eq 0 ) { - write-output "SUCCESS $f"; - } else { - write-output "ERROR $f failed"; - } - } - - # these are the tests that must pass - - python -m unittest -k - checkupdates - exception - import_subcommand - test_lint - test_metadata - test_rewritemeta - test_vcs - tests.test_init - tests.test_main - after_script: - - Copy-Item C:\ProgramData\chocolatey\logs\chocolatey.log - artifacts: - when: always - paths: - - "*.log" - allow_failure: - exit_codes: 1 - - -pages: - image: alpine:latest - stage: deploy - script: - - cp docs/build/html public -r # GL Pages needs the files in a directory named "public" - artifacts: - paths: - - public - needs: - - job: "Build documentation" - optional: true - rules: - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' # only publish pages on default (master) branch - - -# This job pushes the official CI docker image based on the master -# branch, so in fdroid/fdroidserver, it should only run on the master -# branch. Otherwise, tags or other branches will overwrite the docker -# image which is supposed to be what is in master. -docker: - dependencies: - - fdroid build - rules: - - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver" - changes: - - .gitlab-ci.yml - - makebuildserver - - buildserver/* - image: docker:dind - services: - - docker:dind - variables: - RELEASE_IMAGE: $CI_REGISTRY_IMAGE:buildserver - script: - # git ref names can contain many chars that are not allowed in docker tags - - export TEST_IMAGE=$CI_REGISTRY_IMAGE:$(printf $CI_COMMIT_REF_NAME | sed 's,[^a-zA-Z0-9_.-],_,g') - - cd buildserver - - docker build -t $TEST_IMAGE --build-arg GIT_REV_PARSE_HEAD=$(git rev-parse HEAD) . - - docker tag $TEST_IMAGE $RELEASE_IMAGE - - docker tag $TEST_IMAGE ${RELEASE_IMAGE}-bookworm - - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com - # This avoids filling up gitlab.com free tier accounts with unused docker images. - - if test -z "$FDROID_PUSH_DOCKER_IMAGE"; then - echo "Skipping docker push to save quota on your gitlab namespace."; - echo "If you want to enable the push, set FDROID_PUSH_DOCKER_IMAGE in"; - echo "https://gitlab.com/$CI_PROJECT_NAMESPACE/fdroidserver/-/settings/ci_cd#js-cicd-variables-settings"; - exit 0; - fi - - docker push $RELEASE_IMAGE - - docker push $RELEASE_IMAGE-bookworm - - -# PUBLISH is the signing server. It has a very minimal manual setup. -PUBLISH: - image: debian:bookworm-backports - <<: *python-rules-changes - script: - - apt-get update - - apt-get -qy upgrade - - apt-get -qy install --no-install-recommends -t bookworm-backports - androguard - apksigner - curl - default-jdk-headless - git - gpg - gpg-agent - python3-asn1crypto - python3-defusedxml - python3-git - python3-ruamel.yaml - python3-yaml - rsync - - # Run only relevant parts of the test suite, other parts will fail - # because of this minimal base setup. - - python3 -m unittest - tests/test_gpgsign.py - tests/test_metadata.py - tests/test_publish.py - tests/test_signatures.py - tests/test_signindex.py - - - cd tests - - mkdir archive - - mkdir unsigned - - cp urzip-release-unsigned.apk unsigned/info.guardianproject.urzip_100.apk - - grep '^key.*pass' config.yml | sed 's,\x3a ,=,' > $CI_PROJECT_DIR/variables - - sed -Ei 's,^(key.*pass|keystore)\x3a.*,\1\x3a {env\x3a \1},' config.yml - - printf '\ngpghome\x3a {env\x3a gpghome}\n' >> config.yml - - | - tee --append $CI_PROJECT_DIR/variables < FestplattenSchnitzel -Hans-Christoph Steiner diff --git a/.pylint-rcfile b/.pylint-rcfile new file mode 100644 index 00000000..4685d7f4 --- /dev/null +++ b/.pylint-rcfile @@ -0,0 +1,45 @@ +[MASTER] + +# Use multiple processes to speed up Pylint. +jobs=4 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence=HIGH,INFERENCE + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=invalid-name,missing-docstring,no-member + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_,e,f,fp + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + diff --git a/.safety-policy.yml b/.safety-policy.yml deleted file mode 100644 index ea44e7e6..00000000 --- a/.safety-policy.yml +++ /dev/null @@ -1,55 +0,0 @@ ---- - -version: '3.0' - -scanning-settings: - max-depth: 6 - exclude: - -report: - dependency-vulnerabilities: - enabled: true - auto-ignore-in-report: - vulnerabilities: - 52495: - reason: setuptools comes from Debian - expires: '2025-01-31' - 60350: - reason: GitPython comes from Debian https://security-tracker.debian.org/tracker/CVE-2023-40267 - expires: '2025-01-31' - 60789: - reason: GitPython comes from Debian https://security-tracker.debian.org/tracker/CVE-2023-40590 - expires: '2025-01-31' - 60841: - reason: GitPython comes from Debian https://security-tracker.debian.org/tracker/CVE-2023-41040 - expires: '2025-01-31' - 62044: - reason: "F-Droid doesn't fetch pip dependencies directly from hg/mercurial repositories: https://data.safetycli.com/v/62044/f17/" - expires: '2025-01-31' - 63687: - reason: Only affects Windows https://security-tracker.debian.org/tracker/CVE-2024-22190 - expires: '2026-01-31' - 67599: - reason: Only affects pip when using --extra-index-url, which is never the case in fdroidserver CI. - expires: '2026-05-31' - 70612: - reason: jinja2 is not used by fdroidserver, nor any dependencies I could find via debtree and pipdeptree. - expires: '2026-05-31' - 72132: - reason: We get these packages from Debian, zipp is not used in production, and its only a DoS. - expires: '2026-08-31' - 72236: - reason: setuptools is not used in production to download or install packages, they come from Debian. - expires: '2026-08-31' - -fail-scan-with-exit-code: - dependency-vulnerabilities: - enabled: true - fail-on-any-of: - cvss-severity: - - critical - - high - - medium - -security-updates: - dependency-vulnerabilities: diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..a012541b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,91 @@ + +# Use the Android base system since it provides the SDK, etc. +language: java + +matrix: + include: + - os: osx + osx_image: xcode12 + env: ANDROID_SDK_ROOT=/usr/local/share/android-sdk + env: ANDROID_HOME=/usr/local/share/android-sdk + - os: osx + osx_image: xcode10.3 + env: ANDROID_SDK_ROOT=/usr/local/share/android-sdk + env: ANDROID_HOME=/usr/local/share/android-sdk + +android: + components: + - android-23 # required for `fdroid build` test + - build-tools-28.0.3 # required for `fdroid build` test + licenses: + - 'android-sdk-preview-.+' + - 'android-sdk-license-.+' + +# * ensure java8 is installed since Android SDK doesn't work with Java9 +# * Java needs to be at least 1.8.0_131 to have MD5 properly disabled +# https://blogs.oracle.com/java-platform-group/oracle-jre-will-no-longer-trust-md5-signed-code-by-default +# https://opsech.io/posts/2017/Jun/09/openjdk-april-2017-security-update-131-8u131-and-md5-signed-jars.html +# * mercurial is unused and requires Python 2.x +install: + - export HOMEBREW_CURL_RETRIES=10 + - brew update > /dev/null + - if [ "`sw_vers -productVersion | sed 's,10\.\([0-9]*\).*,\1,'`" -ge 14 ]; then + python3 --version; + elif [ "`sw_vers -productVersion | sed 's,10\.\([0-9]*\).*,\1,'`" -gt 10 ]; then + brew uninstall mercurial --force; + brew upgrade python; + else + brew install python3; + fi + - brew install dash bash gnu-sed gradle jenv + - export PATH="/usr/local/opt/gnu-sed/libexec/gnubin:$PATH" + - brew uninstall java --force || true + - brew cask uninstall java --force || true + - brew tap adoptopenjdk/openjdk + - travis_retry brew cask install adoptopenjdk8 + - travis_retry brew cask install android-sdk + + - export AAPT_VERSION=`sed -n "s,^MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py` + - mkdir -p "$ANDROID_HOME/licenses" + - echo -e "\n8933bad161af4178b1185d1a37fbf41ea5269c55" > "$ANDROID_HOME/licenses/android-sdk-license" + - echo -e "\nd56f5187479451eabf01fb78af6dfcb131a6481e" >> "$ANDROID_HOME/licenses/android-sdk-license" + - echo -e "\n24333f8a63b6825ea9c5514f83c2829b004d1fee" >> "$ANDROID_HOME/licenses/android-sdk-license" + - echo -e "\n84831b9409646a918e30573bab4c9c91346d8abd" > "$ANDROID_HOME/licenses/android-sdk-preview-license" + - echo y | travis_retry $ANDROID_HOME/tools/bin/sdkmanager "platform-tools" > /dev/null + - echo y | travis_retry $ANDROID_HOME/tools/bin/sdkmanager "build-tools;$AAPT_VERSION" > /dev/null + - echo y | travis_retry $ANDROID_HOME/tools/bin/sdkmanager "platforms;android-23" > /dev/null + + - travis_retry sudo pip3 install --progress-bar off babel + - travis_retry sudo pip3 install --quiet --progress-bar off --editable . + - sudo rm -rf fdroidserver.egg-info + + - ls -l /System/Library/Java/JavaVirtualMachines || true + - ls -l /Library/Java/JavaVirtualMachines || true + - for f in /Library/Java/JavaVirtualMachines/*.jdk; do jenv add $f; done + - echo $PATH + - echo $JAVA_HOME + - jenv versions + - /usr/libexec/java_home + - java -version + - which java + - javac -version + - which javac + - jarsigner -help + - which jarsigner + - keytool -help + - which keytool + - sudo rm -rf /Library/Java/JavaVirtualMachines/jdk1.8.0_1*.jdk || true + +# The OSX tests seem to run slower, they often timeout. So only run +# the test suite with the installed version of fdroid. +# +# macOS sticks with bash 3.x because of licenses, so avoid use new bash syntax +script: + - /bin/bash --version + - /bin/bash -n gradlew-fdroid tests/run-tests + + - ./tests/run-tests + +after_failure: + - cd $TRAVIS_BUILD_DIR + - ls -lR | curl -F 'clbin=<-' https://clbin.com diff --git a/.vscode/extensions.json b/.vscode/extensions.json deleted file mode 100644 index f0fec078..00000000 --- a/.vscode/extensions.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "recommendations": [ - "ms-python.python", - ] -} diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index da31cd7f..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "python.formatting.blackArgs": [ - "--config=pyproject.toml" - ], - "python.formatting.provider": "black", - "python.linting.banditEnabled": true, - "python.linting.banditArgs": [ - "-ii", - "--ini=.bandit", - ], - "python.linting.enabled": true, - "python.linting.mypyArgs": [ - "--config-file=mypy.ini" - ], - "python.linting.mypyEnabled": true, - "python.linting.flake8Enabled": true, - "python.linting.pylintArgs": [ - "--rcfile=.pylint-rcfile" - ], - "python.linting.pylintEnabled": true, -} diff --git a/.well-known/funding-manifest-urls b/.well-known/funding-manifest-urls deleted file mode 100644 index 9935b4d4..00000000 --- a/.well-known/funding-manifest-urls +++ /dev/null @@ -1 +0,0 @@ -https://f-droid.org/funding.json diff --git a/.yamllint b/.yamllint deleted file mode 100644 index 067a389e..00000000 --- a/.yamllint +++ /dev/null @@ -1,7 +0,0 @@ ---- - -extends: default -rules: - document-start: disable - line-length: disable - truthy: disable diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b61a8f2..3c4f69a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,375 +4,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) -## [2.5.0] - NEXT - -### Removed - -* deploy: `awsaccesskeyid:` and `awssecretkey:` config items removed, use the - standard env vars: `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. - -## [2.4.2] - 2025-06-24 - -### Fixed - -* nightly: fix bug that clones nightly repo to wrong location - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1672 -* Sync translations for all supported languages: es pl ru - -## [2.4.1] - 2025-06-23 - +## [Unreleased] ### Added - -* build: Clearer error messages when working with Git. -* verify: generate .json files that list all reports - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1632 - -### Fixed - -* deploy: use master branch when working complete git-mirror repo - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1666 -* update: use ctime/mtime to control _strip_and_copy_image runs - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1665 -* update: If categories.yml only has icon:, then add name: - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1659 -* update: fix handling of Triple-T 1.0.0 graphics - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1652 -* update: never execute any VCS e.g. git - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1630 -* config: lazyload environment variables in config.yml - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1645 -* config: make localized name/description/icon optional - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1649 -* lint: add repo_key_sha256 to list of valid config keys - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1643 -* build: calculate all combinations of gradle flavors - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1638 -* build: set SOURCE_DATE_EPOCH from app's git otherwise fdroiddata metadata file - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1653 -* Sync translations for all supported languages: ca cs de fr ga ja pl pt pt_BR - pt_PT ru sq tr uk zh_Hans - -### Removed - -## [2.4.0] - 2025-03-25 - -### Added - -* lint: support the base _config.yml_. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1606 - -### Fixed - -* Expand {env: foo} config syntax to be allowed any place a string is. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1610 -* Only show "unsafe permissions on config.yml" when secrets are present. -* Standardized config files on ruamel.yaml with a YAML 1.2 data format. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1611 -* Brought back error when a package has multiple package types (e.g. xapk and - apk). https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1602 -* Reworked test suite to be entirely based on Python unittest (thanks @mindston). - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1587 -* publish/signindex/gpgsign no longer load the _qrcode_ and _requests_ modules, - and can operate without them installed. -* scanner: add bun.lock as lock file of package.json - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1615 -* index: fail if user sets mirrors:isPrimary wrong - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1617 - https://gitlab.com/fdroid/fdroidserver/-/issues/1125 -* Sync translations for all supported languages: bo ca cs de es fr ga hu it ja - ko nb_NO pl pt pt_BR pt_PT ro ru sq sr sw tr uk zh_Hans zh_Hant - -### Removed - -* checkupdates: remove auto_author: config, it is no longer used. -* Purge support for the long-deprecated _config.py_ config file. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1607 - - -## [2.3.5] - 2025-01-20 - -### Fixed - -* Fix issue where APKs with v1-only signatures and targetSdkVersion < 30 could - be maliciously crafted to bypass AllowedAPKSigningKeys - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1588 -* Ignore apksigner v33.x, it has bugs verifying APKs with v3/v3.1 sigs. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1593 -* Sync translations for: ca cs de es fr ga ja pt_BR pt_PT ru sq sr uk zh_Hans - -## [2.3.4] - 2024-12-12 - -### Fixed - -* Fix localhost network tests on systems with IPv6. -* lint: only error out on missing extlib on versions not archived. - -## [2.3.3] - 2024-12-11 - -### Added - -* verify: `--clean-up-verified` to delete files used when verifying an APK if - the verification was successful. - -### Fixed - -* Support Python 3.13 in the full test suite. -* Sync translations for: ca de fr ja pl ro ru sr ta -* update: only generate _index.png_ when making _index.html_, allowing the repo - operator to set a different repo icon, e.g. not the QR Code. - -## [2.3.2] - 2024-11-26 - -### Fixed - -* install: fix downloading from GitHub Releases and Maven Central. -* Sync translations for: ca fa fr pt ru sr ta zh_Hant - -## [2.3.1] - 2024-11-25 - -### Fixed - -* Sync all translations for: cs de es fr ga pt_BR ru sq zh_Hans. -* Drop use of deprecated imghdr library to support Python 3.13. -* Install biplist and pycountry by default on macOS. -* Fixed running test suite out of dist tarball. - -## [2.3.0] - 2024-11-21 - -### Added - -* YAML 1.2 as native format for all _.yml_ files, including metadata and config. -* install: will now fetch _F-Droid.apk_ and install it via `adb`. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1546 -* scanner: scan APK Signing Block for known block types like Google Play - Signature aka "Frosting". - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1555 -* Support Rclone for deploying to many different cloud services. -* deploy: support deploying to GitHub Releases. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1471 -* scanner: support libs.versions.toml - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1526 -* Consider subdir for triple-t metadata discovery in Flutter apps. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1541 -* deploy: added `index_only:` mode for mirroring the index to small hosting - locations. https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1420 -* Support publishing repos in AltStore format. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1465 -* Support indexing iOS IPA app files. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1413 -* deploy: _config/mirrors.yml_ file with support for adding per-mirror metadata, - like `countryCode:`. -* Repo's categories are now set in the config files. -* lint: check syntax of config files. -* publish: `--error-on-failed` to exit when signing/verifying fails. -* scanner: `--refresh` and `refresh_config:` to control triggering a refresh of - the rule sets. -* Terminal output colorization and `--color` argument to control it. -* New languages: Catalan (ca), Irish (ga), Japanese (ja), Serbian (sr), and - Swahili (sw). -* Support donation links from `community_bridge`, `buy_me_a_coffee`. - -### Fixed - -* Use last modified time and file size for caching data about scanned APKs - instead of SHA-256 checksum. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1542 -* `repo_web_base_url:` config for generating per-app URLs for viewing in - browsers. https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1178 -* `fdroid scanner` flags WebAssembly binary _.wasm_ files. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1562 -* Test suite as standard Python `unittest` setup (thanks @ghost.adh). -* scanner: error on dependency files without lock file. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1504 -* nightly: finding APKs in the wrong directory. (thanks @WrenIX) - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1512 -* `AllowedAPKSigningKeys` works with all single-signer APK signatures. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1466 -* Sync all translations for: cs de it ko pl pt pt_BR pt_PT ro ru sq tr uk - zh_Hans zh_Hant. -* Support Androguard 4.x. -* Support Python 3.12. - -### Removed - -* Drop all uses of _stats/known_apks.txt_ and the `update_stats:` config key. - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1547 -* The `maven:` field is now always a string, with `yes` as a legacy special - value. It is no longer treated like a boolean in any case. -* scanner: jcenter is no longer an allowed Maven repo. -* build: `--reset-server` removed (thanks @gotmi1k). - -## [2.2.2] - 2024-04-24 - -### Added - -* Include sdkmanager as dep in setup.py for Homebrew package. - https://github.com/Homebrew/homebrew-core/pull/164510 - -## [2.2.1] - 2023-03-09 - -### Added - -* `download_repo_index_v2()` and `download_repo_index_v2()` API functions - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1323 - -### Fixed - -* Fix OpenJDK detection on different CPU architectures - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1315 - -### Removed - -* Purge all references to `zipalign`, that is delegated to other things - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1316 -* Remove obsolete, unused `buildozer` build type - https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1322 - -## [2.2.0] - 2023-02-20 - -### Added -* Support index-v2 format, localizable Anti-Features, Categories -* New entry point for repos, entry.jar, signed with modern algorithms -* New config/ subdirectory for localizable configuration -* Script entries in metadata files (init, prebuild, build, etc) now handled as - lists so they now support using && or ; in the script, and behave like - .gitlab-ci.yml and other CI YAML. -* GPG signatures for index-v1.json and index-v2.json -* Use default.txt as fallback changelog when inserting fastlane metadata -* scanner: F-Droid signatures now maintained in fdroid/suss -* scanner: maintain signature sources in config.yml, including Exodus Privacy -* scanner: use dexdump for class names -* scanner: directly scan APK files when given a path -* scanner: recursively scan APKs for DEX and ZIP using file magic -* signindex: validate index files before signing -* update: set ArchivePolicy based on VercodeOperation/signature -* Include IPFS CIDv1 in index-v2.json for hosting repos on IPFS -* Per-repo beta channel configuration -* Add Czech translation - -### Fixed - -* apksigner v30 or higher now required for verifying and signing APKs -* 3.9 as minimum supported Python version -* Lots of translation updates -* Better pip packaging -* nightly: big overhaul for reliable operation on all Debian/Ubuntu versions -* Improved logging, fewer confusing verbose messages -* scanner: fix detection of binary files without extension -* import: more reliable operation, including Flutter apps -* Support Java 20 and up - -### Removed -* Remove obsolete `fdroid stats` command - -## [2.1.1] - 2022-09-06 - -* gradlew-fdroid: Include latest versions and checksums -* nightly: update Raw URLs to fix breakage and avoid redirects -* signindex: gpg-sign index-v1.json and deploy it -* update: fix --use-date-from-apk when used with files (#1012) - -## [2.1] - 2022-02-22 - -For a more complete overview, see the [2.1 -milestone](https://gitlab.com/fdroid/fdroidserver/-/milestones/11) - -## [2.0.5] - 2022-09-06 - -### Fixed - -* gradlew-fdroid: Include latest versions and checksums -* nightly: add support for GitHub Actions -* nightly: update Raw URLs to fix breakage and avoid redirects -* update: fix --use-date-from-apk when used with files (#1012) -* Fix GitLab CI - -## [2.0.4] - 2022-06-29 - -### Fixed - -* deploy: ensure progress is instantiated before trying to use it -* signindex: gpg-sign index-v1.json and deploy it - [1080](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1080) - [1124](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1124) - -## [2.0.3] - 2021-07-01 - -### Fixed - -* Support AutoUpdateMode: Version without pattern - [931](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/931) - -## [2.0.2] - 2021-06-01 - -### Fixed - -* fix "ruamel round_trip_dump will be removed" - [932](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/932) - -## [2.0.1] - 2021-03-09 - -### Fixed - -* metadata: stop setting up source repo when running lint/rewritemeta -* scanner: show error if scan_binary fails to run apkanalyzer -* common: properly parse version from NDK's source.properties -* update: stop extracting and storing XML icons, they're useless -* index: raise error rather than crash on bad repo file -* update: handle large, corrupt, or inaccessible fastlane/triple-t files -* Update SPDX License List -* checkupdates: set User-Agent to make gitlab.com happy -* Run push_binary_transparency only once - -## [2.0] - 2021-01-31 - -For a more complete overview, see the [2.0 -milestone](https://gitlab.com/fdroid/fdroidserver/-/milestones/10) - -### Added -* `fdroid update` inserts donation links based on upstream's _FUNDING.yml_ - ([!754](https://gitlab.com/fdroid/fdroidserver/merge_requests/754)) -* Stable, public API for most useful functions - ([!798](https://gitlab.com/fdroid/fdroidserver/merge_requests/798)) -* Load with any YAML lib and use with the API, no more custom parser needed - ([!826](https://gitlab.com/fdroid/fdroidserver/merge_requests/826)) - ([!838](https://gitlab.com/fdroid/fdroidserver/merge_requests/838)) -* _config.yml_ for a safe, easy, standard configuration format +* makebuildserver: added ndk r20 ([!663](https://gitlab.com/fdroid/fdroidserver/merge_requests/663)) -* Config options can be set from environment variables using this syntax: - `keystorepass: {env: keystorepass}` +* added support for gradle 5.5.1 + ([!656](https://gitlab.com/fdroid/fdroidserver/merge_requests/656)) +* add SHA256 to filename of repo graphics ([!669](https://gitlab.com/fdroid/fdroidserver/merge_requests/669)) -* Add SHA256 to filename of repo graphics - ([!669](https://gitlab.com/fdroid/fdroidserver/merge_requests/669)) -* Support for srclibs metadata in YAML format +* support for srclibs metadata in YAML format ([!700](https://gitlab.com/fdroid/fdroidserver/merge_requests/700)) -* Check srclibs and app-metadata files with yamllint +* check srclibs and app-metadata files with yamllint ([!721](https://gitlab.com/fdroid/fdroidserver/merge_requests/721)) -* Added plugin system for adding subcommands to `fdroid` - ([!709](https://gitlab.com/fdroid/fdroidserver/merge_requests/709)) -* `fdroid update`, `fdroid publish`, and `fdroid signindex` now work - with SmartCard HSMs, specifically the NitroKey HSM - ([!779](https://gitlab.com/fdroid/fdroidserver/merge_requests/779)) - ([!782](https://gitlab.com/fdroid/fdroidserver/merge_requests/782)) -* `fdroid update` support for Triple-T Gradle Play Publisher v2.x - ([!683](https://gitlab.com/fdroid/fdroidserver/merge_requests/683)) -* Translated into: bo de es fr hu it ko nb_NO pl pt pt_BR pt_PT ru sq tr uk - zh_Hans zh_Hant ### Fixed -* Smoother process for signing APKs with `apksigner` - ([!736](https://gitlab.com/fdroid/fdroidserver/merge_requests/736)) - ([!821](https://gitlab.com/fdroid/fdroidserver/merge_requests/821)) -* `apksigner` is used by default on new repos -* All parts except _build_ and _publish_ work without the Android SDK - ([!821](https://gitlab.com/fdroid/fdroidserver/merge_requests/821)) -* Description: is now passed to clients unchanged, no HTML conversion - ([!828](https://gitlab.com/fdroid/fdroidserver/merge_requests/828)) -* Lots of improvements for scanning for proprietary code and trackers - ([!748](https://gitlab.com/fdroid/fdroidserver/merge_requests/748)) - ([!REPLACE](https://gitlab.com/fdroid/fdroidserver/merge_requests/REPLACE)) - ([!844](https://gitlab.com/fdroid/fdroidserver/merge_requests/844)) -* `fdroid mirror` now generates complete, working local mirror repos * fix build-logs dissapearing when deploying ([!685](https://gitlab.com/fdroid/fdroidserver/merge_requests/685)) * do not crash when system encoding can not be retrieved @@ -388,31 +33,11 @@ milestone](https://gitlab.com/fdroid/fdroidserver/-/milestones/10) ([!651](https://gitlab.com/fdroid/fdroidserver/merge_requests/651)) * `fdroid init` generates PKCS12 keystores, drop Java < 8 support ([!801](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/801)) -* Parse Version Codes specified in hex - ([!692](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/692)) -* Major refactoring on core parts of code to be more Pythonic - ([!756](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/756)) -* `fdroid init` now works when installed with pip ### Removed -* Removed all support for _.txt_ and _.json_ metadata +* removed support for txt and json metadata ([!772](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/772)) -* dropped support for Debian 8 _jessie_ and 9 _stretch_ -* dropped support for Ubuntu releases older than bionic 18.04 -* dropped `fdroid server update` and `fdroid server init`, - use `fdroid deploy` -* `fdroid dscanner` was removed. - ([!711](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/711)) * `make_current_version_link` is now off by default -* Dropped `force_build_tools` config option - ([!797](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/797)) -* Dropped `accepted_formats` config option, there is only _.yml_ now - ([!818](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/818)) -* `Provides:` was removed as a metadata field - ([!654](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/654)) -* Remove unused `latestapps.dat` - ([!794](https://gitlab.com/fdroid/fdroidserver/-/merge_requests/794)) - ## [1.1.4] - 2019-08-15 ### Fixed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 226c0854..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,66 +0,0 @@ -There are many ways to contribute, you can find out all the ways on our -[Contribute](https://f-droid.org/contribute/) page. Find out how to get -involved, including as a translator, data analyst, tester, helping others, and -much more! - -## Contributing Code - -We want more contributors and want different points of view represented. Some -parts of the code make contributing quick and easy. Other parts make it -difficult and slow, so we ask that contributors have patience. - -To submit a patch, please open a merge request on GitLab. If you are thinking of -making a large contribution, open an issue or merge request before starting -work, to get comments from the community. Someone may be already working on the -same thing, or there may be reasons why that feature isn't implemented. Once -there is agreement, then the work might need to proceed asynchronously with the -core team towards the solution. - -To make it easier to review and accept your merge request, please follow these -guidelines: - -* When at all possible, include tests. These can either be added to an existing - test, or completely new. Practicing test-driven development will make it - easiest to get merged. That usually means starting your work by writing tests. - -* See [help-wanted](https://gitlab.com/fdroid/fdroidserver/-/issues/?sort=updated_desc&state=opened&label_name%5B%5D=help-wanted) - tags for things that maintainers have marked as things they want to see - merged. - -* The amount of technical debt varies widely in this code base. There are some - parts where the code is nicely isolated with good test coverage. There are - other parts that are tangled and complicated, full of technical debt, and - difficult to test. - -* The general approach is to treat the tangled and complicated parts as an - external API (albeit a bad one). That means it needs to stay unchanged as much - as possible. Changes to those parts of the code will trigger a migration, - which can require a lot of time and coordination. When there is time for large - development efforts, we refactor the code to get rid of those areas of - technical debt. - -* We use [_black_](https://black.readthedocs.io/) code format, run `black .` to - format the code. Whenever editing code in any file, the new code should be - formatted as _black_. Some files are not yet fully in _black_ format (see - _pyproject.toml_), our goal is to opportunistically convert the code whenever - possible. As of the time of this writing, forcing the code format on all files - would be too disruptive. The officially supported _black_ version is the one - in Debian/stable. - -* Many of the tests run very fast and can be run interactively in isolation. - Some of the essential test cases run slowly because they do things like - signing files and generating signing keys. - -* Some parts of the code are difficult to test, and currently require a - relatively complete production setup in order to effectively test them. That - is mostly the code around building packages, managing the disposable VM, and - scheduling build jobs to run. - -* For user visible changes (API changes, behaviour changes, etc.), consider - adding a note in _CHANGELOG.md_. This could be a summarizing description of - the change, and could explain the grander details. Have a look through - existing entries for inspiration. Please note that this is NOT simply a copy - of git-log one-liners. Also note that security fixes get an entry in - _CHANGELOG.md_. This file helps users get more in-depth information of what - comes with a specific release without having to sift through the higher noise - ratio in git-log. diff --git a/MANIFEST.in b/MANIFEST.in index 93307ace..064d220e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,48 +9,35 @@ include buildserver/Vagrantfile include CHANGELOG.md include completion/bash-completion include examples/config.yml -include examples/fdroid_exportkeystore.py -include examples/fdroid_export_keystore_to_nitrokey.py -include examples/fdroid_extract_repo_pubkey.py -include examples/fdroid_fetchsrclibs.py -include examples/fdroid_nitrokeyimport.py +include examples/fdroid-icon.png +include examples/makebuildserver.config.py include examples/opensc-fdroid.cfg include examples/public-read-only-s3-bucket-policy.json include examples/template.yml -include examples/Vagrantfile.yaml +include fdroid include gradlew-fdroid include LICENSE -include locale/ba/LC_MESSAGES/fdroidserver.po -include locale/bo/LC_MESSAGES/fdroidserver.po -include locale/ca/LC_MESSAGES/fdroidserver.po -include locale/cs/LC_MESSAGES/fdroidserver.po -include locale/de/LC_MESSAGES/fdroidserver.po -include locale/es/LC_MESSAGES/fdroidserver.po -include locale/fr/LC_MESSAGES/fdroidserver.po -include locale/ga/LC_MESSAGES/fdroidserver.po -include locale/hu/LC_MESSAGES/fdroidserver.po -include locale/it/LC_MESSAGES/fdroidserver.po -include locale/ja/LC_MESSAGES/fdroidserver.po -include locale/ko/LC_MESSAGES/fdroidserver.po -include locale/nb_NO/LC_MESSAGES/fdroidserver.po -include locale/pl/LC_MESSAGES/fdroidserver.po -include locale/pt/LC_MESSAGES/fdroidserver.po -include locale/pt_BR/LC_MESSAGES/fdroidserver.po -include locale/pt_PT/LC_MESSAGES/fdroidserver.po -include locale/ro/LC_MESSAGES/fdroidserver.po -include locale/ru/LC_MESSAGES/fdroidserver.po -include locale/sq/LC_MESSAGES/fdroidserver.po -include locale/sr/LC_MESSAGES/fdroidserver.po -include locale/sw/LC_MESSAGES/fdroidserver.po -include locale/tr/LC_MESSAGES/fdroidserver.po -include locale/uk/LC_MESSAGES/fdroidserver.po -include locale/zh_Hans/LC_MESSAGES/fdroidserver.po -include locale/zh_Hant/LC_MESSAGES/fdroidserver.po +include locale/bo/LC_MESSAGES/fdroidserver.mo +include locale/de/LC_MESSAGES/fdroidserver.mo +include locale/es/LC_MESSAGES/fdroidserver.mo +include locale/fr/LC_MESSAGES/fdroidserver.mo +include locale/hu/LC_MESSAGES/fdroidserver.mo +include locale/it/LC_MESSAGES/fdroidserver.mo +include locale/ko/LC_MESSAGES/fdroidserver.mo +include locale/nb_NO/LC_MESSAGES/fdroidserver.mo +include locale/pl/LC_MESSAGES/fdroidserver.mo +include locale/pt_BR/LC_MESSAGES/fdroidserver.mo +include locale/pt_PT/LC_MESSAGES/fdroidserver.mo +include locale/ru/LC_MESSAGES/fdroidserver.mo +include locale/tr/LC_MESSAGES/fdroidserver.mo +include locale/uk/LC_MESSAGES/fdroidserver.mo +include locale/zh_Hans/LC_MESSAGES/fdroidserver.mo +include locale/zh_Hant/LC_MESSAGES/fdroidserver.mo include makebuildserver include README.md -include tests/aosp_testkey_debug.keystore -include tests/apk.embedded_1.apk +include tests/androguard_test.py include tests/bad-unicode-*.apk +include tests/build.TestCase include tests/build-tools/17.0.0/aapt-output-com.moez.QKSMS_182.txt include tests/build-tools/17.0.0/aapt-output-com.politedroid_3.txt include tests/build-tools/17.0.0/aapt-output-com.politedroid_4.txt @@ -60,10 +47,10 @@ include tests/build-tools/17.0.0/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/17.0.0/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/17.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/17.0.0/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/17.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/17.0.0/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/17.0.0/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/17.0.0/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/17.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/17.0.0/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/17.0.0/aapt-output-souch.smsbypass_9.txt include tests/build-tools/18.1.1/aapt-output-com.moez.QKSMS_182.txt @@ -75,10 +62,10 @@ include tests/build-tools/18.1.1/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/18.1.1/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/18.1.1/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/18.1.1/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/18.1.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/18.1.1/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/18.1.1/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/18.1.1/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/18.1.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/18.1.1/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/18.1.1/aapt-output-souch.smsbypass_9.txt include tests/build-tools/19.0.0/aapt-output-com.moez.QKSMS_182.txt @@ -90,10 +77,10 @@ include tests/build-tools/19.0.0/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/19.0.0/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/19.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/19.0.0/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/19.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/19.0.0/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/19.0.0/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/19.0.0/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/19.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/19.0.0/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/19.0.0/aapt-output-souch.smsbypass_9.txt include tests/build-tools/19.1.0/aapt-output-com.moez.QKSMS_182.txt @@ -105,10 +92,10 @@ include tests/build-tools/19.1.0/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/19.1.0/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/19.1.0/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/19.1.0/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/19.1.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/19.1.0/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/19.1.0/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/19.1.0/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/19.1.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/19.1.0/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/19.1.0/aapt-output-souch.smsbypass_9.txt include tests/build-tools/20.0.0/aapt-output-com.moez.QKSMS_182.txt @@ -120,10 +107,10 @@ include tests/build-tools/20.0.0/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/20.0.0/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/20.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/20.0.0/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/20.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/20.0.0/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/20.0.0/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/20.0.0/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/20.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/20.0.0/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/20.0.0/aapt-output-souch.smsbypass_9.txt include tests/build-tools/21.1.1/aapt-output-com.moez.QKSMS_182.txt @@ -135,10 +122,10 @@ include tests/build-tools/21.1.1/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/21.1.1/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/21.1.1/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/21.1.1/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/21.1.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/21.1.1/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/21.1.1/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/21.1.1/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/21.1.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/21.1.1/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/21.1.1/aapt-output-souch.smsbypass_9.txt include tests/build-tools/21.1.2/aapt-output-com.moez.QKSMS_182.txt @@ -150,10 +137,10 @@ include tests/build-tools/21.1.2/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/21.1.2/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/21.1.2/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/21.1.2/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/21.1.2/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/21.1.2/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/21.1.2/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/21.1.2/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/21.1.2/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/21.1.2/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/21.1.2/aapt-output-souch.smsbypass_9.txt include tests/build-tools/22.0.0/aapt-output-com.moez.QKSMS_182.txt @@ -165,10 +152,10 @@ include tests/build-tools/22.0.0/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/22.0.0/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/22.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/22.0.0/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/22.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/22.0.0/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/22.0.0/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/22.0.0/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/22.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/22.0.0/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/22.0.0/aapt-output-souch.smsbypass_9.txt include tests/build-tools/22.0.1/aapt-output-com.moez.QKSMS_182.txt @@ -180,10 +167,10 @@ include tests/build-tools/22.0.1/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/22.0.1/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/22.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/22.0.1/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/22.0.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/22.0.1/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/22.0.1/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/22.0.1/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/22.0.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/22.0.1/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/22.0.1/aapt-output-souch.smsbypass_9.txt include tests/build-tools/23.0.0/aapt-output-com.moez.QKSMS_182.txt @@ -195,10 +182,10 @@ include tests/build-tools/23.0.0/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/23.0.0/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/23.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/23.0.0/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/23.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/23.0.0/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/23.0.0/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/23.0.0/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/23.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/23.0.0/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/23.0.0/aapt-output-souch.smsbypass_9.txt include tests/build-tools/23.0.1/aapt-output-com.moez.QKSMS_182.txt @@ -210,10 +197,10 @@ include tests/build-tools/23.0.1/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/23.0.1/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/23.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/23.0.1/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/23.0.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/23.0.1/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/23.0.1/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/23.0.1/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/23.0.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/23.0.1/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/23.0.1/aapt-output-souch.smsbypass_9.txt include tests/build-tools/23.0.2/aapt-output-com.moez.QKSMS_182.txt @@ -225,10 +212,10 @@ include tests/build-tools/23.0.2/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/23.0.2/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/23.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/23.0.2/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/23.0.2/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/23.0.2/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/23.0.2/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/23.0.2/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/23.0.2/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/23.0.2/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/23.0.2/aapt-output-souch.smsbypass_9.txt include tests/build-tools/23.0.3/aapt-output-com.moez.QKSMS_182.txt @@ -240,10 +227,10 @@ include tests/build-tools/23.0.3/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/23.0.3/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/23.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/23.0.3/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/23.0.3/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/23.0.3/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/23.0.3/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/23.0.3/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/23.0.3/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/23.0.3/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/23.0.3/aapt-output-souch.smsbypass_9.txt include tests/build-tools/24.0.0/aapt-output-com.moez.QKSMS_182.txt @@ -255,10 +242,10 @@ include tests/build-tools/24.0.0/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/24.0.0/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/24.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/24.0.0/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/24.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/24.0.0/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/24.0.0/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/24.0.0/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/24.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/24.0.0/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/24.0.0/aapt-output-souch.smsbypass_9.txt include tests/build-tools/24.0.1/aapt-output-com.moez.QKSMS_182.txt @@ -270,10 +257,10 @@ include tests/build-tools/24.0.1/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/24.0.1/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/24.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/24.0.1/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/24.0.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/24.0.1/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/24.0.1/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/24.0.1/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/24.0.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/24.0.1/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/24.0.1/aapt-output-souch.smsbypass_9.txt include tests/build-tools/24.0.2/aapt-output-com.moez.QKSMS_182.txt @@ -285,10 +272,10 @@ include tests/build-tools/24.0.2/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/24.0.2/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/24.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/24.0.2/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/24.0.2/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/24.0.2/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/24.0.2/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/24.0.2/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/24.0.2/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/24.0.2/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/24.0.2/aapt-output-souch.smsbypass_9.txt include tests/build-tools/24.0.3/aapt-output-com.moez.QKSMS_182.txt @@ -300,10 +287,10 @@ include tests/build-tools/24.0.3/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/24.0.3/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/24.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/24.0.3/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/24.0.3/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/24.0.3/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/24.0.3/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/24.0.3/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/24.0.3/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/24.0.3/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/24.0.3/aapt-output-souch.smsbypass_9.txt include tests/build-tools/25.0.0/aapt-output-com.moez.QKSMS_182.txt @@ -315,10 +302,10 @@ include tests/build-tools/25.0.0/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/25.0.0/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/25.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/25.0.0/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/25.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/25.0.0/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/25.0.0/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/25.0.0/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/25.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/25.0.0/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/25.0.0/aapt-output-souch.smsbypass_9.txt include tests/build-tools/25.0.1/aapt-output-com.moez.QKSMS_182.txt @@ -330,10 +317,10 @@ include tests/build-tools/25.0.1/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/25.0.1/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/25.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/25.0.1/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/25.0.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/25.0.1/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/25.0.1/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/25.0.1/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/25.0.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/25.0.1/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/25.0.1/aapt-output-souch.smsbypass_9.txt include tests/build-tools/25.0.2/aapt-output-com.moez.QKSMS_182.txt @@ -345,10 +332,10 @@ include tests/build-tools/25.0.2/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/25.0.2/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/25.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/25.0.2/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/25.0.2/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/25.0.2/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/25.0.2/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/25.0.2/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/25.0.2/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/25.0.2/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/25.0.2/aapt-output-souch.smsbypass_9.txt include tests/build-tools/25.0.3/aapt-output-com.moez.QKSMS_182.txt @@ -360,10 +347,10 @@ include tests/build-tools/25.0.3/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/25.0.3/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/25.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/25.0.3/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/25.0.3/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/25.0.3/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/25.0.3/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/25.0.3/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/25.0.3/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/25.0.3/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/25.0.3/aapt-output-souch.smsbypass_9.txt include tests/build-tools/26.0.0/aapt-output-com.moez.QKSMS_182.txt @@ -375,10 +362,10 @@ include tests/build-tools/26.0.0/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/26.0.0/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/26.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/26.0.0/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/26.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/26.0.0/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/26.0.0/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/26.0.0/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/26.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/26.0.0/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/26.0.0/aapt-output-souch.smsbypass_9.txt include tests/build-tools/26.0.1/aapt-output-com.moez.QKSMS_182.txt @@ -390,10 +377,10 @@ include tests/build-tools/26.0.1/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/26.0.1/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/26.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/26.0.1/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/26.0.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/26.0.1/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/26.0.1/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/26.0.1/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/26.0.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/26.0.1/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/26.0.1/aapt-output-souch.smsbypass_9.txt include tests/build-tools/26.0.2/aapt-output-com.moez.QKSMS_182.txt @@ -405,10 +392,10 @@ include tests/build-tools/26.0.2/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/26.0.2/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/26.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/26.0.2/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/26.0.2/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/26.0.2/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/26.0.2/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/26.0.2/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/26.0.2/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/26.0.2/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/26.0.2/aapt-output-souch.smsbypass_9.txt include tests/build-tools/26.0.3/aapt-output-com.moez.QKSMS_182.txt @@ -420,10 +407,10 @@ include tests/build-tools/26.0.3/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/26.0.3/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/26.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/26.0.3/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/26.0.3/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/26.0.3/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/26.0.3/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/26.0.3/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/26.0.3/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/26.0.3/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/26.0.3/aapt-output-souch.smsbypass_9.txt include tests/build-tools/27.0.0/aapt-output-com.moez.QKSMS_182.txt @@ -435,10 +422,10 @@ include tests/build-tools/27.0.0/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/27.0.0/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/27.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/27.0.0/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/27.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/27.0.0/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/27.0.0/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/27.0.0/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/27.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/27.0.0/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/27.0.0/aapt-output-souch.smsbypass_9.txt include tests/build-tools/27.0.1/aapt-output-com.moez.QKSMS_182.txt @@ -450,10 +437,10 @@ include tests/build-tools/27.0.1/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/27.0.1/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/27.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/27.0.1/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/27.0.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/27.0.1/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/27.0.1/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/27.0.1/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/27.0.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/27.0.1/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/27.0.1/aapt-output-souch.smsbypass_9.txt include tests/build-tools/27.0.2/aapt-output-com.moez.QKSMS_182.txt @@ -465,10 +452,10 @@ include tests/build-tools/27.0.2/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/27.0.2/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/27.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/27.0.2/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/27.0.2/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/27.0.2/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/27.0.2/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/27.0.2/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/27.0.2/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/27.0.2/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/27.0.2/aapt-output-souch.smsbypass_9.txt include tests/build-tools/27.0.3/aapt-output-com.moez.QKSMS_182.txt @@ -480,10 +467,10 @@ include tests/build-tools/27.0.3/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/27.0.3/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/27.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/27.0.3/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/27.0.3/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/27.0.3/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/27.0.3/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/27.0.3/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/27.0.3/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/27.0.3/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/27.0.3/aapt-output-souch.smsbypass_9.txt include tests/build-tools/28.0.0/aapt-output-com.moez.QKSMS_182.txt @@ -495,10 +482,10 @@ include tests/build-tools/28.0.0/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/28.0.0/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/28.0.0/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/28.0.0/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/28.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/28.0.0/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/28.0.0/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/28.0.0/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/28.0.0/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/28.0.0/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/28.0.0/aapt-output-souch.smsbypass_9.txt include tests/build-tools/28.0.1/aapt-output-com.moez.QKSMS_182.txt @@ -510,10 +497,10 @@ include tests/build-tools/28.0.1/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/28.0.1/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/28.0.1/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/28.0.1/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/28.0.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/28.0.1/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/28.0.1/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/28.0.1/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/28.0.1/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/28.0.1/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/28.0.1/aapt-output-souch.smsbypass_9.txt include tests/build-tools/28.0.2/aapt-output-com.politedroid_3.txt @@ -524,10 +511,10 @@ include tests/build-tools/28.0.2/aapt-output-duplicate.permisssions_9999999.txt include tests/build-tools/28.0.2/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/28.0.2/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/28.0.2/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/28.0.2/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/28.0.2/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/28.0.2/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/28.0.2/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/28.0.2/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/28.0.2/aapt-output-org.droidtr.keyboard_34.txt include tests/build-tools/28.0.2/aapt-output-souch.smsbypass_9.txt include tests/build-tools/28.0.3/aapt-output-com.example.test.helloworld_1.txt @@ -540,119 +527,85 @@ include tests/build-tools/28.0.3/aapt-output-info.guardianproject.urzip_100.txt include tests/build-tools/28.0.3/aapt-output-info.zwanenburg.caffeinetile_4.txt include tests/build-tools/28.0.3/aapt-output-no.min.target.sdk_987.txt include tests/build-tools/28.0.3/aapt-output-obb.main.oldversion_1444412523.txt +include tests/build-tools/28.0.3/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/28.0.3/aapt-output-obb.main.twoversions_1101613.txt include tests/build-tools/28.0.3/aapt-output-obb.main.twoversions_1101615.txt include tests/build-tools/28.0.3/aapt-output-obb.main.twoversions_1101617.txt -include tests/build-tools/28.0.3/aapt-output-obb.mainpatch.current_1619.txt include tests/build-tools/28.0.3/aapt-output-souch.smsbypass_9.txt include tests/build-tools/generate.sh include tests/check-fdroid-apk -include tests/com.fake.IpaApp_1000000000001.ipa -include tests/config.yml -include tests/config/antiFeatures.yml -include tests/config/categories.yml -include tests/config/de/antiFeatures.yml -include tests/config/fa/antiFeatures.yml -include tests/config/ic_antifeature_ads.xml -include tests/config/ic_antifeature_disabledalgorithm.xml -include tests/config/ic_antifeature_knownvuln.xml -include tests/config/ic_antifeature_nonfreeadd.xml -include tests/config/ic_antifeature_nonfreeassets.xml -include tests/config/ic_antifeature_nonfreedep.xml -include tests/config/ic_antifeature_nonfreenet.xml -include tests/config/ic_antifeature_nosourcesince.xml -include tests/config/ic_antifeature_nsfw.xml -include tests/config/ic_antifeature_tracking.xml -include tests/config/ic_antifeature_upstreamnonfree.xml -include tests/config/ro/antiFeatures.yml -include tests/config/zh-rCN/antiFeatures.yml -include tests/corrupt-featureGraphic.png +include tests/common.TestCase +include tests/complete-ci-tests +include tests/config.py +include tests/deploy.TestCase +include tests/description-parsing.py include tests/dummy-keystore.jks include tests/dump_internal_metadata_format.py +include tests/exception.TestCase include tests/extra/manual-vmtools-test.py include tests/funding-usernames.yaml -include tests/get_android_tools_versions/android-ndk-r10e/RELEASE.TXT -include tests/get_android_tools_versions/android-sdk/ndk-bundle/package.xml -include tests/get_android_tools_versions/android-sdk/ndk-bundle/source.properties -include tests/get_android_tools_versions/android-sdk/ndk/11.2.2725575/source.properties -include tests/get_android_tools_versions/android-sdk/ndk/17.2.4988734/source.properties -include tests/get_android_tools_versions/android-sdk/ndk/21.3.6528147/source.properties -include tests/get_android_tools_versions/android-sdk/patcher/v4/source.properties -include tests/get_android_tools_versions/android-sdk/platforms/android-30/source.properties -include tests/get_android_tools_versions/android-sdk/skiaparser/1/source.properties -include tests/get_android_tools_versions/android-sdk/tools/source.properties +include tests/getsig/getsig.java +include tests/getsig/make.sh +include tests/getsig/run.sh include tests/gnupghome/pubring.gpg include tests/gnupghome/random_seed include tests/gnupghome/secring.gpg include tests/gnupghome/trustdb.gpg include tests/gradle-maven-blocks.yaml -include tests/gradle-release-checksums.py +include tests/import_proxy.py +include tests/import.TestCase +include tests/index.TestCase +include tests/install.TestCase include tests/IsMD5Disabled.java -include tests/issue-1128-min-sdk-30-poc.apk -include tests/issue-1128-poc1.apk -include tests/issue-1128-poc2.apk -include tests/issue-1128-poc3a.apk -include tests/issue-1128-poc3b.apk include tests/janus.apk -include tests/key-tricks.py include tests/keystore.jks -include tests/metadata-rewrite-yml/app.with.special.build.params.yml -include tests/metadata-rewrite-yml/fake.ota.update.yml -include tests/metadata-rewrite-yml/org.fdroid.fdroid.yml +include tests/lint.TestCase include tests/metadata/apk/info.guardianproject.urzip.yaml include tests/metadata/apk/org.dyndns.fules.ck.yaml include tests/metadata/app.with.special.build.params.yml -include tests/metadata/app.with.special.build.params/en-US/antifeatures/50_Ads.txt -include tests/metadata/app.with.special.build.params/en-US/antifeatures/50_Tracking.txt -include tests/metadata/app.with.special.build.params/en-US/antifeatures/Ads.txt -include tests/metadata/app.with.special.build.params/en-US/antifeatures/NoSourceSince.txt -include tests/metadata/app.with.special.build.params/zh-CN/antifeatures/49_Tracking.txt -include tests/metadata/app.with.special.build.params/zh-CN/antifeatures/50_Ads.txt include tests/metadata/com.politedroid.yml -include tests/metadata/dump/app.with.special.build.params.yaml include tests/metadata/dump/com.politedroid.yaml include tests/metadata/dump/org.adaway.yaml include tests/metadata/dump/org.smssecure.smssecure.yaml include tests/metadata/dump/org.videolan.vlc.yaml include tests/metadata/duplicate.permisssions.yml include tests/metadata/fake.ota.update.yml -include tests/metadata/info.guardianproject.checkey.yml include tests/metadata/info.guardianproject.checkey/en-US/description.txt -include tests/metadata/info.guardianproject.checkey/en-US/name.txt include tests/metadata/info.guardianproject.checkey/en-US/phoneScreenshots/checkey-phone.png include tests/metadata/info.guardianproject.checkey/en-US/phoneScreenshots/checkey.png include tests/metadata/info.guardianproject.checkey/en-US/summary.txt -include tests/metadata/info.guardianproject.checkey/ja-JP/name.txt -include tests/metadata/info.guardianproject.urzip.yml +include tests/metadata/info.guardianproject.checkey.yml include tests/metadata/info.guardianproject.urzip/en-US/changelogs/100.txt -include tests/metadata/info.guardianproject.urzip/en-US/changelogs/default.txt include tests/metadata/info.guardianproject.urzip/en-US/full_description.txt include tests/metadata/info.guardianproject.urzip/en-US/images/featureGraphic.png include tests/metadata/info.guardianproject.urzip/en-US/images/icon.png include tests/metadata/info.guardianproject.urzip/en-US/short_description.txt include tests/metadata/info.guardianproject.urzip/en-US/title.txt include tests/metadata/info.guardianproject.urzip/en-US/video.txt +include tests/metadata/info.guardianproject.urzip.yml include tests/metadata/info.zwanenburg.caffeinetile.yml include tests/metadata/no.min.target.sdk.yml include tests/metadata/obb.main.oldversion.yml -include tests/metadata/obb.main.twoversions.yml include tests/metadata/obb.mainpatch.current.yml +include tests/metadata/obb.main.twoversions.yml include tests/metadata/org.adaway.yml include tests/metadata/org.fdroid.ci.test.app.yml include tests/metadata/org.fdroid.fdroid.yml -include tests/metadata/org.maxsdkversion.yml -include tests/metadata/org.smssecure.smssecure.yml include tests/metadata/org.smssecure.smssecure/signatures/134/28969C09.RSA include tests/metadata/org.smssecure.smssecure/signatures/134/28969C09.SF include tests/metadata/org.smssecure.smssecure/signatures/134/MANIFEST.MF include tests/metadata/org.smssecure.smssecure/signatures/135/28969C09.RSA include tests/metadata/org.smssecure.smssecure/signatures/135/28969C09.SF include tests/metadata/org.smssecure.smssecure/signatures/135/MANIFEST.MF +include tests/metadata/org.smssecure.smssecure.yml include tests/metadata/org.videolan.vlc.yml include tests/metadata/raw.template.yml +include tests/metadata-rewrite-yml/app.with.special.build.params.yml +include tests/metadata-rewrite-yml/fake.ota.update.yml +include tests/metadata-rewrite-yml/org.fdroid.fdroid.yml include tests/metadata/souch.smsbypass.yml +include tests/metadata.TestCase include tests/minimal_targetsdk_30_unsigned.apk -include tests/Norway_bouvet_europe_2.obf.zip include tests/no_targetsdk_minsdk1_unsigned.apk include tests/no_targetsdk_minsdk30_unsigned.apk include tests/openssl-version-check-test.py @@ -661,17 +614,16 @@ include tests/org.bitbucket.tickytacky.mirrormirror_2.apk include tests/org.bitbucket.tickytacky.mirrormirror_3.apk include tests/org.bitbucket.tickytacky.mirrormirror_4.apk include tests/org.dyndns.fules.ck_20.apk -include tests/org.sajeg.fallingblocks_3.apk +include tests/publish.TestCase +include tests/repo/categories.txt include tests/repo/com.example.test.helloworld_1.apk include tests/repo/com.politedroid_3.apk include tests/repo/com.politedroid_4.apk include tests/repo/com.politedroid_5.apk include tests/repo/com.politedroid_6.apk include tests/repo/duplicate.permisssions_9999999.apk -include tests/repo/entry.json include tests/repo/fake.ota.update_1234.zip include tests/repo/index-v1.json -include tests/repo/index-v2.json include tests/repo/index.xml include tests/repo/info.zwanenburg.caffeinetile_4.apk include tests/repo/main.1101613.obb.main.twoversions.obb @@ -680,17 +632,16 @@ include tests/repo/main.1434483388.obb.main.oldversion.obb include tests/repo/main.1619.obb.mainpatch.current.obb include tests/repo/no.min.target.sdk_987.apk include tests/repo/obb.main.oldversion_1444412523.apk -include tests/repo/obb.main.twoversions_1101613.apk -include tests/repo/obb.main.twoversions_1101615.apk -include tests/repo/obb.main.twoversions_1101617.apk -include tests/repo/obb.main.twoversions_1101617_src.tar.gz +include tests/repo/obb.mainpatch.current_1619_another-release-key.apk +include tests/repo/obb.mainpatch.current_1619.apk include tests/repo/obb.mainpatch.current/en-US/featureGraphic.png include tests/repo/obb.mainpatch.current/en-US/icon.png include tests/repo/obb.mainpatch.current/en-US/phoneScreenshots/screenshot-main.png include tests/repo/obb.mainpatch.current/en-US/sevenInchScreenshots/screenshot-tablet-main.png -include tests/repo/obb.mainpatch.current_1619.apk -include tests/repo/obb.mainpatch.current_1619_another-release-key.apk -include tests/repo/org.maxsdkversion_4.apk +include tests/repo/obb.main.twoversions_1101613.apk +include tests/repo/obb.main.twoversions_1101615.apk +include tests/repo/obb.main.twoversions_1101617.apk +include tests/repo/obb.main.twoversions_1101617_src.tar.gz include tests/repo/org.videolan.vlc/en-US/icon.png include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot10.png include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot12.png @@ -702,16 +653,16 @@ include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot4.png include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot7.png include tests/repo/org.videolan.vlc/en-US/phoneScreenshots/screenshot9.png include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot0.png -include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot1.png include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot11.png include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot13.png include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot14.png include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot16.png include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot17.png include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot19.png -include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot2.png +include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot1.png include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot21.png include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot23.png +include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot2.png include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot3.png include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot5.png include tests/repo/org.videolan.vlc/en-US/sevenInchScreenshots/screenshot6.png @@ -721,24 +672,13 @@ include tests/repo/souch.smsbypass_9.apk include tests/repo/urzip-*.apk include tests/repo/v1.v2.sig_1020.apk include tests/run-tests -include tests/SANAPPSI.RSA -include tests/SANAPPSI.SF -include tests/shared_test_code.py -include tests/signindex/guardianproject-v1.jar +include tests/scanner.TestCase +include tests/signatures.TestCase include tests/signindex/guardianproject.jar +include tests/signindex/guardianproject-v1.jar include tests/signindex/testy.jar include tests/signindex/unsigned.jar include tests/source-files/at.bitfire.davdroid/build.gradle -include tests/source-files/catalog.test/app/build.gradle -include tests/source-files/catalog.test/build.gradle.kts -include tests/source-files/catalog.test/buildSrc/build.gradle.kts -include tests/source-files/catalog.test/buildSrc/settings.gradle.kts -include tests/source-files/catalog.test/buildSrc2/build.gradle.kts -include tests/source-files/catalog.test/buildSrc2/settings.gradle.kts -include tests/source-files/catalog.test/core/build.gradle -include tests/source-files/catalog.test/gradle/libs.versions.toml -include tests/source-files/catalog.test/libs.versions.toml -include tests/source-files/catalog.test/settings.gradle.kts include tests/source-files/cn.wildfirechat.chat/avenginekit/build.gradle include tests/source-files/cn.wildfirechat.chat/build.gradle include tests/source-files/cn.wildfirechat.chat/chat/build.gradle @@ -754,12 +694,6 @@ include tests/source-files/com.anpmech.launcher/app/build.gradle include tests/source-files/com.anpmech.launcher/app/src/main/AndroidManifest.xml include tests/source-files/com.anpmech.launcher/build.gradle include tests/source-files/com.anpmech.launcher/settings.gradle -include tests/source-files/com.github.jameshnsears.quoteunquote/build.gradle -include tests/source-files/com.github.shadowsocks/core/build.gradle.kts -include tests/source-files/com.github.shadowsocks/mobile/build.gradle.kts -include tests/source-files/com.infomaniak.mail/Core/gradle/core.versions.toml -include tests/source-files/com.infomaniak.mail/gradle/libs.versions.toml -include tests/source-files/com.infomaniak.mail/settings.gradle include tests/source-files/com.integreight.onesheeld/build.gradle include tests/source-files/com.integreight.onesheeld/gradle/wrapper/gradle-wrapper.properties include tests/source-files/com.integreight.onesheeld/localeapi/build.gradle @@ -773,69 +707,37 @@ include tests/source-files/com.integreight.onesheeld/pullToRefreshlibrary/src/ma include tests/source-files/com.integreight.onesheeld/quickReturnHeader/build.gradle include tests/source-files/com.integreight.onesheeld/quickReturnHeader/src/main/AndroidManifest.xml include tests/source-files/com.integreight.onesheeld/settings.gradle -include tests/source-files/com.jens.automation2/app/build.gradle -include tests/source-files/com.jens.automation2/build.gradle include tests/source-files/com.kunzisoft.testcase/build.gradle -include tests/source-files/com.lolo.io.onelist/app/build.gradle.kts -include tests/source-files/com.lolo.io.onelist/build.gradle.kts -include tests/source-files/com.lolo.io.onelist/gradle/libs.versions.toml -include tests/source-files/com.lolo.io.onelist/gradle/wrapper/gradle-wrapper.properties -include tests/source-files/com.lolo.io.onelist/settings.gradle +include tests/source-files/com.nextcloud.client/build.gradle include tests/source-files/com.nextcloud.client.dev/src/generic/fastlane/metadata/android/en-US/full_description.txt include tests/source-files/com.nextcloud.client.dev/src/generic/fastlane/metadata/android/en-US/short_description.txt include tests/source-files/com.nextcloud.client.dev/src/generic/fastlane/metadata/android/en-US/title.txt include tests/source-files/com.nextcloud.client.dev/src/versionDev/fastlane/metadata/android/en-US/full_description.txt include tests/source-files/com.nextcloud.client.dev/src/versionDev/fastlane/metadata/android/en-US/short_description.txt include tests/source-files/com.nextcloud.client.dev/src/versionDev/fastlane/metadata/android/en-US/title.txt -include tests/source-files/com.nextcloud.client/build.gradle include tests/source-files/com.nextcloud.client/src/generic/fastlane/metadata/android/en-US/full_description.txt include tests/source-files/com.nextcloud.client/src/generic/fastlane/metadata/android/en-US/short_description.txt include tests/source-files/com.nextcloud.client/src/generic/fastlane/metadata/android/en-US/title.txt include tests/source-files/com.nextcloud.client/src/versionDev/fastlane/metadata/android/en-US/full_description.txt include tests/source-files/com.nextcloud.client/src/versionDev/fastlane/metadata/android/en-US/short_description.txt include tests/source-files/com.nextcloud.client/src/versionDev/fastlane/metadata/android/en-US/title.txt -include tests/source-files/com.seafile.seadroid2/app/build.gradle -include tests/source-files/com.ubergeek42.WeechatAndroid/app/build.gradle.kts -include tests/source-files/com.ubergeek42.WeechatAndroid/app/src/main/res/values/strings.xml -include tests/source-files/de.varengold.activeTAN/build.gradle -include tests/source-files/dev.patrickgold.florisboard/app/build.gradle.kts include tests/source-files/eu.siacs.conversations/build.gradle include tests/source-files/eu.siacs.conversations/metadata/en-US/name.txt include tests/source-files/fdroid/fdroidclient/AndroidManifest.xml include tests/source-files/fdroid/fdroidclient/build.gradle -include tests/source-files/firebase-allowlisted/app/build.gradle -include tests/source-files/firebase-allowlisted/build.gradle include tests/source-files/firebase-suspect/app/build.gradle include tests/source-files/firebase-suspect/build.gradle -include tests/source-files/flavor.test/build.gradle -include tests/source-files/info.guardianproject.ripple/build.gradle -include tests/source-files/lockfile.test/flutter/.dart_tool/flutter_gen/pubspec.yaml -include tests/source-files/lockfile.test/flutter/pubspec.lock -include tests/source-files/lockfile.test/flutter/pubspec.yaml -include tests/source-files/lockfile.test/javascript/package.json -include tests/source-files/lockfile.test/javascript/yarn.lock -include tests/source-files/lockfile.test/rust/subdir/Cargo.lock -include tests/source-files/lockfile.test/rust/subdir/Cargo.toml -include tests/source-files/lockfile.test/rust/subdir/subdir/subdir/Cargo.toml -include tests/source-files/lockfile.test/rust/subdir2/Cargo.toml +include tests/source-files/firebase-whitelisted/app/build.gradle +include tests/source-files/firebase-whitelisted/build.gradle include tests/source-files/open-keychain/open-keychain/build.gradle include tests/source-files/open-keychain/open-keychain/OpenKeychain/build.gradle include tests/source-files/org.mozilla.rocket/app/build.gradle -include tests/source-files/org.noise_planet.noisecapture/app/build.gradle -include tests/source-files/org.noise_planet.noisecapture/settings.gradle -include tests/source-files/org.noise_planet.noisecapture/sosfilter/build.gradle -include tests/source-files/org.piepmeyer.gauguin/build.gradle.kts -include tests/source-files/org.piepmeyer.gauguin/libs.versions.toml -include tests/source-files/org.piepmeyer.gauguin/settings.gradle.kts include tests/source-files/org.tasks/app/build.gradle.kts include tests/source-files/org.tasks/build.gradle include tests/source-files/org.tasks/build.gradle.kts include tests/source-files/org.tasks/buildSrc/build.gradle.kts include tests/source-files/org.tasks/settings.gradle.kts include tests/source-files/osmandapp/osmand/build.gradle -include tests/source-files/osmandapp/osmand/gradle/wrapper/gradle-wrapper.properties -include tests/source-files/OtakuWorld/build.gradle -include tests/source-files/realm/react-native/android/build.gradle include tests/source-files/se.manyver/android/app/build.gradle include tests/source-files/se.manyver/android/build.gradle include tests/source-files/se.manyver/android/gradle.properties @@ -849,36 +751,12 @@ include tests/source-files/ut.ewh.audiometrytest/app/build.gradle include tests/source-files/ut.ewh.audiometrytest/app/src/main/AndroidManifest.xml include tests/source-files/ut.ewh.audiometrytest/build.gradle include tests/source-files/ut.ewh.audiometrytest/settings.gradle -include tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties include tests/source-files/Zillode/syncthing-silk/build.gradle include tests/SpeedoMeterApp.main_1.apk -include tests/test_build.py -include tests/test_checkupdates.py -include tests/test_common.py -include tests/test_deploy.py -include tests/test_exception.py -include tests/test_gradlew-fdroid -include tests/test_import_subcommand.py -include tests/test_index.py -include tests/test_init.py -include tests/test_install.py -include tests/test_lint.py -include tests/test_main.py -include tests/test_metadata.py -include tests/test_nightly.py -include tests/test_publish.py -include tests/test_rewritemeta.py -include tests/test_scanner.py -include tests/test_signatures.py -include tests/test_signindex.py -include tests/test_update.py -include tests/test_vcs.py -include tests/triple-t-1-graphics/build/de.wivewa.dialer/app/src/main/play/en-US/listing/featureGraphic/play_store_feature_graphic.png -include tests/triple-t-1-graphics/build/de.wivewa.dialer/app/src/main/play/en-US/listing/icon/icon.png -include tests/triple-t-1-graphics/build/de.wivewa.dialer/app/src/main/play/en-US/listing/phoneScreenshots/1.png -include tests/triple-t-1-graphics/metadata/de.wivewa.dialer.yml -include tests/triple-t-2/build/org.piwigo.android/app/.gitignore +include tests/stats/known_apks.txt +include tests/testcommon.py include tests/triple-t-2/build/org.piwigo.android/app/build.gradle +include tests/triple-t-2/build/org.piwigo.android/app/.gitignore include tests/triple-t-2/build/org.piwigo.android/app/src/debug/res/values/constants.xml include tests/triple-t-2/build/org.piwigo.android/app/src/debug/res/values/strings.xml include tests/triple-t-2/build/org.piwigo.android/app/src/main/java/org/piwigo/PiwigoApplication.java @@ -912,34 +790,17 @@ include tests/triple-t-2/build/org.piwigo.android/app/src/main/play/release-note include tests/triple-t-2/build/org.piwigo.android/build.gradle include tests/triple-t-2/build/org.piwigo.android/settings.gradle include tests/triple-t-2/metadata/org.piwigo.android.yml -include tests/triple-t-anysoftkeyboard/build/com.anysoftkeyboard.languagepack.dutch/addons/languages/dutch/apk/src/main/play/listings/en-US/title.txt -include tests/triple-t-anysoftkeyboard/build/com.anysoftkeyboard.languagepack.dutch/ime/app/src/main/play/listings/en-US/title.txt -include tests/triple-t-anysoftkeyboard/build/com.anysoftkeyboard.languagepack.dutch/settings.gradle -include tests/triple-t-anysoftkeyboard/build/com.menny.android.anysoftkeyboard/addons/languages/dutch/apk/src/main/play/listings/en-US/title.txt -include tests/triple-t-anysoftkeyboard/build/com.menny.android.anysoftkeyboard/ime/app/src/main/play/listings/en-US/title.txt -include tests/triple-t-anysoftkeyboard/build/com.menny.android.anysoftkeyboard/settings.gradle -include tests/triple-t-anysoftkeyboard/metadata/com.anysoftkeyboard.languagepack.dutch.yml -include tests/triple-t-anysoftkeyboard/metadata/com.menny.android.anysoftkeyboard.yml -include tests/triple-t-flutter/build/fr.emersion.goguma/android/app/src/main/play/contact-website.txt -include tests/triple-t-flutter/build/fr.emersion.goguma/android/app/src/main/play/listings/en-US/full-description.txt -include tests/triple-t-flutter/build/fr.emersion.goguma/android/app/src/main/play/listings/en-US/short-description.txt -include tests/triple-t-flutter/build/fr.emersion.goguma/android/app/src/main/play/listings/en-US/title.txt -include tests/triple-t-flutter/metadata/fr.emersion.goguma.yml -include tests/triple-t-multiple/build/ch.admin.bag.covidcertificate.verifier/settings.gradle -include tests/triple-t-multiple/build/ch.admin.bag.covidcertificate.verifier/verifier/src/main/play/listings/en-US/title.txt -include tests/triple-t-multiple/build/ch.admin.bag.covidcertificate.verifier/wallet/src/main/play/listings/en-US/title.txt -include tests/triple-t-multiple/build/ch.admin.bag.covidcertificate.wallet/settings.gradle -include tests/triple-t-multiple/build/ch.admin.bag.covidcertificate.wallet/verifier/src/main/play/listings/en-US/title.txt -include tests/triple-t-multiple/build/ch.admin.bag.covidcertificate.wallet/wallet/src/main/play/listings/en-US/title.txt -include tests/triple-t-multiple/metadata/ch.admin.bag.covidcertificate.verifier.yml -include tests/triple-t-multiple/metadata/ch.admin.bag.covidcertificate.wallet.yml +include tests/update.TestCase +include tests/urzip.apk include tests/urzip-badcert.apk include tests/urzip-badsig.apk -include tests/urzip-release-unsigned.apk include tests/urzip-release.apk -include tests/urzip.apk +include tests/urzip-release-unsigned.apk include tests/v2.only.sig_2.apk include tests/valid-package-names/random-package-names include tests/valid-package-names/RandomPackageNames.java include tests/valid-package-names/test.py -include tests/__init__.py +include tests/Norway_bouvet_europe_2.obf.zip +include tests/xref/metadata/aarddict.android.yml +include tests/xref/metadata/org.coolreader.yml +include tests/xref/metadata/org.geometerplus.zlibrary.ui.android.yml diff --git a/README.md b/README.md index 41f725cb..dac49d72 100644 --- a/README.md +++ b/README.md @@ -1,116 +1,88 @@ -
+ + +| CI Builds | fdroidserver | buildserver | fdroid build --all | publishing tools | +|--------------------------|:-------------:|:-----------:|:------------------:|:----------------:| +| GNU/Linux | [![fdroidserver status on GNU/Linux](https://gitlab.com/fdroid/fdroidserver/badges/master/build.svg)](https://gitlab.com/fdroid/fdroidserver/builds) | [![buildserver status](https://jenkins.debian.net/job/reproducible_setup_fdroid_build_environment/badge/icon)](https://jenkins.debian.net/job/reproducible_setup_fdroid_build_environment) | [![fdroid build all status](https://jenkins.debian.net/job/reproducible_fdroid_build_apps/badge/icon)](https://jenkins.debian.net/job/reproducible_fdroid_build_apps/) | [![fdroid test status](https://jenkins.debian.net/job/reproducible_fdroid_test/badge/icon)](https://jenkins.debian.net/job/reproducible_fdroid_test/) | +| macOS | [![fdroidserver status on macOS](https://travis-ci.org/f-droid/fdroidserver.svg?branch=master)](https://travis-ci.org/f-droid/fdroidserver) | | | | -

# F-Droid Server -### Tools for maintaining an F-Droid repository system. -
+Server for [F-Droid](https://f-droid.org), the Free Software repository system +for Android. ---- +The F-Droid server tools provide various scripts and tools that are +used to maintain the main +[F-Droid application repository](https://f-droid.org/packages). You +can use these same tools to create your own additional or alternative +repository for publishing, or to assist in creating, testing and +submitting metadata to the main repository. -## What is F-Droid Server? - -_fdroidserver_ is a suite of tools to publish and work with collections of -Android apps (APK files) and other kinds of packages. It is used to maintain -the [f-droid.org application repository](https://f-droid.org/packages). These -same tools can be used to create additional or alternative repositories for -publishing, or to assist in creating, testing and submitting metadata to the -f-droid.org repository, also known as -[_fdroiddata_](https://gitlab.com/fdroid/fdroiddata). - -For documentation, please see . - -In the beginning, _fdroidserver_ was the complete server-side setup that ran -f-droid.org. Since then, the website and other parts have been split out into -their own projects. The name for this suite of tooling has stayed -_fdroidserver_ even though it no longer contains any proper server component. +For documentation, please see , or you can +find the source for the documentation in +[fdroid/fdroid-website](https://gitlab.com/fdroid/fdroid-website). -## Installing +### What is F-Droid? -There are many ways to install _fdroidserver_, including using a range of -package managers. All of the options are documented on the website: +F-Droid is an installable catalogue of FOSS (Free and Open Source Software) +applications for the Android platform. The client makes it easy to browse, +install, and keep track of updates on your device. + + +### Installing + +There are many ways to install _fdroidserver_, they are documented on +the website: https://f-droid.org/docs/Installing_the_Server_and_Repo_Tools - -## Releases - -The production setup of _fdroidserver_ for f-droid.org is run directly from the -_master_ branch. This is put into production on an schedule (currently weekly). -So development and testing happens in the branches. We track branches using -merge requests. Therefore, there are many WIP and long-lived merge requests. - -There are also stable releases of _fdroidserver_. This is mostly intended for -running custom repositories, where the build process is separate. It can also -be useful as a simple way to get started contributing packages to _fdroiddata_, -since the stable releases are available in package managers. +All sorts of other documentation lives there as well. -## Tests +### Tests -To run the full test suite: - - tests/run-tests - -To run the tests for individual Python modules, see the `tests/test_*.py` files, e.g.: - - python -m unittest tests/test_metadata.py - -It is also possible to run individual tests: - - python -m unittest tests.test_metadata.MetadataTest.test_rewrite_yaml_special_build_params - -There is a growing test suite that has good coverage on a number of key parts of -this code base. It does not yet cover all the code, and there are some parts -where the technical debt makes it difficult to write unit tests. New tests -should be standard Python _unittest_ test cases. Whenever possible, the old -tests written in _bash_ in _tests/run-tests_ should be ported to Python. - -This test suite has built over time a bit haphazardly, so it is not as clean, -organized, or complete as it could be. We welcome contributions. The goal is -to move towards standard Python testing patterns and to expand the unit test -coverage. Before rearchitecting any parts of it, be sure to [contact -us](https://f-droid.org/about) to discuss the changes beforehand. +There are many components to all of the tests for the components in +this git repo. The most commonly used parts of well tested, while +some parts still lack tests. This test suite has built over time a +bit haphazardly, so it is not as clean, organized, or complete as it +could be. We welcome contributions. Before rearchitecting any parts +of it, be sure to [contact us](https://f-droid.org/about) to discuss +the changes beforehand. -### Additional tests for different linux distributions +#### `fdroid` commands -These tests are also run on various configurations through GitLab CI. This is -only enabled for `master@fdroid/fdroidserver` because it takes longer to +The test suite for all of the `fdroid` commands is in the _tests/_ +subdir. _.gitlab-ci.yml_ and _.travis.yml_ run this test suite on +various configurations. + +* _tests/complete-ci-tests_ runs _pylint_ and all tests on two + different pyvenvs +* _tests/run-tests_ runs the whole test suite +* _tests/*.TestCase_ are individual unit tests for all of the `fdroid` + commands, which can be run separately, e.g. `./update.TestCase`. + + +#### Additional tests for different linux distributions + +These tests are also run on various distributions through GitLab CI. This is +only enabled for `master@fdroid/fdroidserver` because it'll take longer to complete than the regular CI tests. Most of the time you won't need to worry -about them, but sometimes it might make sense to also run them for your merge -request. In that case you need to remove [these lines from .gitlab-ci.yml](https://gitlab.com/fdroid/fdroidserver/-/blob/0124b9dde99f9cab19c034cbc7d8cc6005a99b48/.gitlab-ci.yml#L90-91) +about them but sometimes it might make sense to also run them for your merge +request. In that case you need to remove [these lines from +.gitlab-ci.yml](https://gitlab.com/fdroid/fdroidserver/blob/master/.gitlab-ci.yml#L34-35) and push this to a new branch of your fork. Alternatively [run them locally](https://docs.gitlab.com/runner/commands/README.html#gitlab-runner-exec) like this: `gitlab-runner exec docker ubuntu_lts` +#### buildserver -## Documentation - -The API documentation based on the docstrings gets automatically -published [here](https://fdroid.gitlab.io/fdroidserver) on every commit -on the `master` branch. - -It can be built locally via - -```bash -pip install -e .[docs] -cd docs -sphinx-apidoc -o ./source ../fdroidserver -M -e -sphinx-autogen -o generated source/*.rst -make html -``` - -To additionally lint the code call -```bash -pydocstyle fdroidserver --count -``` - -When writing docstrings you should follow the -[numpy style guide](https://numpydoc.readthedocs.io/en/latest/format.html). +The tests for the whole build server setup are entirely separate +because they require at least 200GB of disk space, and 8GB of +RAM. These test scripts are in the root of the project, all starting +with _jenkins-_ since they are run on https://jenkins.debian.net. ## Translation @@ -118,16 +90,4 @@ When writing docstrings you should follow the Everything can be translated. See [Translation and Localization](https://f-droid.org/docs/Translation_and_Localization) for more info. - -
- -[![](https://hosted.weblate.org/widgets/f-droid/-/287x66-white.png)](https://hosted.weblate.org/engage/f-droid) - -
-View translation status for all languages. - -[![](https://hosted.weblate.org/widgets/f-droid/-/fdroidserver/multi-auto.svg)](https://hosted.weblate.org/engage/f-droid/?utm_source=widget) - -
- -
+[![translation status](https://hosted.weblate.org/widgets/f-droid/-/fdroidserver/multi-auto.svg)](https://hosted.weblate.org/engage/f-droid/?utm_source=widget) diff --git a/buildserver/Dockerfile b/buildserver/Dockerfile deleted file mode 100644 index 27ada3f8..00000000 --- a/buildserver/Dockerfile +++ /dev/null @@ -1,74 +0,0 @@ - -FROM debian:bookworm - -ENV LANG=C.UTF-8 \ - DEBIAN_FRONTEND=noninteractive - -RUN echo Etc/UTC > /etc/timezone \ - && echo 'Acquire::Retries "20";' \ - 'APT::Get::Assume-Yes "true";' \ - 'APT::Install-Recommends "0";' \ - 'APT::Install-Suggests "0";' \ - 'Dpkg::Use-Pty "0";' \ - 'quiet "1";' \ - >> /etc/apt/apt.conf.d/99gitlab - -# provision-apt-proxy was deliberately omitted, its not relevant in Docker -COPY provision-android-ndk \ - provision-android-sdk \ - provision-apt-get-install \ - provision-buildserverid \ - provision-gradle \ - setup-env-vars \ - /opt/buildserver/ - -ARG GIT_REV_PARSE_HEAD=unspecified -LABEL org.opencontainers.image.revision=$GIT_REV_PARSE_HEAD - -# setup 'vagrant' user for compatibility -RUN useradd --create-home -s /bin/bash vagrant && echo -n 'vagrant:vagrant' | chpasswd - -# The provision scripts must be run in the same order as in Vagrantfile -# - vagrant needs openssh-client iproute2 ssh sudo -# - ansible needs python3 -# -# Debian Docker images will soon default to HTTPS for apt sources, so force it. -# https://github.com/debuerreotype/docker-debian-artifacts/issues/15 -# -# Ensure fdroidserver's dependencies are marked manual before purging -# unneeded packages, otherwise, all its dependencies get purged. -# -# The official Debian docker images ship without ca-certificates, so -# TLS certificates cannot be verified until that is installed. The -# following code temporarily turns off TLS verification, and enables -# HTTPS, so at least unverified TLS is used for apt-get instead of -# plain HTTP. Once ca-certificates is installed, the CA verification -# is enabled by removing the newly created config file. This set up -# makes the initial `apt-get update` and `apt-get install` look the -# same as verified TLS to the network observer and hides the metadata. -RUN printf "path-exclude=/usr/share/locale/*\npath-exclude=/usr/share/man/*\npath-exclude=/usr/share/doc/*\npath-include=/usr/share/doc/*/copyright\n" >/etc/dpkg/dpkg.cfg.d/01_nodoc \ - && mkdir -p /usr/share/man/man1 \ - && echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/99nocacertificates \ - && find /etc/apt/sources.list* -type f -exec sed -i s,http:,https:, {} \; \ - && apt-get update \ - && apt-get install ca-certificates \ - && rm /etc/apt/apt.conf.d/99nocacertificates \ - && apt-get upgrade \ - && apt-get dist-upgrade \ - && apt-get install openssh-client iproute2 python3 openssh-server sudo \ - && bash /opt/buildserver/setup-env-vars /opt/android-sdk \ - && . /etc/profile.d/bsenv.sh \ - && bash /opt/buildserver/provision-apt-get-install https://deb.debian.org/debian \ - && bash /opt/buildserver/provision-android-sdk "tools;25.2.5" \ - && bash /opt/buildserver/provision-android-ndk /opt/android-sdk/ndk \ - && bash /opt/buildserver/provision-gradle \ - && bash /opt/buildserver/provision-buildserverid $GIT_REV_PARSE_HEAD \ - && rm -rf /vagrant/cache \ - && apt-get autoremove --purge \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Vagrant sudo setup for compatibility -RUN echo 'vagrant ALL = NOPASSWD: ALL' > /etc/sudoers.d/vagrant \ - && chmod 440 /etc/sudoers.d/vagrant \ - && sed -i -e 's/Defaults.*requiretty/#&/' /etc/sudoers diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index 61e3459a..392d3df6 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -1,41 +1,24 @@ + require 'yaml' require 'pathname' -require 'fileutils' - -configfile = { - 'boot_timeout' => 600, - 'cachedir' => File.join(ENV['HOME'], '.cache', 'fdroidserver'), - 'cpus' => 1, - 'debian_mirror' => 'https://deb.debian.org/debian/', - 'hwvirtex' => 'on', - 'memory' => 2048, - 'vm_provider' => 'virtualbox', -} srvpath = Pathname.new(File.dirname(__FILE__)).realpath -configpath = File.join(srvpath, "/Vagrantfile.yaml") -if File.exist? configpath - c = YAML.load_file(configpath) - if c and not c.empty? - c.each do |k,v| - configfile[k] = v - end - end -else - puts "Copying example file to #{configpath}" - FileUtils.cp('../examples/Vagrantfile.yaml', configpath) -end +configfile = YAML.load_file(File.join(srvpath, "/Vagrantfile.yaml")) Vagrant.configure("2") do |config| - if Vagrant.has_plugin?("vagrant-cachier") + # these two caching methods conflict, so only use one at a time + if Vagrant.has_plugin?("vagrant-cachier") and not configfile.has_key? "aptcachedir" config.cache.scope = :box config.cache.auto_detect = false config.cache.enable :apt config.cache.enable :chef end - config.vm.box = "debian/bookworm64" + config.vm.box = configfile['basebox'] + if configfile.has_key? "basebox_version" + config.vm.box_version = configfile['basebox_version'] + end if not configfile.has_key? "vm_provider" or configfile["vm_provider"] == "virtualbox" # default to VirtualBox if not set @@ -53,8 +36,6 @@ Vagrant.configure("2") do |config| libvirt.uri = "qemu:///system" libvirt.cpus = configfile["cpus"] libvirt.memory = configfile["memory"] - # Debian Vagrant image is only 20G, so allocate more - libvirt.machine_virtual_size = 1024 if configfile.has_key? "libvirt_disk_bus" libvirt.disk_bus = configfile["libvirt_disk_bus"] end @@ -67,8 +48,7 @@ Vagrant.configure("2") do |config| else synced_folder_type = '9p' end - config.vm.synced_folder './', '/vagrant', type: synced_folder_type, - SharedFoldersEnableSymlinksCreate: false + config.vm.synced_folder './', '/vagrant', type: synced_folder_type else abort("No supported VM Provider found, set vm_provider in Vagrantfile.yaml!") end @@ -80,30 +60,30 @@ Vagrant.configure("2") do |config| args: [configfile["aptproxy"]] end - config.vm.synced_folder configfile["cachedir"], '/vagrant/cache', - create: true, type: synced_folder_type - + # buildserver/ is shared to the VM's /vagrant by default so the old + # default does not need a custom mount + if configfile["cachedir"] != "buildserver/cache" + config.vm.synced_folder configfile["cachedir"], '/vagrant/cache', + create: true, type: synced_folder_type + end # Make sure dir exists to mount to, since buildserver/ is # automatically mounted as /vagrant in the guest VM. This is more # necessary with 9p synced folders - Dir.mkdir('cache') unless File.exist?('cache') + Dir.mkdir('cache') unless File.exists?('cache') - # Root partition needs to be resized to the new allocated space - config.vm.provision "shell", inline: <<-SHELL - growpart -v -u auto /dev/vda 1 - resize2fs /dev/vda1 - SHELL + # cache .deb packages on the host via a mount trick + if configfile.has_key? "aptcachedir" + config.vm.synced_folder configfile["aptcachedir"], "/var/cache/apt/archives", + owner: 'root', group: 'root', create: true + end - config.vm.provision "shell", name: "setup-env-vars", path: "setup-env-vars", - args: ["/opt/android-sdk"] - config.vm.provision "shell", name: "apt-get-install", path: "provision-apt-get-install", + config.vm.provision "shell", path: "setup-env-vars", + args: ["/home/vagrant/android-sdk"] + config.vm.provision "shell", path: "provision-apt-get-install", args: [configfile['debian_mirror']] - config.vm.provision "shell", name: "android-sdk", path: "provision-android-sdk" - config.vm.provision "shell", name: "android-ndk", path: "provision-android-ndk", - args: ["/opt/android-sdk/ndk"] - config.vm.provision "shell", name: "gradle", path: "provision-gradle" - config.vm.provision "shell", name: "disable-analytics", path: "provision-disable-analytics" - config.vm.provision "shell", name: "buildserverid", path: "provision-buildserverid", - args: [`git rev-parse HEAD`] + config.vm.provision "shell", path: "provision-android-sdk" + config.vm.provision "shell", path: "provision-android-ndk", + args: ["/home/vagrant/android-ndk"] + config.vm.provision "shell", path: "provision-gradle" end diff --git a/buildserver/config.buildserver.yml b/buildserver/config.buildserver.yml index 944535c5..b94fc601 100644 --- a/buildserver/config.buildserver.yml +++ b/buildserver/config.buildserver.yml @@ -1,2 +1,19 @@ -sdk_path: /opt/android-sdk +sdk_path: /home/vagrant/android-sdk +ndk_paths: + r10e: /home/vagrant/android-ndk/r10e + r11c: /home/vagrant/android-ndk/r11c + r12b: /home/vagrant/android-ndk/r12b + r13b: /home/vagrant/android-ndk/r13b + r14b: /home/vagrant/android-ndk/r14b + r15c: /home/vagrant/android-ndk/r15c + r16b: /home/vagrant/android-ndk/r16b + r17c: /home/vagrant/android-ndk/r17c + r18b: /home/vagrant/android-ndk/r18b + r19c: /home/vagrant/android-ndk/r19c + r20b: /home/vagrant/android-ndk/r20b + r21d: /home/vagrant/android-ndk/r21d + +java_paths: + 8: /usr/lib/jvm/java-8-openjdk-amd64 + gradle_version_dir: /opt/gradle/versions diff --git a/buildserver/provision-android-ndk b/buildserver/provision-android-ndk index 63f5eee7..c241bab3 100644 --- a/buildserver/provision-android-ndk +++ b/buildserver/provision-android-ndk @@ -1,30 +1,26 @@ #!/bin/bash # -# $1 is the root dir to install the NDKs into -# $2 and after are the NDK releases to install echo $0 set -e set -x NDK_BASE=$1 -shift test -e $NDK_BASE || mkdir -p $NDK_BASE cd $NDK_BASE -for version in $@; do +if [ ! -e $NDK_BASE/r10e ]; then + 7zr x /vagrant/cache/android-ndk-r10e-linux-x86_64.bin > /dev/null + mv android-ndk-r10e r10e +fi + +for version in r11c r12b r13b r14b r15c r16b r17c r18b r19c r20b r21d; do if [ ! -e ${NDK_BASE}/${version} ]; then - unzip /vagrant/cache/android-ndk-${version}-linux*.zip > /dev/null - mv android-ndk-${version} \ - `sed -En 's,^Pkg.Revision *= *(.+),\1,p' android-ndk-${version}/source.properties` + unzip /vagrant/cache/android-ndk-${version}-linux-x86_64.zip > /dev/null + mv android-ndk-${version} ${version} fi done -# allow gradle/etc to install missing NDK versions -chgrp vagrant $NDK_BASE -chmod g+w $NDK_BASE - -# ensure all users can read and execute the NDK chmod -R a+rX $NDK_BASE/ -find $NDK_BASE/ -type f -executable -exec chmod a+x -- {} + +find $NDK_BASE/ -type f -executable -print0 | xargs -0 chmod a+x diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index 19002a47..d0e73d8a 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -1,4 +1,5 @@ #!/bin/bash +# echo $0 set -e @@ -9,6 +10,19 @@ if [ -z $ANDROID_HOME ]; then exit 1 fi +# TODO remove the rm, this should work with an existing ANDROID_HOME +if [ ! -x $ANDROID_HOME/tools/android ]; then + rm -rf $ANDROID_HOME + mkdir ${ANDROID_HOME} + mkdir ${ANDROID_HOME}/temp + mkdir ${ANDROID_HOME}/platforms + mkdir ${ANDROID_HOME}/build-tools + cd $ANDROID_HOME + + tools=`ls -1 /vagrant/cache/tools_*.zip | sort -n | tail -1` + unzip -qq $tools +fi + # disable the repositories of proprietary stuff disabled=" @version@=1 @@ -26,96 +40,59 @@ for line in $disabled; do echo $line >> ${HOME}/.android/sites-settings.cfg done -# Include old makebuildserver cache that is a Vagrant synced_folder -# for sdkmanager to use. -cachedir=$HOME/.cache/sdkmanager -mkdir -p $cachedir -pushd $cachedir -for f in /vagrant/cache/*.zip; do - test -e $f && ln -s $f + +cd /vagrant/cache + +# make links for `android update sdk` to use and delete +blacklist="build-tools_r17-linux.zip + build-tools_r18.0.1-linux.zip + build-tools_r18.1-linux.zip + build-tools_r18.1.1-linux.zip + build-tools_r19-linux.zip + build-tools_r19.0.1-linux.zip + build-tools_r19.0.2-linux.zip + build-tools_r19.0.3-linux.zip + build-tools_r21-linux.zip + build-tools_r21.0.1-linux.zip + build-tools_r21.0.2-linux.zip + build-tools_r21.1-linux.zip + build-tools_r21.1.1-linux.zip + build-tools_r22-linux.zip + build-tools_r23-linux.zip + android-1.5_r04-linux.zip + android-1.6_r03-linux.zip + android-2.0_r01-linux.zip + android-2.0.1_r01-linux.zip" +latestm2=`ls -1 android_m2repository*.zip | sort -n | tail -1` +for f in $latestm2 android-[0-9]*.zip platform-[0-9]*.zip build-tools_r*-linux.zip; do + rm -f ${ANDROID_HOME}/temp/$f + if [[ $blacklist != *$f* ]]; then + ln -s /vagrant/cache/$f ${ANDROID_HOME}/temp/ + fi done -popd -# TODO do not preinstall 'tools' or 'platform-tools' at all, app builds don't need them -packages=" - tools;25.2.5 - platform-tools - build-tools;19.1.0 - build-tools;20.0.0 - build-tools;21.1.2 - build-tools;22.0.1 - build-tools;23.0.1 - build-tools;23.0.2 - build-tools;23.0.3 - build-tools;24.0.0 - build-tools;24.0.1 - build-tools;24.0.2 - build-tools;24.0.3 - build-tools;25.0.0 - build-tools;25.0.1 - build-tools;25.0.2 - build-tools;25.0.3 - build-tools;26.0.0 - build-tools;26.0.1 - build-tools;26.0.2 - build-tools;26.0.3 - build-tools;27.0.0 - build-tools;27.0.1 - build-tools;27.0.2 - build-tools;27.0.3 - build-tools;28.0.0 - build-tools;28.0.1 - build-tools;28.0.2 - build-tools;28.0.3 - build-tools;29.0.2 - build-tools;29.0.3 - build-tools;30.0.0 - build-tools;30.0.1 - build-tools;30.0.2 - build-tools;30.0.3 - build-tools;31.0.0 - build-tools;32.0.0 - build-tools;33.0.0 - platforms;android-10 - platforms;android-11 - platforms;android-12 - platforms;android-13 - platforms;android-14 - platforms;android-15 - platforms;android-16 - platforms;android-17 - platforms;android-18 - platforms;android-19 - platforms;android-20 - platforms;android-21 - platforms;android-22 - platforms;android-23 - platforms;android-24 - platforms;android-25 - platforms;android-26 - platforms;android-27 - platforms;android-28 - platforms;android-29 - platforms;android-30 - platforms;android-31 - platforms;android-32 - platforms;android-33 -" +# install all cached platforms +cached="" +for f in `ls -1 android-[0-9]*.zip platform-[0-9]*.zip`; do + sdk=`unzip -c $f "*/build.prop" | sed -n 's,^ro.build.version.sdk=,,p'` + cached=,android-${sdk}${cached} +done -if [ $# -gt 0 ]; then - echo found args - packages=$@ -fi +# install all cached build-tools +for f in `ls -1 build-tools*.zip`; do + ver=`unzip -c $f "*/source.properties" | sed -n 's,^Pkg.Revision=,,p'` + if [[ $ver == 24.0.0 ]] && [[ $f =~ .*r24\.0\.1.* ]]; then + # 24.0.1 has the wrong revision in the zip + ver=24.0.1 + fi + cached=,build-tools-${ver}${cached} +done -# temporary test of whether this script ran. It will change once -# 'tools' is no longer installed by default. -if [ ! -x $ANDROID_HOME/tools/bin/sdkmanager ]; then - mkdir -p ${ANDROID_HOME}/ - sdkmanager $packages -fi +${ANDROID_HOME}/tools/android update sdk --no-ui --all \ + --filter platform-tools,extra-android-m2repository${cached} < $ANDROID_HOME/licenses/android-sdk-preview-license-old 84831b9409646a918e30573bab4c9c91346d8abd EOF -cat < $ANDROID_HOME/licenses/intel-android-extra-license +echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.1" +echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout-solver;1.0.1" +echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.2" +echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout-solver;1.0.2" -d975f751698a77b662f1254ddbeed3901e976f5a -EOF - -chmod a+X $(dirname $ANDROID_HOME/) chmod -R a+rX $ANDROID_HOME/ chgrp vagrant $ANDROID_HOME chmod g+w $ANDROID_HOME find $ANDROID_HOME/ -type f -executable -print0 | xargs -0 chmod a+x # allow gradle to install newer build-tools and platforms -mkdir -p $ANDROID_HOME/{build-tools,platforms} chgrp vagrant $ANDROID_HOME/{build-tools,platforms} chmod g+w $ANDROID_HOME/{build-tools,platforms} @@ -160,8 +135,3 @@ chmod g+w $ANDROID_HOME/{build-tools,platforms} test -d $ANDROID_HOME/extras/m2repository || mkdir -p $ANDROID_HOME/extras/m2repository find $ANDROID_HOME/extras/m2repository -type d | xargs chgrp vagrant find $ANDROID_HOME/extras/m2repository -type d | xargs chmod g+w - -# allow gradle/sdkmanager to install extras;android;m2repository -test -d $ANDROID_HOME/extras/android || mkdir -p $ANDROID_HOME/extras/android -find $ANDROID_HOME/extras/android -type d | xargs chgrp vagrant -find $ANDROID_HOME/extras/android -type d | xargs chmod g+w diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index ca39c47b..7619a314 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -10,7 +10,7 @@ export DEBIAN_FRONTEND=noninteractive printf 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";\n' \ > /etc/apt/apt.conf.d/99no-install-recommends -printf 'Acquire::Retries "20";\n' \ +printf 'APT::Acquire::Retries "20";\n' \ > /etc/apt/apt.conf.d/99acquire-retries cat < /etc/apt/apt.conf.d/99no-auto-updates @@ -27,113 +27,115 @@ Dpkg::Use-Pty "0"; quiet "1"; EOF -cat < /etc/apt/apt.conf.d/99confdef -Dpkg::Options { "--force-confdef"; }; -EOF - -echo "man-db man-db/auto-update boolean false" | debconf-set-selections - if echo $debian_mirror | grep '^https' 2>&1 > /dev/null; then apt-get update || apt-get update - apt-get install ca-certificates + apt-get install apt-transport-https ca-certificates fi cat << EOF > /etc/apt/sources.list -deb ${debian_mirror} bookworm main -deb https://security.debian.org/debian-security bookworm-security main -deb ${debian_mirror} bookworm-updates main +deb ${debian_mirror} stretch main +deb http://security.debian.org/debian-security stretch/updates main +deb ${debian_mirror} stretch-updates main EOF -echo "deb ${debian_mirror} bookworm-backports main" > /etc/apt/sources.list.d/backports.list +echo "deb ${debian_mirror} stretch-backports main" > /etc/apt/sources.list.d/stretch-backports.list +echo "deb ${debian_mirror} stretch-backports-sloppy main" > /etc/apt/sources.list.d/stretch-backports-sloppy.list +echo "deb ${debian_mirror} testing main" > /etc/apt/sources.list.d/testing.list +printf "Package: *\nPin: release o=Debian,a=testing\nPin-Priority: -300\n" > /etc/apt/preferences.d/debian-testing + +dpkg --add-architecture i386 apt-get update || apt-get update - -# purge things that might come from the base box, but we don't want -# https://salsa.debian.org/cloud-team/debian-vagrant-images/-/tree/master/config_space/package_config -# cat config_space/package_config/* | sort -u | grep -v '[A-Z#]' - -purge=" - apt-listchanges - apt-utils - bash-completion - bind9-* - bsdextrautils - bzip2 - chrony - cloud-utils - cron - cron-daemon-common - dbus - debconf-i18n - debian-faq - dmidecode - doc-debian - fdisk - file - groff-base - inetutils-telnet - krb5-locales - less - locales - logrotate - lsof - manpages - nano - ncurses-term - netcat-traditional - pciutils - reportbug - rsyslog - tasksel - traceroute - unattended-upgrades - usrmerge - vim-* - wamerican - wget - whiptail - xz-utils -" -# clean up files packages to be purged, then purge the packages -rm -rf /var/run/dbus /var/log/unattended-upgrades -apt-get purge $purge - apt-get upgrade --download-only apt-get upgrade -# again after upgrade in case of keyring changes -apt-get update || apt-get update - packages=" - androguard/bookworm-backports - apksigner - default-jdk-headless - default-jre-headless + androguard/stretch-backports + ant + asn1c + ant-contrib + autoconf + autoconf2.13 + automake + automake1.11 + autopoint + bison + bzr + ca-certificates-java + cmake curl - dexdump - fdroidserver + disorderfs + expect + faketime + flex + gettext + gettext-base + git-core git-svn - gnupg + gperf + gpg/stretch-backports-sloppy + gpgconf/stretch-backports-sloppy + libassuan0/stretch-backports + libgpg-error0/stretch-backports + javacc + libarchive-zip-perl + libexpat1-dev + libgcc1:i386 + libglib2.0-dev + liblzma-dev + libncurses5:i386 + librsvg2-bin + libsaxonb-java + libssl-dev + libstdc++6:i386 + libtool + libtool-bin + make + maven mercurial - patch - python3-magic - python3-packaging + nasm + openjdk-8-jre-headless + openjdk-8-jdk-headless + optipng + p7zip + pkg-config + python-gnupg + python-lxml + python-magic + python-pip + python-setuptools + python3-asn1crypto/stretch-backports + python3-defusedxml + python3-git + python3-gitdb + python3-gnupg + python3-pip + python3-pyasn1 + python3-pyasn1-modules + python3-requests + python3-setuptools + python3-smmap + python3-yaml + python3-ruamel.yaml + python3-pil + python3-paramiko + quilt rsync - sdkmanager/bookworm-backports + scons + sqlite3 + subversion sudo + swig unzip + xsltproc + yasm + zip + zlib1g:i386 " - apt-get install $packages --download-only apt-get install $packages -# fdroidserver comes from git, it was installed just for dependencies -apt-mark manual `apt-cache depends fdroidserver | sed -nE 's,^[| ]*Depends: ([a-z0-9 -]+),\1,p'` -apt-get purge fdroidserver - -# clean up things that will become outdated anyway -apt-get autoremove --purge -apt-get clean -rm -rf /var/lib/apt/lists/* - highestjava=`update-java-alternatives --list | sort -n | tail -1 | cut -d ' ' -f 1` update-java-alternatives --set $highestjava + +# configure headless openjdk to work without gtk accessability dependencies +sed -i -e 's@\(assistive_technologies=org.GNOME.Accessibility.AtkWrapper\)@#\1@' /etc/java-8-openjdk/accessibility.properties diff --git a/buildserver/provision-buildserverid b/buildserver/provision-buildserverid deleted file mode 100644 index f5010c39..00000000 --- a/buildserver/provision-buildserverid +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -e - -test -n "$1" - -echo "Writing buildserver ID ...ID is $1" -set -x -echo "$1" > /home/vagrant/buildserverid -# sync data before we halt() the machine, we had an empty buildserverid otherwise -sync diff --git a/buildserver/provision-disable-analytics b/buildserver/provision-disable-analytics deleted file mode 100644 index e1ec62b7..00000000 --- a/buildserver/provision-disable-analytics +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -ex - -# Flutter -# https://github.com/flutter/flutter/issues/73657 -flutter_conf=/home/vagrant/.flutter -cat < $flutter_conf -{ - "enabled": false -} -EOF -chown -R vagrant:vagrant $flutter_conf -chmod -R 0644 $flutter_conf - diff --git a/buildserver/provision-gradle b/buildserver/provision-gradle index a282a4c5..ce8aefa1 100644 --- a/buildserver/provision-gradle +++ b/buildserver/provision-gradle @@ -10,29 +10,21 @@ vergte() { test -e /opt/gradle/versions || mkdir -p /opt/gradle/versions cd /opt/gradle/versions - -glob="/vagrant/cache/gradle-*.zip" -if compgen -G $glob; then # test if glob matches anything - f=$(ls -1 --sort=version --group-directories-first $glob | tail -1) +for f in /vagrant/cache/gradle-*.zip; do ver=`echo $f | sed 's,.*gradle-\([0-9][0-9.]*\).*\.zip,\1,'` # only use versions greater or equal 2.2.1 if vergte $ver 2.2.1 && [ ! -d /opt/gradle/versions/${ver} ]; then unzip -qq $f mv gradle-${ver} /opt/gradle/versions/${ver} fi -fi +done chmod -R a+rX /opt/gradle test -e /opt/gradle/bin || mkdir -p /opt/gradle/bin -git clone --depth 1 https://gitlab.com/fdroid/gradlew-fdroid.git /home/vagrant/gradlew-fdroid/ -chmod 0755 /home/vagrant/gradlew-fdroid/gradlew-fdroid -chmod -R u+rwX,a+rX,go-w /home/vagrant/gradlew-fdroid/ -ln -fs /home/vagrant/gradlew-fdroid/gradlew-fdroid /opt/gradle/bin/gradle -ln -fs /home/vagrant/gradlew-fdroid/gradlew-fdroid /usr/local/bin/ - -chown -h vagrant:vagrant /opt/gradle/bin/gradle -chown vagrant:vagrant /opt/gradle/versions +ln -fs /home/vagrant/fdroidserver/gradlew-fdroid /opt/gradle/bin/gradle +chown -h vagrant.vagrant /opt/gradle/bin/gradle +chown vagrant.vagrant /opt/gradle/versions chmod 0755 /opt/gradle/versions GRADLE_HOME=/home/vagrant/.gradle @@ -49,5 +41,5 @@ systemProp.org.gradle.internal.http.connectionTimeout=600000 systemProp.org.gradle.internal.http.socketTimeout=600000 EOF -chown -R vagrant:vagrant $GRADLE_HOME/ +chown -R vagrant.vagrant $GRADLE_HOME/ chmod -R a+rX $GRADLE_HOME/ diff --git a/buildserver/setup-env-vars b/buildserver/setup-env-vars index 1c3599e9..19259266 100644 --- a/buildserver/setup-env-vars +++ b/buildserver/setup-env-vars @@ -12,16 +12,9 @@ echo "# generated on "`date` > $bsenv echo export ANDROID_HOME=$1 >> $bsenv echo 'export PATH=$PATH:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:/opt/gradle/bin' >> $bsenv echo "export DEBIAN_FRONTEND=noninteractive" >> $bsenv -echo 'export home_vagrant=/home/vagrant' >> $bsenv -echo 'export fdroidserver=$home_vagrant/fdroidserver' >> $bsenv -echo 'export LC_ALL=C.UTF-8' >> $bsenv chmod 0644 $bsenv # make sure that SSH never hangs at a password or key prompt -mkdir -p /etc/ssh/ssh_config.d/ -cat << EOF >> /etc/ssh/ssh_config.d/fdroid -Host * - StrictHostKeyChecking yes - BatchMode yes -EOF +printf ' StrictHostKeyChecking yes' >> /etc/ssh/ssh_config +printf ' BatchMode yes' >> /etc/ssh/config diff --git a/completion/bash-completion b/completion/bash-completion index 57fcfd12..751e5b24 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -83,7 +83,7 @@ __complete_options() { __complete_build() { opts="-v -q -l -s -t -f -a" - lopts="--verbose --quiet --latest --stop --test --server --skip-scan --scan-binary --no-tarball --force --all --no-refresh" + lopts="--verbose --quiet --latest --stop --test --server --reset-server --skip-scan --scan-binary --no-tarball --force --all --no-refresh" case "${prev}" in :) __vercode @@ -109,8 +109,8 @@ __complete_gpgsign() { } __complete_install() { - opts="-v -q -a -p -n -y" - lopts="--verbose --quiet --all --color --no-color --privacy-mode --no-privacy-mode --no --yes" + opts="-v -q" + lopts="--verbose --quiet --all" case "${cur}" in -*) __complete_options @@ -155,7 +155,7 @@ __complete_publish() { __complete_checkupdates() { opts="-v -q" - lopts="--verbose --quiet --auto --autoonly --commit --allow-dirty" + lopts="--verbose --quiet --auto --autoonly --commit --gplay --allow-dirty" case "${cur}" in -*) __complete_options @@ -251,7 +251,7 @@ __complete_btlog() { __complete_mirror() { opts="-v" - lopts="--all --archive --build-logs --color --no-color --pgp-signatures --src-tarballs --output-dir" + lopts="--all --archive --build-logs --pgp-signatures --src-tarballs --output-dir" __complete_options } @@ -261,6 +261,12 @@ __complete_nightly() { __complete_options } +__complete_stats() { + opts="-v -q -d" + lopts="--verbose --quiet --download" + __complete_options +} + __complete_deploy() { opts="-i -v -q" lopts="--identity-file --local-copy-dir --sync-from-local-copy-dir @@ -270,14 +276,12 @@ __complete_deploy() { __complete_signatures() { opts="-v -q" - lopts="--verbose --color --no-color --no-check-https" + lopts="--verbose --no-check-https" case "${cur}" in -*) __complete_options return 0;; esac - _filedir 'apk' - return 0 } __complete_signindex() { @@ -289,7 +293,7 @@ __complete_signindex() { __complete_init() { opts="-v -q -d" lopts="--verbose --quiet --distinguished-name --keystore - --repo-keyalias --android-home --no-prompt --color --no-color" + --repo-keyalias --android-home --no-prompt" __complete_options } @@ -298,6 +302,7 @@ btlog \ build \ checkupdates \ deploy \ +dscanner \ gpgsign \ import \ init \ @@ -311,6 +316,7 @@ rewritemeta \ scanner \ signatures \ signindex \ +stats \ update \ verify \ " diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index d0c3cbf1..00000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 6247f7e2..00000000 --- a/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index c20542de..00000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,78 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys - -sys.path.insert(0, os.path.abspath('../../fdroidserver')) - -# -- Project information ----------------------------------------------------- - -project = 'fdroidserver' -copyright = '2021, The F-Droid Project' -author = 'The F-Droid Project' - - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'numpydoc', - 'sphinx.ext.autodoc', - 'sphinx.ext.autosummary', - "sphinx.ext.intersphinx", -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = "pydata_sphinx_theme" - -html_theme_options = { - "gitlab_url": "https://gitlab.com/fdroid/fdroidserver", - "show_prev_next": False, - "navbar_end": ["search-field.html", "navbar-icon-links.html"], -} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -html_sidebars = { - "**": [], -} - -#html_sidebars = { -# '**': ['globaltoc.html', 'sourcelink.html', 'searchbox.html'], -# 'using/windows': ['windowssidebar.html', 'searchbox.html'], -#} - -html_split_index = True -#numpydoc_validation_checks = {"all"} - -intersphinx_mapping = { - "python": ("https://docs.python.org/3/", None), -} diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index fcd4dfe3..00000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. fdroidserver documentation master file, created by - sphinx-quickstart on Mon May 3 10:06:52 2021. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to fdroidserver's documentation! -======================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - -These pages contain the autogenerated module docu based on the current `sources `_. - -Indices and tables -================== - - -* Under :ref:`modindex` the different fdroidserver modules are listed. -* In :ref:`genindex` you'll find all methods sorted alphabetically. diff --git a/examples/Vagrantfile.yaml b/examples/Vagrantfile.yaml deleted file mode 100644 index 276f0179..00000000 --- a/examples/Vagrantfile.yaml +++ /dev/null @@ -1,54 +0,0 @@ ---- - -# You may want to alter these before running ./makebuildserver - -# In the process of setting up the build server, many gigs of files -# are downloaded (Android SDK components, gradle, etc). These are -# cached so that they are not redownloaded each time. By default, -# these are stored in ~/.cache/fdroidserver -# -# cachedir: buildserver/cache - -# To specify which Debian mirror the build server VM should use, by -# default it uses http.debian.net, which auto-detects which is the -# best mirror to use. -# -# debian_mirror: https://debian.osuosl.org/debian/ - -# The amount of RAM the build server will have (default: 2048) -# memory: 3584 - -# The number of CPUs the build server will have -# cpus: 1 - -# Debian package proxy server - if you have one -# aptproxy: http://192.168.0.19:8000 - -# If this is running on an older machine or on a virtualized system, -# it can run a lot slower. If the provisioning fails with a warning -# about the timeout, extend the timeout here. (default: 600 seconds) -# -# boot_timeout: 1200 - -# By default, this whole process uses VirtualBox as the provider, but -# QEMU+KVM is also supported via the libvirt plugin to vagrant. If -# this is run within a KVM guest, then libvirt's QEMU+KVM will be used -# automatically. It can also be manually enabled by uncommenting -# below: -# -# vm_provider: libvirt - -# By default libvirt uses 'virtio' for both network and disk drivers. -# Some systems (eg. nesting VMware ESXi) do not support virtio. As a -# workaround for such rare cases, this setting allows to configure -# KVM/libvirt to emulate hardware rather than using virtio. -# -# libvirt_disk_bus: sata -# libvirt_nic_model_type: rtl8139 - -# Sometimes, it is not possible to use the 9p synced folder type with -# libvirt, like if running a KVM buildserver instance inside of a -# VMware ESXi guest. In that case, using NFS or another method is -# required. -# -# synced_folder_type: nfs diff --git a/examples/config.yml b/examples/config.yml index ae4e7008..e022e389 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -5,20 +5,23 @@ # Custom path to the Android SDK, defaults to $ANDROID_HOME # sdk_path: $ANDROID_HOME -# Paths to installed versions of the Android NDK. This will be -# automatically filled out from well known sources like -# $ANDROID_HOME/ndk-bundle and $ANDROID_HOME/ndk/*. If a required -# version is missing in the buildserver VM, it will be automatically -# downloaded and installed into the standard $ANDROID_HOME/ndk/ -# directory. Manually setting it here will override the auto-detected -# values. The keys can either be the "release" (e.g. r21e) or the -# "revision" (e.g. 21.4.7075529). -# +# Custom paths to various versions of the Android NDK, defaults to 'r12b' set +# to $ANDROID_NDK. Most users will have the latest at $ANDROID_NDK, which is +# used by default. If a version is missing or assigned to None, it is assumed +# not installed. # ndk_paths: -# r10e: $ANDROID_HOME/android-ndk-r10e -# r17: "" -# 21.4.7075529: ~/Android/Ndk -# r22b: null +# r10e: None +# r11c: None +# r12b: $ANDROID_NDK +# r13b: None +# r14b: None +# r15c: None +# r16b: None +# r17c: None +# r18b: None +# r19c: None +# r20b: None +# r21d: None # Directory to store downloaded tools in (i.e. gradle versions) # By default, these are stored in ~/.cache/fdroidserver @@ -48,46 +51,25 @@ # The same policy is applied to the archive repo, if there is one. # repo_maxage: 0 -# Canonical URL of the repositoy, needs to end in /repo. Is is used to identity -# the repo in the client, as well. -# repo_url: https://MyFirstFDroidRepo.org/fdroid/repo -# -# Base URL for per-package pages on the website of this repo, -# i.e. https://f-droid.org/packages// This should be accessible -# with a browser. Setting it to null or not setting this disables the -# feature. -# repo_web_base_url: https://MyFirstFDroidRepo.org/packages/ -# -# repo_name: My First F-Droid Repo Demo -# repo_description: >- -# This is a repository of apps to be used with F-Droid. Applications -# in this repository are either official binaries built by the -# original application developers, or are binaries built from source -# by the admin of f-droid.org using the tools on -# https://gitlab.com/fdroid. +repo_url: https://MyFirstFDroidRepo.org/fdroid/repo +repo_name: My First F-Droid Repo Demo +repo_icon: fdroid-icon.png +repo_description: >- + This is a repository of apps to be used with F-Droid. Applications in this + repository are either official binaries built by the original application + developers, or are binaries built from source by the admin of f-droid.org + using the tools on https://gitlab.com/u/fdroid. # As above, but for the archive repo. -# -# archive_url: https://f-droid.org/archive -# archive_web_base_url: -# archive_name: My First F-Droid Archive Demo -# archive_description: >- -# The repository of older versions of packages from the main demo repository. - # archive_older sets the number of versions kept in the main repo, with all # older ones going to the archive. Set it to 0, and there will be no archive # repository, and no need to define the other archive_ values. -# -# archive_older: 3 - -# The repo's icon defaults to a file called 'icon.png' in the 'icons' -# folder for each section, e.g. repo/icons/icon.png and -# archive/icons/icon.png. To use a different filename for the icons, -# set the filename here. You must still copy it into place in -# repo/icons/ and/or archive/icons/. -# -# repo_icon: myicon.png -# archive_icon: myicon.png +archive_older: 3 +archive_url: https://f-droid.org/archive +archive_name: My First F-Droid Archive Demo +archive_icon: fdroid-icon.png +archive_description: >- + The repository of older versions of packages from the main demo repository. # This allows a specific kind of insecure APK to be included in the # 'repo' section. Since April 2017, APK signatures that use MD5 are @@ -117,7 +99,7 @@ # Optionally, override home directory for gpg # gpghome: /home/fdroid/somewhere/else/.gnupg -# The ID of a GPG key for making detached signatures for APKs. Optional. +# The ID of a GPG key for making detached signatures for apks. Optional. # gpgkey: 1DBA2E89 # The key (from the keystore defined below) to be used for signing the @@ -186,12 +168,6 @@ # serverwebroot: # - foo.com:/usr/share/nginx/www/fdroid # - bar.info:/var/www/fdroid -# -# There is a special mode to only deploy the index file: -# -# serverwebroot: -# - url: 'me@b.az:/srv/fdroid' -# index_only: true # When running fdroid processes on a remote server, it is possible to @@ -206,49 +182,14 @@ # deploy_process_logs: true # The full URL to a git remote repository. You can include -# multiple servers to mirror to by adding strings to a YAML list or map. +# multiple servers to mirror to by wrapping the whole thing in {} or [], and +# including the servergitmirrors strings in a comma-separated list. # Servers listed here will also be automatically inserted in the mirrors list. # # servergitmirrors: https://github.com/user/repo # servergitmirrors: # - https://github.com/user/repo # - https://gitlab.com/user/repo -# -# servergitmirrors: -# - url: https://github.com/user/repo -# - url: https://gitlab.com/user/repo -# index_only: true - - -# These settings allow using `fdroid deploy` for publishing APK files from -# your repository to GitHub Releases. (You should also run `fdroid update` -# every time before deploying to GitHub releases to update index files.) Here's -# an example for this deployment automation: -# https://github.com/f-droid/fdroidclient/releases/ -# -# Currently, versions which are assigned to a release channel (e.g. alpha or -# beta releases) are ignored. -# -# In the example below, tokens are read from environment variables. Putting -# tokens directly into the config file is also supported but discouraged. It is -# highly recommended to use a "Fine-grained personal access token", which is -# restricted to the minimum required permissions, which are: -# * Metadata - read -# * Contents - read/write -# (https://github.com/settings/personal-access-tokens/new) -# -# github_token: {env: GITHUB_TOKEN} -# github_releases: -# - projectUrl: https://github.com/f-droid/fdroidclient -# packageNames: -# - org.fdroid.basic -# - org.fdroid.fdroid -# release_notes_prepend: | -# Re-post of official F-Droid App release from https://f-droid.org -# - projectUrl: https://github.com/example/app -# packageNames: com.example.app -# token: {env: GITHUB_TOKEN_EXAMPLE} - # Most git hosting services have hard size limits for each git repo. # `fdroid deploy` will delete the git history when the git mirror repo @@ -269,18 +210,6 @@ # mirrors: # - https://foo.bar/fdroid # - http://foobarfoobarfoobar.onion/fdroid -# -# Or additional metadata can also be included by adding key/value pairs: -# -# mirrors: -# - url: https://foo.bar/fdroid -# countryCode: BA -# - url: http://foobarfoobarfoobar.onion/fdroid -# -# The list of mirrors can also be maintained in config/mirrors.yml, a -# standalone YAML file in the optional configuration directory. In -# that case, mirrors: should be removed from this file (config.yml). - # optionally specify which identity file to use when using rsync or git over SSH # @@ -305,33 +234,19 @@ # # sync_from_local_copy_dir: true -# To deploy to an AWS S3 "bucket" in the US East region, set the -# bucket name in the config, then set the environment variables -# AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY using the values from -# the AWS Management Console. See -# https://rclone.org/s3/#authentication + +# To upload the repo to an Amazon S3 bucket using `fdroid server +# update`. Warning, this deletes and recreates the whole fdroid/ +# directory each time. This prefers s3cmd, but can also use +# apache-libcloud. To customize how s3cmd interacts with the cloud +# provider, create a 's3cfg' file next to this file (config.yml), and +# those settings will be used instead of any 'aws' variable below. +# Secrets can be fetched from environment variables to ensure that +# they are not leaked as part of this file. # -# awsbucket: myawsfdroidbucket - - -# For extended options for syncing to cloud drive and object store -# services, `fdroid deploy' wraps Rclone. Rclone is a full featured -# sync tool for a huge variety of cloud services. Set up your services -# using `rclone config`, then specify each config name to deploy the -# awsbucket: to. Using rclone_config: overrides the default AWS S3 US -# East setup, and will only sync to the services actually specified. -# -# awsbucket: myawsfdroidbucket -# rclone_config: -# - aws-sample-config -# - rclone-supported-service-config - - -# By default Rclone uses the user's default configuration file at -# ~/.config/rclone/rclone.conf To specify a custom configuration file, -# please add the full path to the configuration file as below. -# -# path_to_custom_rclone_config: /home/mycomputer/somedir/example.conf +# awsbucket: myawsfdroid +# awsaccesskeyid: SEE0CHAITHEIMAUR2USA +# awssecretkey: {env: awssecretkey} # If you want to force 'fdroid server' to use a non-standard serverwebroot. @@ -342,12 +257,12 @@ # nonstandardwebroot: false -# If you want to upload the release APK file to androidobservatory.org +# If you want to upload the release apk file to androidobservatory.org # # androidobservatory: false -# If you want to upload the release APK file to virustotal.com +# If you want to upload the release apk file to virustotal.com # You have to enter your profile apikey to enable the upload. # # virustotal_apikey: 9872987234982734 @@ -357,6 +272,13 @@ # virustotal_apikey: {env: virustotal_apikey} +# The build logs can be posted to a mediawiki instance, like on f-droid.org. +# wiki_protocol: http +# wiki_server: server +# wiki_path: /wiki/ +# wiki_user: login +# wiki_password: 1234 + # Keep a log of all generated index files in a git repo to provide a # "binary transparency" log for anyone to check the history of the # binaries that are published. This is in the form of a "git remote", @@ -364,6 +286,27 @@ # configured to allow push access (e.g. ssh key, username/password, etc) # binary_transparency_remote: git@gitlab.com:fdroid/binary-transparency-log.git +# Only set this to true when running a repository where you want to generate +# stats, and only then on the master build servers, not a development +# machine. If you want to keep the "added" and "last updated" dates for each +# app and APK in your repo, then you should enable this. +# update_stats: true + +# When used with stats, this is a list of IP addresses that are ignored for +# calculation purposes. +# stats_ignore: [] + +# Server stats logs are retrieved from. Required when update_stats is True. +# stats_server: example.com + +# User stats logs are retrieved from. Required when update_stats is True. +# stats_user: bob + +# Use the following to push stats to a Carbon instance: +# stats_to_carbon: false +# carbon_host: 0.0.0.0 +# carbon_port: 2003 + # Set this to true to always use a build server. This saves specifying the # --server option on dedicated secure build server hosts. # build_server_always: true @@ -403,31 +346,9 @@ # generating our default list. (https://pypi.org/project/spdx-license-list) # # You can override our default list of allowed licenes by setting this option. -# Just supply a custom list of licene names you would like to allow. To disable -# checking licenses by the linter, assign an empty value to lint_licenses. +# Just supply a custom list of licene names you would like to allow. Setting +# this to `None` disables this lint check. # # lint_licenses: # - Custom-License-A # - Another-License - -# `fdroid scanner` can scan for signatures from various sources. By default -# it's configured to only use F-Droids official SUSS collection. We have -# support for these special collections: -# * 'exodus' - official exodus-privacy.org signatures -# * 'etip' - exodus privacy investigation platfrom community contributed -# signatures -# * 'suss' - official F-Droid: Suspicious or Unwanted Software Signatures -# You can also configure scanner to use custom collections of signatures here. -# They have to follow the format specified in the SUSS readme. -# (https://gitlab.com/fdroid/fdroid-suss/#cache-file-data-format) -# -# scanner_signature_sources: -# - suss -# - exodus -# - https://example.com/signatures.json - -# The scanner can use signature sources from the internet. These are -# cached locally. To force them to be refreshed from the network on -# every run, set this to true: -# -# refresh_scanner: true diff --git a/examples/fdroid-icon.png b/examples/fdroid-icon.png new file mode 100644 index 00000000..0c0d4173 Binary files /dev/null and b/examples/fdroid-icon.png differ diff --git a/examples/fdroid_clean_repos.py b/examples/fdroid_clean_repos.py deleted file mode 100644 index 6b19cacc..00000000 --- a/examples/fdroid_clean_repos.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 -# -# an fdroid plugin for resetting app VCSs to the latest version for the metadata - -import argparse -import logging - -from fdroidserver import _, common, metadata -from fdroidserver.exception import VCSException - -fdroid_summary = 'reset app VCSs to the latest version' - - -def main(): - parser = argparse.ArgumentParser( - usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]" - ) - common.setup_global_opts(parser) - parser.add_argument( - "appid", - nargs='*', - help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"), - ) - metadata.add_metadata_arguments(parser) - options = common.parse_args(parser) - apps = common.read_app_args( - options.appid, allow_version_codes=True, sort_by_time=True - ) - common.read_config() - - for appid, app in apps.items(): - if "Builds" in app and len(app["Builds"]) > 0: - build = app.get('Builds')[-1] - logging.info(_("Cleaning up '{appid}' VCS").format(appid=appid)) - try: - vcs, build_dir = common.setup_vcs(app) - vcs.gotorevision(build.commit) - if build.submodules: - vcs.initsubmodules() - - except VCSException: - pass - - -if __name__ == "__main__": - main() diff --git a/examples/fdroid_export_keystore_to_nitrokey.py b/examples/fdroid_export_keystore_to_nitrokey.py deleted file mode 100644 index 6e920a78..00000000 --- a/examples/fdroid_export_keystore_to_nitrokey.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -# -# an fdroid plugin for exporting a repo's keystore in standard PEM format - -import os -from argparse import ArgumentParser - -from fdroidserver import common -from fdroidserver.common import FDroidPopen -from fdroidserver.exception import BuildException - -fdroid_summary = "export the repo's keystore file to a NitroKey HSM" - - -def run(cmd, error): - envs = {'LC_ALL': 'C.UTF-8', - 'PIN': config['smartcard_pin'], - 'FDROID_KEY_STORE_PASS': config['keystorepass'], - 'FDROID_KEY_PASS': config['keypass']} - p = FDroidPopen(cmd, envs=envs) - if p.returncode != 0: - raise BuildException(error, p.output) - - -def main(): - global config - parser = ArgumentParser() - common.setup_global_opts(parser) - common.parse_args(parser) - config = common.read_config() - destkeystore = config['keystore'].replace('.jks', '.p12').replace('/', '_') - exportkeystore = config['keystore'].replace('.jks', '.pem').replace('/', '_') - if os.path.exists(destkeystore) or os.path.exists(exportkeystore): - raise BuildException('%s exists!' % exportkeystore) - run([config['keytool'], '-importkeystore', - '-srckeystore', config['keystore'], - '-srcalias', config['repo_keyalias'], - '-srcstorepass:env', 'FDROID_KEY_STORE_PASS', - '-srckeypass:env', 'FDROID_KEY_PASS', - '-destkeystore', destkeystore, - '-deststorepass:env', 'FDROID_KEY_STORE_PASS', - '-deststoretype', 'PKCS12'], - 'Failed to convert to PKCS12!') -# run(['openssl', 'pkcs12', '-in', destkeystore, -# '-passin', 'env:FDROID_KEY_STORE_PASS', '-nokeys', -# '-out', exportkeystore, -# '-passout', 'env:FDROID_KEY_STORE_PASS'], -# 'Failed to convert to PEM!') - run(['pkcs15-init', '--delete-objects', 'privkey,pubkey', - '--id', '3', '--store-private-key', destkeystore, - '--format', 'pkcs12', '--auth-id', '3', - '--verify-pin', '--pin', 'env:PIN'], - '') - run(['pkcs15-init', '--delete-objects', 'privkey,pubkey', - '--id', '2', '--store-private-key', destkeystore, - '--format', 'pkcs12', '--auth-id', '3', - '--verify-pin', '--pin', 'env:PIN'], - '') - - -if __name__ == "__main__": - main() diff --git a/examples/fdroid_exportkeystore.py b/examples/fdroid_exportkeystore.py deleted file mode 100644 index f2a16980..00000000 --- a/examples/fdroid_exportkeystore.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 -# -# an fdroid plugin for exporting a repo's keystore in standard PEM format - -import os -from argparse import ArgumentParser - -from fdroidserver import common -from fdroidserver.common import FDroidPopen -from fdroidserver.exception import BuildException - -fdroid_summary = 'export the keystore in standard PEM format' - - -def main(): - parser = ArgumentParser() - common.setup_global_opts(parser) - common.parse_args(parser) - config = common.read_config() - env_vars = {'LC_ALL': 'C.UTF-8', - 'FDROID_KEY_STORE_PASS': config['keystorepass'], - 'FDROID_KEY_PASS': config['keypass']} - destkeystore = config['keystore'].replace('.jks', '.p12').replace('/', '_') - exportkeystore = config['keystore'].replace('.jks', '.pem').replace('/', '_') - if os.path.exists(destkeystore) or os.path.exists(exportkeystore): - raise BuildException('%s exists!' % exportkeystore) - p = FDroidPopen([config['keytool'], '-importkeystore', - '-srckeystore', config['keystore'], - '-srcalias', config['repo_keyalias'], - '-srcstorepass:env', 'FDROID_KEY_STORE_PASS', - '-srckeypass:env', 'FDROID_KEY_PASS', - '-destkeystore', destkeystore, - '-deststoretype', 'PKCS12', - '-deststorepass:env', 'FDROID_KEY_STORE_PASS', - '-destkeypass:env', 'FDROID_KEY_PASS'], - envs=env_vars) - if p.returncode != 0: - raise BuildException("Failed to convert to PKCS12!", p.output) - p = FDroidPopen(['openssl', 'pkcs12', '-in', destkeystore, - '-passin', 'env:FDROID_KEY_STORE_PASS', '-nokeys', - '-out', exportkeystore, - '-passout', 'env:FDROID_KEY_STORE_PASS'], - envs=env_vars) - if p.returncode != 0: - raise BuildException("Failed to convert to PEM!", p.output) - - -if __name__ == "__main__": - main() diff --git a/examples/fdroid_extract_repo_pubkey.py b/examples/fdroid_extract_repo_pubkey.py deleted file mode 100644 index cb5a895c..00000000 --- a/examples/fdroid_extract_repo_pubkey.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 -# -# an fdroid plugin print the repo_pubkey from a repo's keystore -# - -from argparse import ArgumentParser - -from fdroidserver import common, index - -fdroid_summary = 'export the keystore in standard PEM format' - - -def main(): - parser = ArgumentParser() - common.setup_global_opts(parser) - common.parse_args(parser) - common.read_config() - pubkey, repo_pubkey_fingerprint = index.extract_pubkey() - print('repo_pubkey = "%s"' % pubkey.decode()) - - -if __name__ == "__main__": - main() diff --git a/examples/fdroid_fetchsrclibs.py b/examples/fdroid_fetchsrclibs.py deleted file mode 100644 index aba6f7fa..00000000 --- a/examples/fdroid_fetchsrclibs.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -# -# an fdroid plugin for setting up srclibs -# -# The 'fdroid build' gitlab-ci job uses --on-server, which does not -# set up the srclibs. This plugin does the missing setup. - -import argparse -import os -import pprint - -from fdroidserver import _, common, metadata - -fdroid_summary = 'prepare the srclibs for `fdroid build --on-server`' - - -def main(): - parser = argparse.ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") - common.setup_global_opts(parser) - parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]")) - metadata.add_metadata_arguments(parser) - options = common.parse_args(parser) - apps = common.read_app_args(options.appid, allow_version_codes=True, sort_by_time=True) - common.read_config() - srclib_dir = os.path.join('build', 'srclib') - os.makedirs(srclib_dir, exist_ok=True) - srclibpaths = [] - for appid, app in apps.items(): - vcs, _ignored = common.setup_vcs(app) - for build in app.get('Builds', []): - vcs.gotorevision(build.commit, refresh=False) - if build.submodules: - vcs.initsubmodules() - else: - vcs.deinitsubmodules() - for lib in build.srclibs: - srclibpaths.append(common.getsrclib(lib, srclib_dir, prepare=False, build=build)) - print('Set up srclibs:') - pprint.pprint(srclibpaths) - - -if __name__ == "__main__": - main() diff --git a/examples/fdroid_nitrokeyimport.py b/examples/fdroid_nitrokeyimport.py deleted file mode 100644 index d17a6186..00000000 --- a/examples/fdroid_nitrokeyimport.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 - -from argparse import ArgumentParser - -from fdroidserver import common -from fdroidserver.common import FDroidPopen -from fdroidserver.exception import BuildException - -fdroid_summary = 'import the local keystore into a SmartCard HSM' - - -def main(): - parser = ArgumentParser() - common.setup_global_opts(parser) - common.parse_args(parser) - config = common.read_config() - env_vars = { - 'LC_ALL': 'C.UTF-8', - 'FDROID_KEY_STORE_PASS': config['keystorepass'], - 'FDROID_KEY_PASS': config['keypass'], - 'SMARTCARD_PIN': str(config['smartcard_pin']), - } - p = FDroidPopen([config['keytool'], '-importkeystore', - '-srcalias', config['repo_keyalias'], - '-srckeystore', config['keystore'], - '-srcstorepass:env', 'FDROID_KEY_STORE_PASS', - '-srckeypass:env', 'FDROID_KEY_PASS', - '-destalias', config['repo_keyalias'], - '-destkeystore', 'NONE', - '-deststoretype', 'PKCS11', - '-providerName', 'SunPKCS11-OpenSC', - '-providerClass', 'sun.security.pkcs11.SunPKCS11', - '-providerArg', 'opensc-fdroid.cfg', - '-deststorepass:env', 'SMARTCARD_PIN', - '-J-Djava.security.debug=sunpkcs11'], - envs=env_vars) - if p.returncode != 0: - raise BuildException("Failed to import into HSM!", p.output) - - -if __name__ == "__main__": - main() diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py new file mode 100644 index 00000000..cb47f95f --- /dev/null +++ b/examples/makebuildserver.config.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# +# You may want to alter these before running ./makebuildserver + +# Name of the Vagrant basebox to use, by default it will be downloaded +# from Vagrant Cloud. For release builds setup, generate the basebox +# locally using https://gitlab.com/fdroid/basebox, add it to Vagrant, +# then set this to the local basebox name. +# This defaults to "fdroid/basebox-stretch64" which will download a +# prebuilt basebox from https://app.vagrantup.com/fdroid. +# +# (If you change this value you have to supply the `--clean` option on +# your next `makebuildserver` run.) +# +# basebox = "basebox-stretch64" + +# This allows you to pin your basebox to a specific versions. It defaults +# the most recent basebox version which can be aumotaically verifyed by +# `makebuildserver`. +# Please note that vagrant does not support versioning of locally added +# boxes, so we can't support that either. +# +# (If you change this value you have to supply the `--clean` option on +# your next `makebuildserver` run.) +# +# basebox_version = "0.1" + +# In the process of setting up the build server, many gigs of files +# are downloaded (Android SDK components, gradle, etc). These are +# cached so that they are not redownloaded each time. By default, +# these are stored in ~/.cache/fdroidserver +# +# cachedir = 'buildserver/cache' + +# A big part of creating a new instance is downloading packages from Debian. +# This setups up a folder in ~/.cache/fdroidserver to cache the downloaded +# packages when rebuilding the build server from scratch. This requires +# that virtualbox-guest-utils is installed. +# +# apt_package_cache = True + +# The buildserver can use some local caches to speed up builds, +# especially when the internet connection is slow and/or expensive. +# If enabled, the buildserver setup will look for standard caches in +# your HOME dir and copy them to the buildserver VM. Be aware: this +# will reduce the isolation of the buildserver from your host machine, +# so the buildserver will provide an environment only as trustworthy +# as the host machine's environment. +# +# copy_caches_from_host = True + +# To specify which Debian mirror the build server VM should use, by +# default it uses http.debian.net, which auto-detects which is the +# best mirror to use. +# +# debian_mirror = 'http://ftp.uk.debian.org/debian/' + +# The amount of RAM the build server will have (default: 2048) +# memory = 3584 + +# The number of CPUs the build server will have +# cpus = 1 + +# Debian package proxy server - if you have one +# aptproxy = "http://192.168.0.19:8000" + +# If this is running on an older machine or on a virtualized system, +# it can run a lot slower. If the provisioning fails with a warning +# about the timeout, extend the timeout here. (default: 600 seconds) +# +# boot_timeout = 1200 + +# By default, this whole process uses VirtualBox as the provider, but +# QEMU+KVM is also supported via the libvirt plugin to vagrant. If +# this is run within a KVM guest, then libvirt's QEMU+KVM will be used +# automatically. It can also be manually enabled by uncommenting +# below: +# +# vm_provider = 'libvirt' + +# By default libvirt uses 'virtio' for both network and disk drivers. +# Some systems (eg. nesting VMware ESXi) do not support virtio. As a +# workaround for such rare cases, this setting allows to configure +# KVM/libvirt to emulate hardware rather than using virtio. +# +# libvirt_disk_bus = 'sata' +# libvirt_nic_model_type = 'rtl8139' + +# Sometimes, it is not possible to use the 9p synced folder type with +# libvirt, like if running a KVM buildserver instance inside of a +# VMware ESXi guest. In that case, using NFS or another method is +# required. +# +# synced_folder_type = 'nfs' diff --git a/examples/mirror-to-mirror.sh b/examples/mirror-to-mirror.sh new file mode 100644 index 00000000..ecb4c10e --- /dev/null +++ b/examples/mirror-to-mirror.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# This script syncs the entire repo to the primary mirrors. It is +# meant to run in a cronjob quite frequently, as often as there are +# files to send. +# +# This script expects the receiving side to have the following +# preceeding the ssh key entry in ~/.ssh/authorized_keys: +# command="rsync --server -logDtpre.iLsfx --log-format=X --delete --delay-updates . /path/to/htdocs/fdroid/",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty +# +set -e +( +flock -n 200 +set -e +cd /home/fdroid +for section in repo archive; do + echo "Started $section at `date`:" + for host in fdroid@mirror.f-droid.org fdroid@ftp-push.lysator.liu.se; do + set -x + # be super careful with the trailing slashes here! if one is wrong, it'll delete the entire section! + rsync --archive --delay-updates --progress --delete \ + /home/fdroid/public_html/${section} \ + ${host}:/srv/fdroid-mirror.at.or.at/htdocs/fdroid/ & + set +x + done + wait +done +) 200>/var/lock/root_fdroidmirrortomirror diff --git a/fdroidserver/__init__.py b/fdroidserver/__init__.py index fdf64421..1e1cd42f 100644 --- a/fdroidserver/__init__.py +++ b/fdroidserver/__init__.py @@ -1,23 +1,20 @@ + import gettext import glob import os import sys + # support running straight from git and standard installs rootpaths = [ os.path.realpath(os.path.join(os.path.dirname(__file__), '..')), - os.path.realpath( - os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'share') - ), + os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'share')), os.path.join(sys.prefix, 'share'), ] localedir = None for rootpath in rootpaths: - found_mo = glob.glob( - os.path.join(rootpath, 'locale', '*', 'LC_MESSAGES', 'fdroidserver.mo') - ) - if len(found_mo) > 0: + if len(glob.glob(os.path.join(rootpath, 'locale', '*', 'LC_MESSAGES', 'fdroidserver.mo'))) > 0: localedir = os.path.join(rootpath, 'locale') break @@ -26,53 +23,34 @@ gettext.textdomain('fdroidserver') _ = gettext.gettext -from fdroidserver.exception import ( - FDroidException, - MetaDataException, - VerificationException, # NOQA: E402 -) - +from fdroidserver.exception import (FDroidException, + MetaDataException, + VerificationException) # NOQA: E402 FDroidException # NOQA: B101 MetaDataException # NOQA: B101 VerificationException # NOQA: B101 -from fdroidserver.common import genkeystore as generate_keystore # NOQA: E402 -from fdroidserver.common import verify_apk_signature - +from fdroidserver.common import (verify_apk_signature, + genkeystore as generate_keystore) # NOQA: E402 verify_apk_signature # NOQA: B101 generate_keystore # NOQA: B101 -from fdroidserver.index import ( - download_repo_index, - download_repo_index_v1, - download_repo_index_v2, - get_mirror_service_urls, -) -from fdroidserver.index import make as make_index # NOQA: E402 - +from fdroidserver.index import (download_repo_index, + get_mirror_service_urls, + make as make_index) # NOQA: E402 download_repo_index # NOQA: B101 -download_repo_index_v1 # NOQA: B101 -download_repo_index_v2 # NOQA: B101 get_mirror_service_urls # NOQA: B101 make_index # NOQA: B101 -from fdroidserver.update import ( - process_apk, - process_apks, - scan_apk, - scan_repo_files, # NOQA: E402 -) - +from fdroidserver.update import (process_apk, + process_apks, + scan_apk, + scan_repo_files) # NOQA: E402 process_apk # NOQA: B101 process_apks # NOQA: B101 scan_apk # NOQA: B101 scan_repo_files # NOQA: B101 -from fdroidserver.deploy import ( - update_awsbucket, - update_servergitmirrors, - update_serverwebroot, # NOQA: E402 - update_serverwebroots, -) - +from fdroidserver.deploy import (update_awsbucket, + update_servergitmirrors, + update_serverwebroot) # NOQA: E402 update_awsbucket # NOQA: B101 update_servergitmirrors # NOQA: B101 -update_serverwebroots # NOQA: B101 update_serverwebroot # NOQA: B101 diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 71c39b2c..d90a4316 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -18,20 +18,19 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import importlib.metadata -import logging -import os -import pkgutil import re import sys -from argparse import ArgumentError -from collections import OrderedDict - -import git +import os +import locale +import pkgutil +import logging import fdroidserver.common import fdroidserver.metadata from fdroidserver import _ +from argparse import ArgumentError +from collections import OrderedDict + COMMANDS = OrderedDict([ ("build", _("Build a package from source")), @@ -42,12 +41,13 @@ COMMANDS = OrderedDict([ ("deploy", _("Interact with the repo HTTP server")), ("verify", _("Verify the integrity of downloaded packages")), ("checkupdates", _("Check for updates to applications")), - ("import", _("Extract application metadata from a source repository")), + ("import", _("Add a new application from its source code")), ("install", _("Install built packages on devices")), ("readmeta", _("Read all the metadata files and exit")), ("rewritemeta", _("Rewrite all the metadata files")), ("lint", _("Warn about possible metadata errors")), ("scanner", _("Scan the source code of a package")), + ("stats", _("Update the stats of the repo")), ("signindex", _("Sign indexes created using update --nosign")), ("btlog", _("Update the binary transparency log for a URL")), ("signatures", _("Extract signatures from APKs")), @@ -70,13 +70,9 @@ def print_help(available_plugins=None): def preparse_plugin(module_name, module_dir): - """No summary. - - Simple regex based parsing for plugin scripts. - - So we don't have to import them when we just need the summary, - but not plan on executing this particular plugin. - """ + """simple regex based parsing for plugin scripts, + so we don't have to import them when we just need the summary, + but not plan on executing this particular plugin.""" if '.' in module_name: raise ValueError("No '.' allowed in fdroid plugin modules: '{}'" .format(module_name)) @@ -136,7 +132,7 @@ def main(): sys.exit(0) command = sys.argv[1] - if command not in COMMANDS and command not in available_plugins: + if command not in COMMANDS and command not in available_plugins.keys(): if command in ('-h', '--help'): print_help(available_plugins=available_plugins) sys.exit(0) @@ -144,21 +140,32 @@ def main(): print(_("""ERROR: The "server" subcommand has been removed, use "deploy"!""")) sys.exit(1) elif command == '--version': - try: - print(importlib.metadata.version("fdroidserver")) - sys.exit(0) - except importlib.metadata.PackageNotFoundError: - pass - try: - print( - git.repo.Repo( - os.path.dirname(os.path.dirname(__file__)) - ).git.describe(always=True, tags=True) - ) - sys.exit(0) - except git.exc.InvalidGitRepositoryError: - print(_('No version information could be found.')) - sys.exit(1) + output = _('no version info found!') + cmddir = os.path.realpath(os.path.dirname(os.path.dirname(__file__))) + moduledir = os.path.realpath(os.path.dirname(fdroidserver.common.__file__) + '/..') + if cmddir == moduledir: + # running from git + os.chdir(cmddir) + if os.path.isdir('.git'): + import subprocess + try: + output = subprocess.check_output(['git', 'describe'], + stderr=subprocess.STDOUT, + universal_newlines=True) + except subprocess.CalledProcessError: + output = 'git commit ' + subprocess.check_output(['git', 'rev-parse', 'HEAD'], + universal_newlines=True) + elif os.path.exists('setup.py'): + import re + m = re.search(r'''.*[\s,\(]+version\s*=\s*["']([0-9a-z.]+)["'].*''', + open('setup.py').read(), flags=re.MULTILINE) + if m: + output = m.group(1) + '\n' + else: + from pkg_resources import get_distribution + output = get_distribution('fdroidserver').version + '\n' + print(output) + sys.exit(0) else: print(_("Command '%s' not recognised.\n" % command)) print_help(available_plugins=available_plugins) @@ -182,18 +189,16 @@ def main(): "can not be specified at the same time.")) sys.exit(1) - # Trick argparse into displaying the right usage when --help is used. + # Trick optparse into displaying the right usage when --help is used. sys.argv[0] += ' ' + command del sys.argv[1] if command in COMMANDS.keys(): - # import is named import_subcommand internally b/c import is reserved by Python - command = 'import_subcommand' if command == 'import' else command mod = __import__('fdroidserver.' + command, None, None, [command]) else: mod = __import__(available_plugins[command]['name'], None, None, [command]) - system_encoding = sys.getdefaultencoding() + system_langcode, system_encoding = locale.getdefaultlocale() if system_encoding is None or system_encoding.lower() not in ('utf-8', 'utf8'): logging.warning(_("Encoding is set to '{enc}' fdroid might run " "into encoding issues. Please set it to 'UTF-8' " diff --git a/fdroidserver/_yaml.py b/fdroidserver/_yaml.py deleted file mode 100644 index 260f67c0..00000000 --- a/fdroidserver/_yaml.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (C) 2025, Hans-Christoph Steiner -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Standard YAML parsing and dumping. - -YAML 1.2 is the preferred format for all data files. When loading -F-Droid formats like config.yml and .yml, YAML 1.2 is -forced, and older YAML constructs should be considered an error. - -It is OK to load and dump files in other YAML versions if they are -externally defined formats, like FUNDING.yml. In those cases, these -common instances might not be appropriate to use. - -There is a separate instance for dumping based on the "round trip" aka -"rt" mode. The "rt" mode maintains order while the "safe" mode sorts -the output. Also, yaml.version is not forced in the dumper because that -makes it write out a "%YAML 1.2" header. F-Droid's formats are -explicitly defined as YAML 1.2 and meant to be human-editable. So that -header gets in the way. - -""" - -import ruamel.yaml - -yaml = ruamel.yaml.YAML(typ='safe') -yaml.version = (1, 2) - -yaml_dumper = ruamel.yaml.YAML(typ='rt') - - -def config_dump(config, fp=None): - """Dump config data in YAML 1.2 format without headers. - - This outputs YAML in a string that is suitable for use in regexps - and string replacements, as well as complete files. It is therefore - explicitly set up to avoid writing out headers and footers. - - This is modeled after PyYAML's yaml.dump(), which can dump to a file - or return a string. - - https://yaml.dev/doc/ruamel.yaml/example/#Output_of_%60dump()%60_as_a_string - - """ - dumper = ruamel.yaml.YAML(typ='rt') - dumper.default_flow_style = False - dumper.explicit_start = False - dumper.explicit_end = False - if fp is None: - with ruamel.yaml.compat.StringIO() as fp: - dumper.dump(config, fp) - return fp.getvalue() - dumper.dump(config, fp) diff --git a/fdroidserver/apksigcopier.py b/fdroidserver/apksigcopier.py deleted file mode 100644 index f36de2eb..00000000 --- a/fdroidserver/apksigcopier.py +++ /dev/null @@ -1,1019 +0,0 @@ -#!/usr/bin/python3 -# encoding: utf-8 -# SPDX-FileCopyrightText: 2023 FC Stegerman -# SPDX-License-Identifier: GPL-3.0-or-later - -# -- ; {{{1 -# -# File : apksigcopier -# Maintainer : FC Stegerman -# Date : 2023-02-08 -# -# Copyright : Copyright (C) 2023 FC Stegerman -# Version : v1.1.1 -# License : GPLv3+ -# -# -- ; }}}1 - -""" -Copy/extract/patch android apk signatures & compare apks. - -apksigcopier is a tool for copying android APK signatures from a signed APK to -an unsigned one (in order to verify reproducible builds). - -It can also be used to compare two APKs with different signatures; this requires -apksigner. - - -CLI -=== - -$ apksigcopier extract [OPTIONS] SIGNED_APK OUTPUT_DIR -$ apksigcopier patch [OPTIONS] METADATA_DIR UNSIGNED_APK OUTPUT_APK -$ apksigcopier copy [OPTIONS] SIGNED_APK UNSIGNED_APK OUTPUT_APK -$ apksigcopier compare [OPTIONS] FIRST_APK SECOND_APK - -The following environment variables can be set to 1, yes, or true to -override the default behaviour: - -* set APKSIGCOPIER_EXCLUDE_ALL_META=1 to exclude all metadata files -* set APKSIGCOPIER_COPY_EXTRA_BYTES=1 to copy extra bytes after data (e.g. a v2 sig) -* set APKSIGCOPIER_SKIP_REALIGNMENT=1 to skip realignment of ZIP entries - - -API -=== - ->> from apksigcopier import do_extract, do_patch, do_copy, do_compare ->> do_extract(signed_apk, output_dir, v1_only=NO) ->> do_patch(metadata_dir, unsigned_apk, output_apk, v1_only=NO) ->> do_copy(signed_apk, unsigned_apk, output_apk, v1_only=NO) ->> do_compare(first_apk, second_apk, unsigned=False) - -You can use False, None, and True instead of NO, AUTO, and YES respectively. - -The following global variables (which default to False), can be set to -override the default behaviour: - -* set exclude_all_meta=True to exclude all metadata files -* set copy_extra_bytes=True to copy extra bytes after data (e.g. a v2 sig) -* set skip_realignment=True to skip realignment of ZIP entries -""" - -import glob -import json -import os -import re -import struct -import sys -import zipfile -import zlib -from collections import namedtuple -from typing import ( - Any, - BinaryIO, - Callable, - Dict, - Iterable, - Iterator, - Optional, - Tuple, - Union, -) - -__version__ = "1.1.1" -NAME = "apksigcopier" - -if sys.version_info >= (3, 8): - from typing import Literal - NoAutoYes = Literal["no", "auto", "yes"] -else: - NoAutoYes = str - -DateTime = Tuple[int, int, int, int, int, int] -NoAutoYesBoolNone = Union[NoAutoYes, bool, None] -ZipInfoDataPairs = Iterable[Tuple[zipfile.ZipInfo, bytes]] - -SIGBLOCK, SIGOFFSET = "APKSigningBlock", "APKSigningBlockOffset" -NOAUTOYES: Tuple[NoAutoYes, NoAutoYes, NoAutoYes] = ("no", "auto", "yes") -NO, AUTO, YES = NOAUTOYES -APK_META = re.compile(r"^META-INF/([0-9A-Za-z_-]+\.(SF|RSA|DSA|EC)|MANIFEST\.MF)$") -META_EXT: Tuple[str, ...] = ("SF", "RSA|DSA|EC", "MF") -COPY_EXCLUDE: Tuple[str, ...] = ("META-INF/MANIFEST.MF",) -DATETIMEZERO: DateTime = (1980, 0, 0, 0, 0, 0) - -################################################################################ -# -# NB: these values are all from apksigner (the first element of each tuple, same -# as APKZipInfo) or signflinger/zipflinger, except for external_attr w/ 0664 -# permissions and flag_bits 0x08, added for completeness. -# -# NB: zipflinger changed from 0666 to 0644 in commit 895ba5fba6ab84617dd67e38f456a8f96aa37ff0 -# -# https://android.googlesource.com/platform/tools/apksig -# src/main/java/com/android/apksig/internal/zip/{CentralDirectoryRecord,LocalFileRecord,ZipUtils}.java -# https://android.googlesource.com/platform/tools/base -# signflinger/src/com/android/signflinger/SignedApk.java -# zipflinger/src/com/android/zipflinger/{CentralDirectoryRecord,LocalFileHeader,Source}.java -# -################################################################################ - -VALID_ZIP_META = dict( - compresslevel=(9, 1), # best compression, best speed - create_system=(0, 3), # fat, unx - create_version=(20, 0), # 2.0, 0.0 - external_attr=(0, # N/A - 0o100644 << 16, # regular file rw-r--r-- - 0o100664 << 16, # regular file rw-rw-r-- - 0o100666 << 16), # regular file rw-rw-rw- - extract_version=(20, 0), # 2.0, 0.0 - flag_bits=(0x800, 0, 0x08, 0x808), # 0x800 = utf8, 0x08 = data_descriptor -) - -ZipData = namedtuple("ZipData", ("cd_offset", "eocd_offset", "cd_and_eocd")) - -exclude_all_meta = False # exclude all metadata files in copy_apk() -copy_extra_bytes = False # copy extra bytes after data in copy_apk() -skip_realignment = False # skip realignment of ZIP entries in copy_apk() - - -class APKSigCopierError(Exception): - """Base class for errors.""" - - -class APKSigningBlockError(APKSigCopierError): - """Something wrong with the APK Signing Block.""" - - -class NoAPKSigningBlock(APKSigningBlockError): - """APK Signing Block Missing.""" - - -class ZipError(APKSigCopierError): - """Something wrong with ZIP file.""" - - -# FIXME: is there a better alternative? -class ReproducibleZipInfo(zipfile.ZipInfo): - """Reproducible ZipInfo hack.""" - - _override: Dict[str, Any] = {} - - def __init__(self, zinfo: zipfile.ZipInfo, **override: Any) -> None: - # pylint: disable=W0231 - if override: - self._override = {**self._override, **override} - for k in self.__slots__: - if hasattr(zinfo, k): - setattr(self, k, getattr(zinfo, k)) - - def __getattribute__(self, name: str) -> Any: - if name != "_override": - try: - return self._override[name] - except KeyError: - pass - return object.__getattribute__(self, name) - - -# See VALID_ZIP_META -class APKZipInfo(ReproducibleZipInfo): - """Reproducible ZipInfo for APK files.""" - - COMPRESSLEVEL = 9 - - _override = dict( - compress_type=8, - create_system=0, - create_version=20, - date_time=DATETIMEZERO, - external_attr=0, - extract_version=20, - flag_bits=0x800, - ) - - -def noautoyes(value: NoAutoYesBoolNone) -> NoAutoYes: - """ - Turn False into NO, None into AUTO, and True into YES. - - >>> from apksigcopier import noautoyes, NO, AUTO, YES - >>> noautoyes(False) == NO == noautoyes(NO) - True - >>> noautoyes(None) == AUTO == noautoyes(AUTO) - True - >>> noautoyes(True) == YES == noautoyes(YES) - True - - """ - if isinstance(value, str): - if value not in NOAUTOYES: - raise ValueError("expected NO, AUTO, or YES") - return value - try: - return {False: NO, None: AUTO, True: YES}[value] - except KeyError: - raise ValueError("expected False, None, or True") # pylint: disable=W0707 - - -def is_meta(filename: str) -> bool: - """ - Check whether filename is a JAR metadata file. - - Returns whether filename is a v1 (JAR) signature file (.SF), signature block - file (.RSA, .DSA, or .EC), or manifest (MANIFEST.MF). - - See https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html - - >>> from apksigcopier import is_meta - >>> is_meta("classes.dex") - False - >>> is_meta("META-INF/CERT.SF") - True - >>> is_meta("META-INF/CERT.RSA") - True - >>> is_meta("META-INF/MANIFEST.MF") - True - >>> is_meta("META-INF/OOPS") - False - - """ - return APK_META.fullmatch(filename) is not None - - -def exclude_from_copying(filename: str) -> bool: - """ - Check whether to exclude a file during copy_apk(). - - Excludes filenames in COPY_EXCLUDE (i.e. MANIFEST.MF) by default; when - exclude_all_meta is set to True instead, excludes all metadata files as - matched by is_meta(). - - Directories are always excluded. - - >>> import apksigcopier - >>> from apksigcopier import exclude_from_copying - >>> exclude_from_copying("classes.dex") - False - >>> exclude_from_copying("foo/") - True - >>> exclude_from_copying("META-INF/") - True - >>> exclude_from_copying("META-INF/MANIFEST.MF") - True - >>> exclude_from_copying("META-INF/CERT.SF") - False - >>> exclude_from_copying("META-INF/OOPS") - False - - >>> apksigcopier.exclude_all_meta = True - >>> exclude_from_copying("classes.dex") - False - >>> exclude_from_copying("META-INF/") - True - >>> exclude_from_copying("META-INF/MANIFEST.MF") - True - >>> exclude_from_copying("META-INF/CERT.SF") - True - >>> exclude_from_copying("META-INF/OOPS") - False - - """ - return exclude_meta(filename) if exclude_all_meta else exclude_default(filename) - - -def exclude_default(filename: str) -> bool: - """ - Like exclude_from_copying(). - - Excludes directories and filenames in COPY_EXCLUDE (i.e. MANIFEST.MF). - """ - return is_directory(filename) or filename in COPY_EXCLUDE - - -def exclude_meta(filename: str) -> bool: - """Like exclude_from_copying(); excludes directories and all metadata files.""" - return is_directory(filename) or is_meta(filename) - - -def is_directory(filename: str) -> bool: - """ZIP entries with filenames that end with a '/' are directories.""" - return filename.endswith("/") - - -################################################################################ -# -# There is usually a 132-byte virtual entry at the start of an APK signed with a -# v1 signature by signflinger/zipflinger; almost certainly this is a default -# manifest ZIP entry created at initialisation, deleted (from the CD but not -# from the file) during v1 signing, and eventually replaced by a virtual entry. -# -# >>> (30 + len("META-INF/MANIFEST.MF") + -# ... len("Manifest-Version: 1.0\r\n" -# ... "Created-By: Android Gradle 7.1.3\r\n" -# ... "Built-By: Signflinger\r\n\r\n")) -# 132 -# -# NB: they could be a different size, depending on Created-By and Built-By. -# -# FIXME: could virtual entries occur elsewhere as well? -# -# https://android.googlesource.com/platform/tools/base -# signflinger/src/com/android/signflinger/SignedApk.java -# zipflinger/src/com/android/zipflinger/{LocalFileHeader,ZipArchive}.java -# -################################################################################ - -def zipflinger_virtual_entry(size: int) -> bytes: - """Create zipflinger virtual entry.""" - if size < 30: - raise ValueError("Minimum size for virtual entries is 30 bytes") - return ( - # header extract_version flag_bits - b"\x50\x4b\x03\x04" b"\x00\x00" b"\x00\x00" - # compress_type (1981,1,1,1,1,2) crc32 - b"\x00\x00" b"\x21\x08\x21\x02" b"\x00\x00\x00\x00" - # compress_size file_size filename length - b"\x00\x00\x00\x00" b"\x00\x00\x00\x00" b"\x00\x00" - ) + int.to_bytes(size - 30, 2, "little") + b"\x00" * (size - 30) - - -def detect_zfe(apkfile: str) -> Optional[int]: - """ - Detect zipflinger virtual entry. - - Returns the size of the virtual entry if found, None otherwise. - - Raises ZipError if the size is less than 30 or greater than 4096, or the - data isn't all zeroes. - """ - with open(apkfile, "rb") as fh: - zfe_start = zipflinger_virtual_entry(30)[:28] # w/o len(extra) - if fh.read(28) == zfe_start: - zfe_size = 30 + int.from_bytes(fh.read(2), "little") - if not (30 <= zfe_size <= 4096): - raise ZipError("Unsupported virtual entry size") - if not fh.read(zfe_size - 30) == b"\x00" * (zfe_size - 30): - raise ZipError("Unsupported virtual entry data") - return zfe_size - return None - - -################################################################################ -# -# https://en.wikipedia.org/wiki/ZIP_(file_format) -# https://source.android.com/docs/security/features/apksigning/v2#apk-signing-block-format -# -# ================================= -# | Contents of ZIP entries | -# ================================= -# | APK Signing Block | -# | ----------------------------- | -# | | size (w/o this) uint64 LE | | -# | | ... | | -# | | size (again) uint64 LE | | -# | | "APK Sig Block 42" (16B) | | -# | ----------------------------- | -# ================================= -# | ZIP Central Directory | -# ================================= -# | ZIP End of Central Directory | -# | ----------------------------- | -# | | 0x06054b50 ( 4B) | | -# | | ... (12B) | | -# | | CD Offset ( 4B) | | -# | | ... | | -# | ----------------------------- | -# ================================= -# -################################################################################ - - -# FIXME: makes certain assumptions and doesn't handle all valid ZIP files! -# FIXME: support zip64? -# FIXME: handle utf8 filenames w/o utf8 flag (as produced by zipflinger)? -# https://android.googlesource.com/platform/tools/apksig -# src/main/java/com/android/apksig/ApkSigner.java -def copy_apk(unsigned_apk: str, output_apk: str, *, - copy_extra: Optional[bool] = None, - exclude: Optional[Callable[[str], bool]] = None, - realign: Optional[bool] = None, - zfe_size: Optional[int] = None) -> DateTime: - """ - Copy APK like apksigner would, excluding files matched by exclude_from_copying(). - - Adds a zipflinger virtual entry of zfe_size bytes if one is not already - present and zfe_size is not None. - - Returns max date_time. - - The following global variables (which default to False), can be set to - override the default behaviour: - - * set exclude_all_meta=True to exclude all metadata files - * set copy_extra_bytes=True to copy extra bytes after data (e.g. a v2 sig) - * set skip_realignment=True to skip realignment of ZIP entries - - The default behaviour can also be changed using the keyword-only arguments - exclude, copy_extra, and realign; these take precedence over the global - variables when not None. NB: exclude is a callable, not a bool; realign is - the inverse of skip_realignment. - - >>> import apksigcopier, os, zipfile - >>> apk = "test/apks/apks/golden-aligned-in.apk" - >>> with zipfile.ZipFile(apk, "r") as zf: - ... infos_in = zf.infolist() - >>> with tempfile.TemporaryDirectory() as tmpdir: - ... out = os.path.join(tmpdir, "out.apk") - ... apksigcopier.copy_apk(apk, out) - ... with zipfile.ZipFile(out, "r") as zf: - ... infos_out = zf.infolist() - (2017, 5, 15, 11, 28, 40) - >>> for i in infos_in: - ... print(i.filename) - META-INF/ - META-INF/MANIFEST.MF - AndroidManifest.xml - classes.dex - temp.txt - lib/armeabi/fake.so - resources.arsc - temp2.txt - >>> for i in infos_out: - ... print(i.filename) - AndroidManifest.xml - classes.dex - temp.txt - lib/armeabi/fake.so - resources.arsc - temp2.txt - >>> infos_in[2] - - >>> infos_out[0] - - >>> repr(infos_in[2:]) == repr(infos_out) - True - - """ - if copy_extra is None: - copy_extra = copy_extra_bytes - if exclude is None: - exclude = exclude_from_copying - if realign is None: - realign = not skip_realignment - with zipfile.ZipFile(unsigned_apk, "r") as zf: - infos = zf.infolist() - zdata = zip_data(unsigned_apk) - offsets = {} - with open(unsigned_apk, "rb") as fhi, open(output_apk, "w+b") as fho: - if zfe_size: - zfe = zipflinger_virtual_entry(zfe_size) - if fhi.read(zfe_size) != zfe: - fho.write(zfe) - fhi.seek(0) - for info in sorted(infos, key=lambda info: info.header_offset): - off_i = fhi.tell() - if info.header_offset > off_i: - # copy extra bytes - fho.write(fhi.read(info.header_offset - off_i)) - hdr = fhi.read(30) - if hdr[:4] != b"\x50\x4b\x03\x04": - raise ZipError("Expected local file header signature") - n, m = struct.unpack(" bytes: - align = 4096 if info.filename.endswith(".so") else 4 - old_off = 30 + n + m + info.header_offset - new_off = 30 + n + m + off_o - old_xtr = hdr[30 + n:30 + n + m] - new_xtr = b"" - while len(old_xtr) >= 4: - hdr_id, size = struct.unpack(" len(old_xtr) - 4: - break - if not (hdr_id == 0 and size == 0): - if hdr_id == 0xd935: - if size >= 2: - align = int.from_bytes(old_xtr[4:6], "little") - else: - new_xtr += old_xtr[:size + 4] - old_xtr = old_xtr[size + 4:] - if old_off % align == 0 and new_off % align != 0: - if pad_like_apksigner: - pad = (align - (new_off - m + len(new_xtr) + 6) % align) % align - xtr = new_xtr + struct.pack(" None: - while size > 0: - data = fhi.read(min(size, blocksize)) - if not data: - break - size -= len(data) - fho.write(data) - if size != 0: - raise ZipError("Unexpected EOF") - - -def extract_meta(signed_apk: str) -> Iterator[Tuple[zipfile.ZipInfo, bytes]]: - """ - Extract v1 signature metadata files from signed APK. - - Yields (ZipInfo, data) pairs. - - >>> from apksigcopier import extract_meta - >>> apk = "test/apks/apks/golden-aligned-v1v2v3-out.apk" - >>> meta = tuple(extract_meta(apk)) - >>> [ x.filename for x, _ in meta ] - ['META-INF/RSA-2048.SF', 'META-INF/RSA-2048.RSA', 'META-INF/MANIFEST.MF'] - >>> for line in meta[0][1].splitlines()[:4]: - ... print(line.decode()) - Signature-Version: 1.0 - Created-By: 1.0 (Android) - SHA-256-Digest-Manifest: hz7AxDJU9Namxoou/kc4Z2GVRS9anCGI+M52tbCsXT0= - X-Android-APK-Signed: 2, 3 - >>> for line in meta[2][1].splitlines()[:2]: - ... print(line.decode()) - Manifest-Version: 1.0 - Created-By: 1.8.0_45-internal (Oracle Corporation) - - """ - with zipfile.ZipFile(signed_apk, "r") as zf_sig: - for info in zf_sig.infolist(): - if is_meta(info.filename): - yield info, zf_sig.read(info.filename) - - -def extract_differences(signed_apk: str, extracted_meta: ZipInfoDataPairs) \ - -> Optional[Dict[str, Any]]: - """ - Extract ZIP metadata differences from signed APK. - - >>> import apksigcopier as asc, pprint - >>> apk = "test/apks/apks/debuggable-boolean.apk" - >>> meta = tuple(asc.extract_meta(apk)) - >>> [ x.filename for x, _ in meta ] - ['META-INF/CERT.SF', 'META-INF/CERT.RSA', 'META-INF/MANIFEST.MF'] - >>> diff = asc.extract_differences(apk, meta) - >>> pprint.pprint(diff) - {'files': {'META-INF/CERT.RSA': {'flag_bits': 2056}, - 'META-INF/CERT.SF': {'flag_bits': 2056}, - 'META-INF/MANIFEST.MF': {'flag_bits': 2056}}} - - >>> meta[2][0].extract_version = 42 - >>> try: - ... asc.extract_differences(apk, meta) - ... except asc.ZipError as e: - ... print(e) - Unsupported extract_version - - >>> asc.validate_differences(diff) is None - True - >>> diff["files"]["META-INF/OOPS"] = {} - >>> asc.validate_differences(diff) - ".files key 'META-INF/OOPS' is not a metadata file" - >>> del diff["files"]["META-INF/OOPS"] - >>> diff["files"]["META-INF/CERT.RSA"]["compresslevel"] = 42 - >>> asc.validate_differences(diff) - ".files['META-INF/CERT.RSA'].compresslevel has an unexpected value" - >>> diff["oops"] = 42 - >>> asc.validate_differences(diff) - 'contains unknown key(s)' - - """ - differences: Dict[str, Any] = {} - files = {} - for info, data in extracted_meta: - diffs = {} - for k in VALID_ZIP_META: - if k != "compresslevel": - v = getattr(info, k) - if v != APKZipInfo._override[k]: - if v not in VALID_ZIP_META[k]: - raise ZipError(f"Unsupported {k}") - diffs[k] = v - level = _get_compresslevel(signed_apk, info, data) - if level != APKZipInfo.COMPRESSLEVEL: - diffs["compresslevel"] = level - if diffs: - files[info.filename] = diffs - if files: - differences["files"] = files - zfe_size = detect_zfe(signed_apk) - if zfe_size: - differences["zipflinger_virtual_entry"] = zfe_size - return differences or None - - -def validate_differences(differences: Dict[str, Any]) -> Optional[str]: - """ - Validate differences dict. - - Returns None if valid, error otherwise. - """ - if set(differences) - {"files", "zipflinger_virtual_entry"}: - return "contains unknown key(s)" - if "zipflinger_virtual_entry" in differences: - if type(differences["zipflinger_virtual_entry"]) is not int: - return ".zipflinger_virtual_entry is not an int" - if not (30 <= differences["zipflinger_virtual_entry"] <= 4096): - return ".zipflinger_virtual_entry is < 30 or > 4096" - if "files" in differences: - if not isinstance(differences["files"], dict): - return ".files is not a dict" - for name, info in differences["files"].items(): - if not is_meta(name): - return f".files key {name!r} is not a metadata file" - if not isinstance(info, dict): - return f".files[{name!r}] is not a dict" - if set(info) - set(VALID_ZIP_META): - return f".files[{name!r}] contains unknown key(s)" - for k, v in info.items(): - if v not in VALID_ZIP_META[k]: - return f".files[{name!r}].{k} has an unexpected value" - return None - - -def _get_compresslevel(apkfile: str, info: zipfile.ZipInfo, data: bytes) -> int: - if info.compress_type != 8: - raise ZipError("Unsupported compress_type") - crc = _get_compressed_crc(apkfile, info) - for level in VALID_ZIP_META["compresslevel"]: - comp = zlib.compressobj(level, 8, -15) - if zlib.crc32(comp.compress(data) + comp.flush()) == crc: - return level - raise ZipError("Unsupported compresslevel") - - -def _get_compressed_crc(apkfile: str, info: zipfile.ZipInfo) -> int: - with open(apkfile, "rb") as fh: - fh.seek(info.header_offset) - hdr = fh.read(30) - if hdr[:4] != b"\x50\x4b\x03\x04": - raise ZipError("Expected local file header signature") - n, m = struct.unpack(" None: - """ - Add v1 signature metadata to APK (removes v2 sig block, if any). - - >>> import apksigcopier as asc - >>> unsigned_apk = "test/apks/apks/golden-aligned-in.apk" - >>> signed_apk = "test/apks/apks/golden-aligned-v1v2v3-out.apk" - >>> meta = tuple(asc.extract_meta(signed_apk)) - >>> [ x.filename for x, _ in meta ] - ['META-INF/RSA-2048.SF', 'META-INF/RSA-2048.RSA', 'META-INF/MANIFEST.MF'] - >>> with zipfile.ZipFile(unsigned_apk, "r") as zf: - ... infos_in = zf.infolist() - >>> with tempfile.TemporaryDirectory() as tmpdir: - ... out = os.path.join(tmpdir, "out.apk") - ... asc.copy_apk(unsigned_apk, out) - ... asc.patch_meta(meta, out) - ... with zipfile.ZipFile(out, "r") as zf: - ... infos_out = zf.infolist() - (2017, 5, 15, 11, 28, 40) - >>> for i in infos_in: - ... print(i.filename) - META-INF/ - META-INF/MANIFEST.MF - AndroidManifest.xml - classes.dex - temp.txt - lib/armeabi/fake.so - resources.arsc - temp2.txt - >>> for i in infos_out: - ... print(i.filename) - AndroidManifest.xml - classes.dex - temp.txt - lib/armeabi/fake.so - resources.arsc - temp2.txt - META-INF/RSA-2048.SF - META-INF/RSA-2048.RSA - META-INF/MANIFEST.MF - - """ - with zipfile.ZipFile(output_apk, "r") as zf_out: - for info in zf_out.infolist(): - if is_meta(info.filename): - raise ZipError("Unexpected metadata") - with zipfile.ZipFile(output_apk, "a") as zf_out: - for info, data in extracted_meta: - if differences and "files" in differences: - more = differences["files"].get(info.filename, {}).copy() - else: - more = {} - level = more.pop("compresslevel", APKZipInfo.COMPRESSLEVEL) - zinfo = APKZipInfo(info, date_time=date_time, **more) - zf_out.writestr(zinfo, data, compresslevel=level) - - -def extract_v2_sig(apkfile: str, expected: bool = True) -> Optional[Tuple[int, bytes]]: - """ - Extract APK Signing Block and offset from APK. - - When successful, returns (sb_offset, sig_block); otherwise raises - NoAPKSigningBlock when expected is True, else returns None. - - >>> import apksigcopier as asc - >>> apk = "test/apks/apks/golden-aligned-v1v2v3-out.apk" - >>> sb_offset, sig_block = asc.extract_v2_sig(apk) - >>> sb_offset - 8192 - >>> len(sig_block) - 4096 - - >>> apk = "test/apks/apks/golden-aligned-in.apk" - >>> try: - ... asc.extract_v2_sig(apk) - ... except asc.NoAPKSigningBlock as e: - ... print(e) - No APK Signing Block - - """ - cd_offset = zip_data(apkfile).cd_offset - with open(apkfile, "rb") as fh: - fh.seek(cd_offset - 16) - if fh.read(16) != b"APK Sig Block 42": - if expected: - raise NoAPKSigningBlock("No APK Signing Block") - return None - fh.seek(-24, os.SEEK_CUR) - sb_size2 = int.from_bytes(fh.read(8), "little") - fh.seek(-sb_size2 + 8, os.SEEK_CUR) - sb_size1 = int.from_bytes(fh.read(8), "little") - if sb_size1 != sb_size2: - raise APKSigningBlockError("APK Signing Block sizes not equal") - fh.seek(-8, os.SEEK_CUR) - sb_offset = fh.tell() - sig_block = fh.read(sb_size2 + 8) - return sb_offset, sig_block - - -# FIXME: OSError for APKs < 1024 bytes [wontfix] -def zip_data(apkfile: str, count: int = 1024) -> ZipData: - """ - Extract central directory, EOCD, and offsets from ZIP. - - Returns ZipData. - - >>> import apksigcopier - >>> apk = "test/apks/apks/golden-aligned-v1v2v3-out.apk" - >>> data = apksigcopier.zip_data(apk) - >>> data.cd_offset, data.eocd_offset - (12288, 12843) - >>> len(data.cd_and_eocd) - 577 - - """ - with open(apkfile, "rb") as fh: - fh.seek(-count, os.SEEK_END) - data = fh.read() - pos = data.rfind(b"\x50\x4b\x05\x06") - if pos == -1: - raise ZipError("Expected end of central directory record (EOCD)") - fh.seek(pos - len(data), os.SEEK_CUR) - eocd_offset = fh.tell() - fh.seek(16, os.SEEK_CUR) - cd_offset = int.from_bytes(fh.read(4), "little") - fh.seek(cd_offset) - cd_and_eocd = fh.read() - return ZipData(cd_offset, eocd_offset, cd_and_eocd) - - -# FIXME: can we determine signed_sb_offset? -def patch_v2_sig(extracted_v2_sig: Tuple[int, bytes], output_apk: str) -> None: - """ - Implant extracted v2/v3 signature into APK. - - >>> import apksigcopier as asc - >>> unsigned_apk = "test/apks/apks/golden-aligned-in.apk" - >>> signed_apk = "test/apks/apks/golden-aligned-v1v2v3-out.apk" - >>> meta = tuple(asc.extract_meta(signed_apk)) - >>> v2_sig = asc.extract_v2_sig(signed_apk) - >>> with tempfile.TemporaryDirectory() as tmpdir: - ... out = os.path.join(tmpdir, "out.apk") - ... date_time = asc.copy_apk(unsigned_apk, out) - ... asc.patch_meta(meta, out, date_time=date_time) - ... asc.extract_v2_sig(out, expected=False) is None - ... asc.patch_v2_sig(v2_sig, out) - ... asc.extract_v2_sig(out) == v2_sig - ... with open(signed_apk, "rb") as a, open(out, "rb") as b: - ... a.read() == b.read() - True - True - True - - """ - signed_sb_offset, signed_sb = extracted_v2_sig - data_out = zip_data(output_apk) - if signed_sb_offset < data_out.cd_offset: - raise APKSigningBlockError("APK Signing Block offset < central directory offset") - padding = b"\x00" * (signed_sb_offset - data_out.cd_offset) - offset = len(signed_sb) + len(padding) - with open(output_apk, "r+b") as fh: - fh.seek(data_out.cd_offset) - fh.write(padding) - fh.write(signed_sb) - fh.write(data_out.cd_and_eocd) - fh.seek(data_out.eocd_offset + offset + 16) - fh.write(int.to_bytes(data_out.cd_offset + offset, 4, "little")) - - -def patch_apk(extracted_meta: ZipInfoDataPairs, extracted_v2_sig: Optional[Tuple[int, bytes]], - unsigned_apk: str, output_apk: str, *, - differences: Optional[Dict[str, Any]] = None, - exclude: Optional[Callable[[str], bool]] = None) -> None: - """Patch extracted_meta + extracted_v2_sig (if not None) onto unsigned_apk and save as output_apk.""" - if differences and "zipflinger_virtual_entry" in differences: - zfe_size = differences["zipflinger_virtual_entry"] - else: - zfe_size = None - date_time = copy_apk(unsigned_apk, output_apk, exclude=exclude, zfe_size=zfe_size) - patch_meta(extracted_meta, output_apk, date_time=date_time, differences=differences) - if extracted_v2_sig is not None: - patch_v2_sig(extracted_v2_sig, output_apk) - - -# FIXME: support multiple signers? -def do_extract(signed_apk: str, output_dir: str, v1_only: NoAutoYesBoolNone = NO, - *, ignore_differences: bool = False) -> None: - """ - Extract signatures from signed_apk and save in output_dir. - - The v1_only parameter controls whether the absence of a v1 signature is - considered an error or not: - * use v1_only=NO (or v1_only=False) to only accept (v1+)v2/v3 signatures; - * use v1_only=AUTO (or v1_only=None) to automatically detect v2/v3 signatures; - * use v1_only=YES (or v1_only=True) to ignore any v2/v3 signatures. - """ - v1_only = noautoyes(v1_only) - extracted_meta = tuple(extract_meta(signed_apk)) - if len(extracted_meta) not in (len(META_EXT), 0): - raise APKSigCopierError("Unexpected or missing metadata files in signed_apk") - for info, data in extracted_meta: - name = os.path.basename(info.filename) - with open(os.path.join(output_dir, name), "wb") as fh: - fh.write(data) - if v1_only == YES: - if not extracted_meta: - raise APKSigCopierError("Expected v1 signature") - return - expected = v1_only == NO - extracted_v2_sig = extract_v2_sig(signed_apk, expected=expected) - if extracted_v2_sig is None: - if not extracted_meta: - raise APKSigCopierError("Expected v1 and/or v2/v3 signature, found neither") - return - signed_sb_offset, signed_sb = extracted_v2_sig - with open(os.path.join(output_dir, SIGOFFSET), "w") as fh: - fh.write(str(signed_sb_offset) + "\n") - with open(os.path.join(output_dir, SIGBLOCK), "wb") as fh: - fh.write(signed_sb) - if not ignore_differences: - differences = extract_differences(signed_apk, extracted_meta) - if differences: - with open(os.path.join(output_dir, "differences.json"), "w") as fh: - json.dump(differences, fh, sort_keys=True, indent=2) - fh.write("\n") - - -# FIXME: support multiple signers? -def do_patch(metadata_dir: str, unsigned_apk: str, output_apk: str, - v1_only: NoAutoYesBoolNone = NO, *, - exclude: Optional[Callable[[str], bool]] = None, - ignore_differences: bool = False) -> None: - """ - Patch signatures from metadata_dir onto unsigned_apk and save as output_apk. - - The v1_only parameter controls whether the absence of a v1 signature is - considered an error or not: - * use v1_only=NO (or v1_only=False) to only accept (v1+)v2/v3 signatures; - * use v1_only=AUTO (or v1_only=None) to automatically detect v2/v3 signatures; - * use v1_only=YES (or v1_only=True) to ignore any v2/v3 signatures. - """ - v1_only = noautoyes(v1_only) - extracted_meta = [] - differences = None - for pat in META_EXT: - files = [fn for ext in pat.split("|") for fn in - glob.glob(os.path.join(metadata_dir, "*." + ext))] - if len(files) != 1: - continue - info = zipfile.ZipInfo("META-INF/" + os.path.basename(files[0])) - with open(files[0], "rb") as fh: - extracted_meta.append((info, fh.read())) - if len(extracted_meta) not in (len(META_EXT), 0): - raise APKSigCopierError("Unexpected or missing files in metadata_dir") - if v1_only == YES: - extracted_v2_sig = None - else: - sigoffset_file = os.path.join(metadata_dir, SIGOFFSET) - sigblock_file = os.path.join(metadata_dir, SIGBLOCK) - if v1_only == AUTO and not os.path.exists(sigblock_file): - extracted_v2_sig = None - else: - with open(sigoffset_file, "r") as fh: - signed_sb_offset = int(fh.read()) - with open(sigblock_file, "rb") as fh: - signed_sb = fh.read() - extracted_v2_sig = signed_sb_offset, signed_sb - differences_file = os.path.join(metadata_dir, "differences.json") - if not ignore_differences and os.path.exists(differences_file): - with open(differences_file, "r") as fh: - try: - differences = json.load(fh) - except json.JSONDecodeError as e: - raise APKSigCopierError(f"Invalid differences.json: {e}") # pylint: disable=W0707 - error = validate_differences(differences) - if error: - raise APKSigCopierError(f"Invalid differences.json: {error}") - if not extracted_meta and extracted_v2_sig is None: - raise APKSigCopierError("Expected v1 and/or v2/v3 signature, found neither") - patch_apk(extracted_meta, extracted_v2_sig, unsigned_apk, output_apk, - differences=differences, exclude=exclude) - - -def do_copy(signed_apk: str, unsigned_apk: str, output_apk: str, - v1_only: NoAutoYesBoolNone = NO, *, - exclude: Optional[Callable[[str], bool]] = None, - ignore_differences: bool = False) -> None: - """ - Copy signatures from signed_apk onto unsigned_apk and save as output_apk. - - The v1_only parameter controls whether the absence of a v1 signature is - considered an error or not: - * use v1_only=NO (or v1_only=False) to only accept (v1+)v2/v3 signatures; - * use v1_only=AUTO (or v1_only=None) to automatically detect v2/v3 signatures; - * use v1_only=YES (or v1_only=True) to ignore any v2/v3 signatures. - """ - v1_only = noautoyes(v1_only) - extracted_meta = tuple(extract_meta(signed_apk)) - differences = None - if v1_only == YES: - extracted_v2_sig = None - else: - extracted_v2_sig = extract_v2_sig(signed_apk, expected=v1_only == NO) - if extracted_v2_sig is not None and not ignore_differences: - differences = extract_differences(signed_apk, extracted_meta) - patch_apk(extracted_meta, extracted_v2_sig, unsigned_apk, output_apk, - differences=differences, exclude=exclude) - -# vim: set tw=80 sw=4 sts=4 et fdm=marker : diff --git a/fdroidserver/asynchronousfilereader/__init__.py b/fdroidserver/asynchronousfilereader/__init__.py index 7ba02b69..e8aa35e5 100644 --- a/fdroidserver/asynchronousfilereader/__init__.py +++ b/fdroidserver/asynchronousfilereader/__init__.py @@ -1,8 +1,9 @@ -"""Simple thread based asynchronous file reader for Python. - +""" AsynchronousFileReader ====================== +Simple thread based asynchronous file reader for Python. + see https://github.com/soxofaan/asynchronousfilereader MIT License @@ -12,7 +13,6 @@ Copyright (c) 2014 Stefaan Lippens __version__ = '0.2.1' import threading - try: # Python 2 from Queue import Queue @@ -22,9 +22,10 @@ except ImportError: class AsynchronousFileReader(threading.Thread): - """Helper class to implement asynchronous reading of a file in a separate thread. - - Pushes read lines on a queue to be consumed in another thread. + """ + Helper class to implement asynchronous reading of a file + in a separate thread. Pushes read lines on a queue to + be consumed in another thread. """ def __init__(self, fd, queue=None, autostart=True): @@ -39,7 +40,9 @@ class AsynchronousFileReader(threading.Thread): self.start() def run(self): - """Read lines and put them on the queue (the body of the tread).""" + """ + The body of the tread: read lines and put them on the queue. + """ while True: line = self._fd.readline() if not line: @@ -47,10 +50,15 @@ class AsynchronousFileReader(threading.Thread): self.queue.put(line) def eof(self): - """Check whether there is no more content to expect.""" + """ + Check whether there is no more content to expect. + """ return not self.is_alive() and self.queue.empty() def readlines(self): - """Get currently available lines.""" + """ + Get currently available lines. + """ while not self.queue.empty(): yield self.queue.get() + diff --git a/fdroidserver/btlog.py b/fdroidserver/btlog.py index 7ca3ddbf..f1e83c90 100755 --- a/fdroidserver/btlog.py +++ b/fdroidserver/btlog.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -"""Update the binary transparency log for a URL.""" # # btlog.py - part of the FDroid server tools # Copyright (C) 2017, Hans-Christoph Steiner @@ -27,64 +26,51 @@ # client app so its not easy for the server to distinguish this from # the F-Droid client. + import collections +import defusedxml.minidom +import git import glob +import os import json import logging -import os +import requests import shutil import tempfile import zipfile from argparse import ArgumentParser -from typing import Optional -import defusedxml.minidom -import git -import requests - -from . import _, common, deploy +from . import _ +from . import common +from . import deploy from .exception import FDroidException -def make_binary_transparency_log( - repodirs: collections.abc.Iterable, - btrepo: str = 'binary_transparency', - url: Optional[str] = None, - commit_title: str = 'fdroid update', -): - """Log the indexes in a standalone git repo to serve as a "binary transparency" log. +options = None - Parameters - ---------- - repodirs - The directories of the F-Droid repository to generate the binary - transparency log for. - btrepo - The path to the Git repository of the binary transparency log. - url - The URL of the F-Droid repository to generate the binary transparency - log for. - commit_title - The commit title for commits in the binary transparency log Git - repository. - Notes - ----- - Also see https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies . - """ +def make_binary_transparency_log(repodirs, btrepo='binary_transparency', + url=None, + commit_title='fdroid update'): + '''Log the indexes in a standalone git repo to serve as a "binary + transparency" log. + + see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies + + ''' + logging.info('Committing indexes to ' + btrepo) if os.path.exists(os.path.join(btrepo, '.git')): gitrepo = git.Repo(btrepo) else: if not os.path.exists(btrepo): os.mkdir(btrepo) - gitrepo = git.Repo.init(btrepo, initial_branch=deploy.GIT_BRANCH) + gitrepo = git.Repo.init(btrepo) if not url: url = common.config['repo_url'].rstrip('/') with open(os.path.join(btrepo, 'README.md'), 'w') as fp: - fp.write( - """ + fp.write(""" # Binary Transparency Log for %s This is a log of the signed app index metadata. This is stored in a @@ -94,17 +80,15 @@ F-Droid repository was a publicly released file. For more info on this idea: * https://wiki.mozilla.org/Security/Binary_Transparency -""" - % url[: url.rindex('/')] # strip '/repo' - ) - gitrepo.index.add(['README.md']) +""" % url[:url.rindex('/')]) # strip '/repo' + gitrepo.index.add(['README.md', ]) gitrepo.index.commit('add README') for repodir in repodirs: cpdir = os.path.join(btrepo, repodir) if not os.path.exists(cpdir): os.mkdir(cpdir) - for f in ('index.xml', 'index-v1.json', 'index-v2.json', 'entry.json'): + for f in ('index.xml', 'index-v1.json'): repof = os.path.join(repodir, f) if not os.path.exists(repof): continue @@ -119,8 +103,8 @@ For more info on this idea: output = json.load(fp, object_pairs_hook=collections.OrderedDict) with open(dest, 'w') as fp: json.dump(output, fp, indent=2) - gitrepo.index.add([repof]) - for f in ('index.jar', 'index-v1.jar', 'entry.jar'): + gitrepo.index.add([repof, ]) + for f in ('index.jar', 'index-v1.jar'): repof = os.path.join(repodir, f) if not os.path.exists(repof): continue @@ -132,7 +116,7 @@ For more info on this idea: jarout.writestr(info, jarin.read(info.filename)) jarout.close() jarin.close() - gitrepo.index.add([repof]) + gitrepo.index.add([repof, ]) output_files = [] for root, dirs, files in os.walk(repodir): @@ -153,45 +137,27 @@ For more info on this idea: fslogfile = os.path.join(cpdir, 'filesystemlog.json') with open(fslogfile, 'w') as fp: json.dump(output, fp, indent=2) - gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json')]) + gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ]) for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')): - gitrepo.index.add([os.path.join(repodir, os.path.basename(f))]) + gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ]) gitrepo.index.commit(commit_title) def main(): - """Generate or update a binary transparency log for a F-Droid repository. + global options - The behaviour of this function is influenced by the configuration file as - well as command line parameters. - - Raises - ------ - :exc:`~fdroidserver.exception.FDroidException` - If the specified or default Git repository does not exist. - - """ parser = ArgumentParser() common.setup_global_opts(parser) - parser.add_argument( - "--git-repo", - default=os.path.join(os.getcwd(), 'binary_transparency'), - help=_("Path to the git repo to use as the log"), - ) - parser.add_argument( - "-u", - "--url", - default='https://f-droid.org', - help=_("The base URL for the repo to log (default: https://f-droid.org)"), - ) - parser.add_argument( - "--git-remote", - default=None, - help=_("Push the log to this git remote repository"), - ) - options = common.parse_args(parser) + parser.add_argument("--git-repo", + default=os.path.join(os.getcwd(), 'binary_transparency'), + help=_("Path to the git repo to use as the log")) + parser.add_argument("-u", "--url", default='https://f-droid.org', + help=_("The base URL for the repo to log (default: https://f-droid.org)")) + parser.add_argument("--git-remote", default=None, + help=_("Push the log to this git remote repository")) + options = parser.parse_args() if options.verbose: logging.getLogger("requests").setLevel(logging.INFO) @@ -202,8 +168,7 @@ def main(): if not os.path.exists(options.git_repo): raise FDroidException( - '"%s" does not exist! Create it, or use --git-repo' % options.git_repo - ) + '"%s" does not exist! Create it, or use --git-repo' % options.git_repo) session = requests.Session() @@ -216,20 +181,14 @@ def main(): os.makedirs(tempdir, exist_ok=True) gitrepodir = os.path.join(options.git_repo, repodir) os.makedirs(gitrepodir, exist_ok=True) - for f in ( - 'entry.jar', - 'entry.json', - 'index-v1.jar', - 'index-v1.json', - 'index-v2.json', - 'index.jar', - 'index.xml', - ): + for f in ('index.jar', 'index.xml', 'index-v1.jar', 'index-v1.json'): dlfile = os.path.join(tempdir, f) dlurl = options.url + '/' + repodir + '/' + f http_headers_file = os.path.join(gitrepodir, f + '.HTTP-headers.json') - headers = {'User-Agent': 'F-Droid 0.102.3'} + headers = { + 'User-Agent': 'F-Droid 0.102.3' + } etag = None if os.path.exists(http_headers_file): with open(http_headers_file) as fp: @@ -237,9 +196,7 @@ def main(): r = session.head(dlurl, headers=headers, allow_redirects=False) if r.status_code != 200: - logging.debug( - 'HTTP Response (%d), did not download %s' % (r.status_code, dlurl) - ) + logging.debug('HTTP Response (' + str(r.status_code) + '), did not download ' + dlurl) continue if etag and etag == r.headers.get('ETag'): logging.debug('ETag matches, did not download ' + dlurl) @@ -260,9 +217,7 @@ def main(): if new_files: os.chdir(tempdirbase) - make_binary_transparency_log( - repodirs, options.git_repo, options.url, 'fdroid btlog' - ) + make_binary_transparency_log(repodirs, options.git_repo, options.url, 'fdroid btlog') if options.git_remote: deploy.push_binary_transparency(options.git_repo, options.git_remote) shutil.rmtree(tempdirbase, ignore_errors=True) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 2e716c10..a4fe326b 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -"""Build a package from source.""" # # build.py - part of the FDroid server tools # Copyright (C) 2010-2014, Ciaran Gultnieks, ciaran@ciarang.com @@ -18,70 +17,53 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import argparse -import glob -import logging import os +import shutil +import glob +import subprocess import posixpath import re -import shutil -import subprocess +import resource +import sys import tarfile -import tempfile import threading -import time import traceback -from gettext import ngettext -from pathlib import Path - +import time import requests +import tempfile +import argparse +from configparser import ConfigParser +import logging +from gettext import ngettext -from . import _, common, metadata, net, scanner, vmtools +from . import _ +from . import common +from . import net +from . import metadata +from . import scanner +from . import vmtools from .common import FDroidPopen -from .exception import BuildException, FDroidException, VCSException +from .exception import FDroidException, BuildException, VCSException try: import paramiko except ImportError: pass -buildserverid = None -ssh_channel = None - # Note that 'force' here also implies test mode. def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): """Do a build on the builder vm. - Parameters - ---------- - app - The metadata of the app to build. - build - The build of the app to build. - vcs - The version control system controller object of the app. - build_dir - The local source-code checkout directory of the app. - output_dir - The target folder for the build result. - log_dir - The directory in the VM where the build logs are getting stored. - force - Don't refresh the already cloned repository and make the build stop on - exceptions. - - Raises - ------ - :exc:`~fdroidserver.exception.BuildException` - If Paramiko is not installed, a srclib directory or srclib metadata - file is unexpectedly missing, the build process in the VM failed or - output files of the build process are missing. - :exc:`~fdroidserver.exception.FDroidException` - If the Buildserver ID could not be obtained or copying a directory to - the server failed. + :param app: app metadata dict + :param build: + :param vcs: version control system controller object + :param build_dir: local source-code checkout of app + :param output_dir: target folder for the build result + :param force: """ - global buildserverid, ssh_channel + + global buildserverid try: paramiko @@ -101,6 +83,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c', 'cat /home/vagrant/buildserverid'], cwd='builder').strip().decode() + status_output['buildserverid'] = buildserverid logging.debug(_('Fetched buildserverid from VM: {buildserverid}') .format(buildserverid=buildserverid)) except Exception as e: @@ -113,7 +96,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): # Open SSH connection... logging.info("Connecting to virtual machine...") sshs = paramiko.SSHClient() - sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # nosec B507 only connects to local VM + sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) sshs.connect(sshinfo['hostname'], username=sshinfo['user'], port=sshinfo['port'], timeout=300, look_for_keys=False, key_filename=sshinfo['idfile']) @@ -127,9 +110,9 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): # Put all the necessary files in place... ftp.chdir(homedir) + # Helper to copy the contents of a directory to the server... def send_dir(path): - """Copy the contents of a directory to the server.""" - logging.debug("rsyncing %s to %s" % (path, ftp.getcwd())) + logging.debug("rsyncing " + path + " to " + ftp.getcwd()) # TODO this should move to `vagrant rsync` from >= v1.5 try: subprocess.check_output(['rsync', '--recursive', '--perms', '--links', '--quiet', '--rsh=' @@ -144,14 +127,16 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): sshinfo['user'] + "@" + sshinfo['hostname'] + ":" + ftp.getcwd()], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: - raise FDroidException(str(e), e.output.decode()) from e + raise FDroidException(str(e), e.output.decode()) logging.info("Preparing server for build...") serverpath = os.path.abspath(os.path.dirname(__file__)) ftp.mkdir('fdroidserver') ftp.chdir('fdroidserver') ftp.put(os.path.join(serverpath, '..', 'fdroid'), 'fdroid') + ftp.put(os.path.join(serverpath, '..', 'gradlew-fdroid'), 'gradlew-fdroid') ftp.chmod('fdroid', 0o755) # nosec B103 permissions are appropriate + ftp.chmod('gradlew-fdroid', 0o755) # nosec B103 permissions are appropriate send_dir(os.path.join(serverpath)) ftp.chdir(homedir) @@ -234,55 +219,51 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): # Execute the build script... logging.info("Starting build...") - ssh_channel = sshs.get_transport().open_session() - ssh_channel.get_pty() + chan = sshs.get_transport().open_session() + chan.get_pty() cmdline = posixpath.join(homedir, 'fdroidserver', 'fdroid') cmdline += ' build --on-server' if force: cmdline += ' --force --test' if options.verbose: cmdline += ' --verbose' - if options.refresh_scanner or config.get('refresh_scanner'): - cmdline += ' --refresh-scanner' if options.skipscan: cmdline += ' --skip-scan' if options.notarball: cmdline += ' --no-tarball' - if (options.scan_binary or config.get('scan_binary')) and not options.skipscan: - cmdline += ' --scan-binary' cmdline += " %s:%s" % (app.id, build.versionCode) - ssh_channel.exec_command('bash --login -c "' + cmdline + '"') # nosec B601 inputs are sanitized + chan.exec_command('bash --login -c "' + cmdline + '"') # nosec B601 inputs are sanitized # Fetch build process output ... try: - cmd_stdout = ssh_channel.makefile('rb', 1024) + cmd_stdout = chan.makefile('rb', 1024) output = bytes() - output += common.get_android_tools_version_log().encode() - while not ssh_channel.exit_status_ready(): + output += common.get_android_tools_version_log(build.ndk_path()).encode() + while not chan.exit_status_ready(): line = cmd_stdout.readline() if line: if options.verbose: - logging.debug("buildserver > " + str(line, 'utf-8', 'replace').rstrip()) + logging.debug("buildserver > " + str(line, 'utf-8').rstrip()) output += line else: time.sleep(0.05) for line in cmd_stdout.readlines(): if options.verbose: - logging.debug("buildserver > " + str(line, 'utf-8', 'replace').rstrip()) + logging.debug("buildserver > " + str(line, 'utf-8').rstrip()) output += line finally: cmd_stdout.close() # Check build process exit status ... logging.info("...getting exit status") - returncode = ssh_channel.recv_exit_status() + returncode = chan.recv_exit_status() if returncode != 0: if timeout_event.is_set(): message = "Timeout exceeded! Build VM force-stopped for {0}:{1}" else: message = "Build.py failed on server for {0}:{1}" raise BuildException(message.format(app.id, build.versionName), - str(output, 'utf-8', 'replace')) + None if options.verbose else str(output, 'utf-8')) # Retreive logs... toolsversion_log = common.get_toolsversion_logname(app, build) @@ -300,15 +281,15 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): else: ftp.chdir(posixpath.join(homedir, 'unsigned')) apkfile = common.get_release_filename(app, build) - tarball = common.get_src_tarball_name(app.id, build.versionCode) + tarball = common.getsrcname(app, build) try: ftp.get(apkfile, os.path.join(output_dir, apkfile)) if not options.notarball: ftp.get(tarball, os.path.join(output_dir, tarball)) - except Exception as exc: + except Exception: raise BuildException( "Build failed for {0}:{1} - missing output files".format( - app.id, build.versionName), str(output, 'utf-8', 'replace')) from exc + app.id, build.versionName), None if options.verbose else str(output, 'utf-8')) ftp.close() finally: @@ -326,15 +307,6 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): def force_gradle_build_tools(build_dir, build_tools): - """Manipulate build tools version used in top level gradle file. - - Parameters - ---------- - build_dir - The directory to start looking for gradle files. - build_tools - The build tools version that should be forced to use. - """ for root, dirs, files in os.walk(build_dir): for filename in files: if not filename.endswith('.gradle'): @@ -349,7 +321,7 @@ def force_gradle_build_tools(build_dir, build_tools): def transform_first_char(string, method): - """Use method() on the first character of string.""" + """Uses method() on the first character of string.""" if len(string) == 0: return string if len(string) == 1: @@ -357,36 +329,16 @@ def transform_first_char(string, method): return method(string[0]) + string[1:] +def add_failed_builds_entry(failed_builds, appid, build, entry): + failed_builds.append([appid, int(build.versionCode), str(entry)]) + + def get_metadata_from_apk(app, build, apkfile): - """Get the required metadata from the built APK. + """get the required metadata from the built APK - VersionName is allowed to be a blank string, i.e. '' - - Parameters - ---------- - app - The app metadata used to build the APK. - build - The build that resulted in the APK. - apkfile - The path of the APK file. - - Returns - ------- - versionCode - The versionCode from the APK or from the metadata is build.novcheck is - set. - versionName - The versionName from the APK or from the metadata is build.novcheck is - set. - - Raises - ------ - :exc:`~fdroidserver.exception.BuildException` - If native code should have been built but was not packaged, no version - information or no package ID could be found or there is a mismatch - between the package ID in the metadata and the one found in the APK. + versionName is allowed to be a blank string, i.e. '' """ + appid, versionCode, versionName = common.get_apk_id(apkfile) native_code = common.get_native_code(apkfile) @@ -406,74 +358,22 @@ def get_metadata_from_apk(app, build, apkfile): def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh): - """Do a build locally. - - Parameters - ---------- - app - The metadata of the app to build. - build - The build of the app to build. - vcs - The version control system controller object of the app. - build_dir - The local source-code checkout directory of the app. - output_dir - The target folder for the build result. - log_dir - The directory in the VM where the build logs are getting stored. - srclib_dir - The path to the srclibs directory, usually 'build/srclib'. - extlib_dir - The path to the extlibs directory, usually 'build/extlib'. - tmp_dir - The temporary directory for building the source tarball. - force - Don't refresh the already cloned repository and make the build stop on - exceptions. - onserver - Assume the build is happening inside the VM. - refresh - Enable fetching the latest refs from the VCS remote. - - Raises - ------ - :exc:`~fdroidserver.exception.BuildException` - If running a `sudo` command failed, locking the root account failed, - `sudo` couldn't be removed, cleaning the build environment failed, - skipping the scanning has been requested but `scandelete` is present, - errors occurred during scanning, running the `build` commands from the - metadata failed, building native code failed, building with the - specified build method failed, no output could be found with build - method `maven`, more or less than one APK were found with build method - `gradle`, less or more than one APKs match the `output` glob specified - in the metadata, running a `postbuild` command specified in the - metadata failed, the built APK is debuggable, the unsigned APK is not - at the expected location, the APK does not contain the expected - `versionName` and `versionCode` or undesired package names have been - found in the APK. - :exc:`~fdroidserver.exception.FDroidException` - If no Android NDK version could be found and the build isn't run in a - builder VM, the selected Android NDK is not a directory. - """ + """Do a build locally.""" ndk_path = build.ndk_path() if build.ndk or (build.buildjni and build.buildjni != ['no']): if not ndk_path: - logging.warning("Android NDK version '%s' could not be found!" % build.ndk) - logging.warning("Configured versions:") + logging.critical("Android NDK version '%s' could not be found!" % build.ndk or 'r12b') + logging.critical("Configured versions:") for k, v in config['ndk_paths'].items(): if k.endswith("_orig"): continue - logging.warning(" %s: %s" % (k, v)) - if onserver: - common.auto_install_ndk(build) - else: - raise FDroidException() + logging.critical(" %s: %s" % (k, v)) + raise FDroidException() elif not os.path.isdir(ndk_path): logging.critical("Android NDK '%s' is not a directory!" % ndk_path) raise FDroidException() - common.set_FDroidPopen_env(app, build) + common.set_FDroidPopen_env(build) # create ..._toolsversion.log when running in builder vm if onserver: @@ -482,7 +382,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext logging.info("Running 'sudo' commands in %s" % os.getcwd()) p = FDroidPopen(['sudo', 'DEBIAN_FRONTEND=noninteractive', - 'bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', '; '.join(build.sudo)]) + 'bash', '-x', '-c', build.sudo]) if p.returncode != 0: raise BuildException("Error running sudo command for %s:%s" % (app.id, build.versionName), p.output) @@ -500,7 +400,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext log_path = os.path.join(log_dir, common.get_toolsversion_logname(app, build)) with open(log_path, 'w') as f: - f.write(common.get_android_tools_version_log()) + f.write(common.get_android_tools_version_log(build.ndk_path())) else: if build.sudo: logging.warning('%s:%s runs this on the buildserver with sudo:\n\t%s\nThese commands were skipped because fdroid build is not running on a dedicated build server.' @@ -535,13 +435,13 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext if build.preassemble: gradletasks += build.preassemble - flavors = build.gradle - if flavors == ['yes']: - flavors = [] + flavours = build.gradle + if flavours == ['yes']: + flavours = [] - flavors_cmd = ''.join([transform_first_char(flav, str.upper) for flav in flavors]) + flavours_cmd = ''.join([transform_first_char(flav, str.upper) for flav in flavours]) - gradletasks += ['assemble' + flavors_cmd + 'Release'] + gradletasks += ['assemble' + flavours_cmd + 'Release'] cmd = [config['gradle']] if build.gradleprops: @@ -550,6 +450,9 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext cmd += ['clean'] p = FDroidPopen(cmd, cwd=root_dir, envs={"GRADLE_VERSION_DIR": config['gradle_version_dir'], "CACHEDIR": config['cachedir']}) + elif bmethod == 'buildozer': + pass + elif bmethod == 'ant': logging.info("Cleaning Ant project...") p = FDroidPopen(['ant', 'clean'], cwd=root_dir) @@ -613,7 +516,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext if not options.notarball: # Build the source tarball right before we build the release... logging.info("Creating source tarball...") - tarname = common.get_src_tarball_name(app.id, build.versionCode) + tarname = common.getsrcname(app, build) tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz") def tarexc(t): @@ -624,13 +527,13 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext # Run a build command if one is required... if build.build: logging.info("Running 'build' commands in %s" % root_dir) - cmd = common.replace_config_vars("; ".join(build.build), build) + cmd = common.replace_config_vars(build.build, build) # Substitute source library paths into commands... for name, number, libpath in srclibpaths: cmd = cmd.replace('$$' + name + '$$', os.path.join(os.getcwd(), libpath)) - p = FDroidPopen(['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir) + p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running build command for %s:%s" % @@ -692,6 +595,73 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext bindir = os.path.join(root_dir, 'target') + elif bmethod == 'buildozer': + logging.info("Building Kivy project using buildozer...") + + # parse buildozer.spez + spec = os.path.join(root_dir, 'buildozer.spec') + if not os.path.exists(spec): + raise BuildException("Expected to find buildozer-compatible spec at {0}" + .format(spec)) + defaults = {'orientation': 'landscape', 'icon': '', + 'permissions': '', 'android.api': "19"} + bconfig = ConfigParser(defaults, allow_no_value=True) + bconfig.read(spec) + + # update spec with sdk and ndk locations to prevent buildozer from + # downloading. + loc_ndk = common.env['ANDROID_NDK'] + loc_sdk = common.env['ANDROID_SDK'] + if loc_ndk == '$ANDROID_NDK': + loc_ndk = loc_sdk + '/ndk-bundle' + + bc_ndk = None + bc_sdk = None + try: + bc_ndk = bconfig.get('app', 'android.sdk_path') + except Exception: + pass + try: + bc_sdk = bconfig.get('app', 'android.ndk_path') + except Exception: + pass + + if bc_sdk is None: + bconfig.set('app', 'android.sdk_path', loc_sdk) + if bc_ndk is None: + bconfig.set('app', 'android.ndk_path', loc_ndk) + + fspec = open(spec, 'w') + bconfig.write(fspec) + fspec.close() + + logging.info("sdk_path = %s" % loc_sdk) + logging.info("ndk_path = %s" % loc_ndk) + + p = None + # execute buildozer + cmd = ['buildozer', 'android', 'release'] + try: + p = FDroidPopen(cmd, cwd=root_dir) + except Exception: + pass + + # buidozer not installed ? clone repo and run + if (p is None or p.returncode != 0): + cmd = ['git', 'clone', 'https://github.com/kivy/buildozer.git'] + p = subprocess.Popen(cmd, cwd=root_dir, shell=False) + p.wait() + if p.returncode != 0: + raise BuildException("Distribute build failed") + + cmd = ['python', 'buildozer/buildozer/scripts/client.py', 'android', 'release'] + p = FDroidPopen(cmd, cwd=root_dir) + + # expected to fail. + # Signing will fail if not set by environnment vars (cf. p4a docs). + # But the unsigned apk will be ok. + p.returncode = 0 + elif bmethod == 'gradle': logging.info("Building Gradle project...") @@ -714,16 +684,9 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext bindir = os.path.join(root_dir, 'bin') - if os.path.isdir(os.path.join(build_dir, '.git')): - commit_id = str(common.get_head_commit_id(build_dir)) - else: - commit_id = build.commit - if p is not None and p.returncode != 0: - raise BuildException("Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id), - p.output) - logging.info("Successfully built version {versionName} of {appid} from {commit_id}" - .format(versionName=build.versionName, appid=app.id, commit_id=commit_id)) + raise BuildException("Build failed for %s:%s" % (app.id, build.versionName), p.output) + logging.info("Successfully built version " + build.versionName + ' of ' + app.id) omethod = build.output_method() if omethod == 'maven': @@ -747,6 +710,26 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext src = m.group(1) src = os.path.join(bindir, src) + '.apk' + elif omethod == 'buildozer': + src = None + for apks_dir in [ + os.path.join(root_dir, '.buildozer', 'android', 'platform', 'build', 'dists', bconfig.get('app', 'title'), 'bin'), + ]: + for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']: + apks = glob.glob(os.path.join(apks_dir, apkglob)) + + if len(apks) > 1: + raise BuildException('More than one resulting apks found in %s' % apks_dir, + '\n'.join(apks)) + if len(apks) == 1: + src = apks[0] + break + if src is not None: + break + + if src is None: + raise BuildException('Failed to find any output apks') + elif omethod == 'gradle': src = None apk_dirs = [ @@ -757,11 +740,11 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext # really old path os.path.join(root_dir, 'build', 'apk'), ] - # If we build with gradle flavors with gradle plugin >= 3.0 the APK will be in - # a subdirectory corresponding to the flavor command used, but with different + # If we build with gradle flavours with gradle plugin >= 3.0 the apk will be in + # a subdirectory corresponding to the flavour command used, but with different # capitalization. - if flavors_cmd: - apk_dirs.append(os.path.join(root_dir, 'build', 'outputs', 'apk', transform_first_char(flavors_cmd, str.lower), 'release')) + if flavours_cmd: + apk_dirs.append(os.path.join(root_dir, 'build', 'outputs', 'apk', transform_first_char(flavours_cmd, str.lower), 'release')) for apks_dir in apk_dirs: for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']: apks = glob.glob(os.path.join(apks_dir, apkglob)) @@ -794,55 +777,31 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext raise BuildException('No apks match %s' % globpath) src = os.path.normpath(apks[0]) - # Run a postbuild command if one is required... - if build.postbuild: - logging.info(f"Running 'postbuild' commands in {root_dir}") - cmd = common.replace_config_vars("; ".join(build.postbuild), build) - - # Substitute source library paths into commands... - for name, number, libpath in srclibpaths: - cmd = cmd.replace(f"$${name}$$", str(Path.cwd() / libpath)) - - cmd = cmd.replace('$$OUT$$', str(Path(src).resolve())) - - p = FDroidPopen(['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir) - - if p.returncode != 0: - raise BuildException("Error running postbuild command for " - f"{app.id}:{build.versionName}", p.output) - # Make sure it's not debuggable... - if common.is_debuggable_or_testOnly(src): - raise BuildException( - "%s: debuggable or testOnly set in AndroidManifest.xml" % src - ) + if common.is_apk_and_debuggable(src): + raise BuildException("APK is debuggable") # By way of a sanity check, make sure the version and version - # code in our new APK match what we expect... + # code in our new apk match what we expect... logging.debug("Checking " + src) if not os.path.exists(src): - raise BuildException("Unsigned APK is not at expected location of " + src) + raise BuildException("Unsigned apk is not at expected location of " + src) if common.get_file_extension(src) == 'apk': vercode, version = get_metadata_from_apk(app, build, src) if version != build.versionName or vercode != build.versionCode: raise BuildException(("Unexpected version/version code in output;" - " APK: '%s' / '%d', " - " Expected: '%s' / '%d'") - % (version, vercode, build.versionName, - build.versionCode)) + " APK: '%s' / '%s', " + " Expected: '%s' / '%s'") + % (version, str(vercode), build.versionName, + str(build.versionCode))) if (options.scan_binary or config.get('scan_binary')) and not options.skipscan: if scanner.scan_binary(src): - raise BuildException("Found blocklisted packages in final apk!") + raise BuildException("Found blacklisted packages in final apk!") - # Copy the unsigned APK to our destination directory for further + # Copy the unsigned apk to our destination directory for further # processing (by publish.py)... - dest = os.path.join( - output_dir, - common.get_release_filename( - app, build, common.get_file_extension(src) - ) - ) + dest = os.path.join(output_dir, common.get_release_filename(app, build)) shutil.copyfile(src, dest) # Move the source tarball into the output directory... @@ -854,53 +813,23 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, test, server, force, onserver, refresh): - """Build a particular version of an application, if it needs building. - - Parameters - ---------- - app - The metadata of the app to build. - build - The build of the app to build. - build_dir - The local source-code checkout directory of the app. - output_dir - The directory where the build output will go. Usually this is the - 'unsigned' directory. - log_dir - The directory in the VM where the build logs are getting stored. - also_check_dir - An additional location for checking if the build is necessary (usually - the archive repo). - srclib_dir - The path to the srclibs directory, usually 'build/srclib'. - extlib_dir - The path to the extlibs directory, usually 'build/extlib'. - tmp_dir - The temporary directory for building the source tarball of the app to - build. - repo_dir - The repo directory - used for checking if the build is necessary. - vcs - The version control system controller object of the app to build. - test - True if building in test mode, in which case the build will always - happen, even if the output already exists. In test mode, the output - directory should be a temporary location, not any of the real ones. - server - Use buildserver VM for building. - force - Build app regardless of disabled state or scanner errors. - onserver - Assume the build is happening inside the VM. - refresh - Enable fetching the latest refs from the VCS remote. - - Returns - ------- - status - True if the build was done, False if it wasn't necessary. """ + Build a particular version of an application, if it needs building. + + :param output_dir: The directory where the build output will go. Usually + this is the 'unsigned' directory. + :param repo_dir: The repo directory - used for checking if the build is + necessary. + :param also_check_dir: An additional location for checking if the build + is necessary (usually the archive repo) + :param test: True if building in test mode, in which case the build will + always happen, even if the output already exists. In test mode, the + output directory should be a temporary location, not any of the real + ones. + + :returns: True if the build was done, False if it wasn't necessary. + """ + dest_file = common.get_release_filename(app, build) dest = os.path.join(output_dir, dest_file) @@ -926,10 +855,6 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, # grabbing the source now. vcs.gotorevision(build.commit, refresh) - # Initialise submodules if required - if build.submodules: - vcs.initsubmodules() - build_server(app, build, vcs, build_dir, output_dir, log_dir, force) else: build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh) @@ -937,40 +862,16 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, def force_halt_build(timeout): - """Halt the currently running Vagrant VM, to be called from a Timer. - - Parameters - ---------- - timeout - The timeout in seconds. - """ + """Halt the currently running Vagrant VM, to be called from a Timer""" logging.error(_('Force halting build after {0} sec timeout!').format(timeout)) timeout_event.set() - if ssh_channel: - ssh_channel.close() vm = vmtools.get_build_vm('builder') - vm.destroy() - - -def keep_when_not_allowed(): - """Control if APKs signed by keys not in AllowedAPKSigningKeys are removed.""" - return ( - (options is not None and options.keep_when_not_allowed) - or (config is not None and config.get('keep_when_not_allowed')) - or common.default_config['keep_when_not_allowed'] - ) + vm.halt() def parse_commandline(): - """Parse the command line. + """Parse the command line. Returns options, parser.""" - Returns - ------- - options - The resulting options parsed from the command line arguments. - parser - The argument parser. - """ parser = argparse.ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) parser.add_argument("appid", nargs='*', help=_("application ID with optional versionCode in the form APPID[:VERCODE]")) @@ -982,6 +883,8 @@ def parse_commandline(): help=_("Test mode - put output in the tmp directory only, and always build, even if the output already exists.")) parser.add_argument("--server", action="store_true", default=False, help=_("Use build server")) + parser.add_argument("--reset-server", action="store_true", default=False, + help=_("Reset and create a brand new build server, even if the existing one appears to be ok.")) # this option is internal API for telling fdroid that # it's running inside a buildserver vm. parser.add_argument("--on-server", dest="onserver", action="store_true", default=False, @@ -994,18 +897,14 @@ def parse_commandline(): help=_("Don't create a source tarball, useful when testing a build")) parser.add_argument("--no-refresh", dest="refresh", action="store_false", default=True, help=_("Don't refresh the repository, useful when testing a build with no internet connection")) - parser.add_argument("-r", "--refresh-scanner", dest="refresh_scanner", action="store_true", default=False, - help=_("Refresh and cache scanner rules and signatures from the network")) parser.add_argument("-f", "--force", action="store_true", default=False, help=_("Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode.")) parser.add_argument("-a", "--all", action="store_true", default=False, help=_("Build all applications available")) - parser.add_argument("--keep-when-not-allowed", default=False, action="store_true", - help=argparse.SUPPRESS) parser.add_argument("-w", "--wiki", default=False, action="store_true", - help=argparse.SUPPRESS) + help=_("Update the wiki")) metadata.add_metadata_arguments(parser) - options = common.parse_args(parser) + options = parser.parse_args() metadata.warnings_action = options.W # Force --stop with --on-server to get correct exit code @@ -1020,6 +919,7 @@ def parse_commandline(): options = None config = None +buildserverid = None fdroidserverid = None start_timestamp = time.gmtime() status_output = None @@ -1027,22 +927,7 @@ timeout_event = threading.Event() def main(): - """Build a package from source. - The behaviour of this function is influenced by the configuration file as - well as command line parameters. - - Raises - ------ - :exc:`~fdroidserver.exception.FDroidException` - If more than one local metadata file has been found, no app metadata - has been found, there are no apps to process, downloading binaries for - checking the reproducibility of a built binary failed, the built binary - is different from supplied reference binary, the reference binary is - signed with a different signing key than expected, a VCS error occured - while building an app or a different error occured while building an - app. - """ global options, config, buildserverid, fdroidserverid options, parser = parse_commandline() @@ -1067,10 +952,18 @@ def main(): if not options.appid and not options.all: parser.error("option %s: If you really want to build all the apps, use --all" % "all") - config = common.read_config() + config = common.read_config(options) if config['build_server_always']: options.server = True + if options.reset_server and not options.server: + parser.error("option %s: Using --reset-server without --server makes no sense" % "reset-server") + + if options.onserver or not options.server: + for d in ['build-tools', 'platform-tools', 'tools']: + if not os.path.isdir(os.path.join(config['sdk_path'], d)): + raise FDroidException(_("Android SDK '{path}' does not have '{dirname}' installed!") + .format(path=config['sdk_path'], dirname=d)) log_dir = 'logs' if not os.path.isdir(log_dir): @@ -1110,57 +1003,58 @@ def main(): srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') - apps = common.read_app_args(options.appid, allow_version_codes=True, sort_by_time=True) + # Read all app and srclib metadata + pkgs = common.read_pkg_args(options.appid, True) + allapps = metadata.read_metadata(pkgs, options.refresh, sort_by_time=True) + apps = common.read_app_args(options.appid, allapps, True) for appid, app in list(apps.items()): - if (app.get('Disabled') and not options.force) or not app.get('RepoType') or not app.get('Builds', []): + if (app.Disabled and not options.force) or not app.RepoType or not app.builds: del apps[appid] if not apps: raise FDroidException("No apps to process.") # make sure enough open files are allowed to process everything - try: - import resource # not available on Windows - - soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) - if len(apps) > soft: - try: - soft = len(apps) * 2 - if soft > hard: - soft = hard - resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) - logging.debug(_('Set open file limit to {integer}') - .format(integer=soft)) - except (OSError, ValueError) as e: - logging.warning(_('Setting open file limit failed: ') + str(e)) - except ImportError: - pass + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if len(apps) > soft: + try: + soft = len(apps) * 2 + if soft > hard: + soft = hard + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) + logging.debug(_('Set open file limit to {integer}') + .format(integer=soft)) + except (OSError, ValueError) as e: + logging.warning(_('Setting open file limit failed: ') + str(e)) if options.latest: for app in apps.values(): - for build in reversed(app.get('Builds', [])): + for build in reversed(app.builds): if build.disable and not options.force: continue - app['Builds'] = [build] + app.builds = [build] break - if not options.onserver: - common.write_running_status_json(status_output) + if options.wiki: + import mwclient + site = mwclient.Site((config['wiki_protocol'], config['wiki_server']), + path=config['wiki_path']) + site.login(config['wiki_user'], config['wiki_password']) # Build applications... failed_builds = [] - build_succeeded_ids = [] + build_succeeded = [] status_output['failedBuilds'] = failed_builds - status_output['successfulBuildIds'] = build_succeeded_ids - # Only build for 72 hours, then stop gracefully. - endtime = time.time() + 72 * 60 * 60 + status_output['successfulBuilds'] = build_succeeded + # Only build for 36 hours, then stop gracefully. + endtime = time.time() + 36 * 60 * 60 max_build_time_reached = False for appid, app in apps.items(): first = True - for build in app.get('Builds', []): + for build in app.builds: if time.time() > endtime: max_build_time_reached = True break @@ -1169,7 +1063,7 @@ def main(): if build.timeout is None: timeout = 7200 else: - timeout = build.timeout + timeout = int(build.timeout) if options.server and timeout > 0: logging.debug(_('Setting {0} sec timeout for this build').format(timeout)) timer = threading.Timer(timeout, force_halt_build, [timeout]) @@ -1178,7 +1072,12 @@ def main(): else: timer = None + wikilog = None + build_starttime = common.get_wiki_timestamp() tools_version_log = '' + if not options.onserver: + tools_version_log = common.get_android_tools_version_log(build.ndk_path()) + common.write_running_status_json(status_output) try: # For the first build of a particular app, we need to set up @@ -1188,7 +1087,8 @@ def main(): vcs, build_dir = common.setup_vcs(app) first = False - logging.debug("Checking %s:%s" % (appid, build.versionCode)) + logging.info("Using %s" % vcs.clientversion()) + logging.debug("Checking " + build.versionName) if trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test, @@ -1201,9 +1101,9 @@ def main(): tools_version_log = ''.join(f.readlines()) os.remove(toolslog) - if url := build.binary or app.Binaries: + if app.Binaries is not None: # This is an app where we build from source, and - # verify the APK contents against a developer's + # verify the apk contents against a developer's # binary. We get that binary now, and save it # alongside our built one in the 'unsigend' # directory. @@ -1213,10 +1113,11 @@ def main(): "developer supplied reference " "binaries: '{path}'" .format(path=binaries_dir)) + url = app.Binaries url = url.replace('%v', build.versionName) url = url.replace('%c', str(build.versionCode)) logging.info("...retrieving " + url) - of = re.sub(r'\.apk$', '.binary.apk', common.get_release_filename(app, build)) + of = re.sub(r'.apk$', '.binary.apk', common.get_release_filename(app, build)) of = os.path.join(binaries_dir, of) try: net.download_file(url, local_filename=of) @@ -1236,12 +1137,8 @@ def main(): compare_result = \ common.verify_apks(of, unsigned_apk, tmpdir) if compare_result: - if options.test: - logging.warning(_('Keeping failed build "{apkfilename}"') - .format(apkfilename=unsigned_apk)) - else: - logging.debug('removing %s', unsigned_apk) - os.remove(unsigned_apk) + logging.debug('removing %s', unsigned_apk) + os.remove(unsigned_apk) logging.debug('removing %s', of) os.remove(of) compare_result = compare_result.split('\n') @@ -1262,86 +1159,66 @@ def main(): 'supplied reference binary ' 'successfully') - used_key = common.apk_signer_fingerprint(of) - expected_keys = app['AllowedAPKSigningKeys'] - if used_key is None: - logging.warn(_('reference binary missing ' - 'signature')) - elif len(expected_keys) == 0: - logging.warn(_('AllowedAPKSigningKeys missing ' - 'but reference binary supplied')) - elif used_key not in expected_keys: - if options.test or keep_when_not_allowed(): - logging.warning(_('Keeping failed build "{apkfilename}"') - .format(apkfilename=unsigned_apk)) - else: - logging.debug('removing %s', unsigned_apk) - os.remove(unsigned_apk) - logging.debug('removing %s', of) - os.remove(of) - raise FDroidException('supplied reference ' - 'binary signed with ' - '{signer} instead of ' - 'with {expected}'. - format(signer=used_key, - expected=expected_keys)) - else: - logging.info(_('supplied reference binary has ' - 'allowed signer {signer}'). - format(signer=used_key)) - - build_succeeded_ids.append([app['id'], build.versionCode]) - - if not options.onserver: - common.write_running_status_json(status_output) + build_succeeded.append(app) + wikilog = "Build succeeded" except VCSException as vcse: reason = str(vcse).split('\n', 1)[0] if options.verbose else str(vcse) logging.error("VCS error while building app %s: %s" % ( appid, reason)) if options.stop: - logging.debug("Error encountered, stopping by user request.") + logging.debug("Error encoutered, stopping by user request.") common.force_exit(1) - failed_builds.append((appid, build.versionCode)) - common.deploy_build_log_with_rsync( - appid, build.versionCode, "".join(traceback.format_exc()) - ) - if not options.onserver: - common.write_running_status_json(status_output) - + add_failed_builds_entry(failed_builds, appid, build, vcse) + wikilog = str(vcse) except FDroidException as e: - tstamp = time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) with open(os.path.join(log_dir, appid + '.log'), 'a+') as f: f.write('\n\n============================================================\n') f.write('versionCode: %s\nversionName: %s\ncommit: %s\n' % (build.versionCode, build.versionName, build.commit)) f.write('Build completed at ' - + tstamp + '\n') + + common.get_wiki_timestamp() + '\n') f.write('\n' + tools_version_log + '\n') f.write(str(e)) logging.error("Could not build app %s: %s" % (appid, e)) if options.stop: - logging.debug("Error encountered, stopping by user request.") + logging.debug("Error encoutered, stopping by user request.") common.force_exit(1) - failed_builds.append((appid, build.versionCode)) - common.deploy_build_log_with_rsync( - appid, build.versionCode, "".join(traceback.format_exc()) - ) - if not options.onserver: - common.write_running_status_json(status_output) - - except Exception: + add_failed_builds_entry(failed_builds, appid, build, e) + wikilog = e.get_wikitext() + except Exception as e: logging.error("Could not build app %s due to unknown error: %s" % ( appid, traceback.format_exc())) if options.stop: - logging.debug("Error encountered, stopping by user request.") + logging.debug("Error encoutered, stopping by user request.") common.force_exit(1) - failed_builds.append((appid, build.versionCode)) - common.deploy_build_log_with_rsync( - appid, build.versionCode, "".join(traceback.format_exc()) - ) - if not options.onserver: - common.write_running_status_json(status_output) + add_failed_builds_entry(failed_builds, appid, build, e) + wikilog = str(e) + + if options.wiki and wikilog: + try: + # Write a page with the last build log for this version code + lastbuildpage = appid + '/lastbuild_' + build.versionCode + newpage = site.Pages[lastbuildpage] + with open(os.path.join('tmp', 'fdroidserverid')) as fp: + fdroidserverid = fp.read().rstrip() + txt = "* build session started at " + common.get_wiki_timestamp(start_timestamp) + '\n' \ + + "* this build started at " + build_starttime + '\n' \ + + "* this build completed at " + common.get_wiki_timestamp() + '\n' \ + + common.get_git_describe_link() \ + + '* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/' \ + + fdroidserverid + ' ' + fdroidserverid + ']\n\n' + if buildserverid: + txt += '* buildserverid: [https://gitlab.com/fdroid/fdroidserver/commit/' \ + + buildserverid + ' ' + buildserverid + ']\n\n' + txt += tools_version_log + '\n\n' + txt += '== Build Log ==\n\n' + wikilog + newpage.save(txt, summary='Build log') + # Redirect from /lastbuild to the most recent build log + newpage = site.Pages[appid + '/lastbuild'] + newpage.save('#REDIRECT [[' + lastbuildpage + ']]', summary='Update redirect') + except Exception as e: + logging.error("Error while attempting to publish build log: %s" % e) if timer: timer.cancel() # kill the watchdog timer @@ -1351,44 +1228,60 @@ def main(): logging.info("Stopping after global build timeout...") break - for app in build_succeeded_ids: - logging.info("success: %s" % app[0]) + for app in build_succeeded: + logging.info("success: %s" % (app.id)) if not options.verbose: for fb in failed_builds: - logging.info('Build for app {}:{} failed'.format(*fb)) + logging.info('Build for app {}:{} failed:\n{}'.format(*fb)) logging.info(_("Finished")) - if len(build_succeeded_ids) > 0: + if len(build_succeeded) > 0: logging.info(ngettext("{} build succeeded", - "{} builds succeeded", len(build_succeeded_ids)).format(len(build_succeeded_ids))) + "{} builds succeeded", len(build_succeeded)).format(len(build_succeeded))) if len(failed_builds) > 0: logging.info(ngettext("{} build failed", "{} builds failed", len(failed_builds)).format(len(failed_builds))) - if options.server: + if options.wiki: + wiki_page_path = 'build_' + time.strftime('%s', start_timestamp) + newpage = site.Pages[wiki_page_path] + txt = '' + txt += "* command line: %s\n" % ' '.join(sys.argv) + txt += "* started at %s\n" % common.get_wiki_timestamp(start_timestamp) + txt += "* completed at %s\n" % common.get_wiki_timestamp() + if buildserverid: + txt += ('* buildserverid: [https://gitlab.com/fdroid/fdroidserver/commit/{id} {id}]\n' + .format(id=buildserverid)) + if fdroidserverid: + txt += ('* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/{id} {id}]\n' + .format(id=fdroidserverid)) if os.cpu_count(): - status_output['hostOsCpuCount'] = os.cpu_count() + txt += "* host processors: %d\n" % os.cpu_count() if os.path.isfile('/proc/meminfo') and os.access('/proc/meminfo', os.R_OK): with open('/proc/meminfo') as fp: for line in fp: m = re.search(r'MemTotal:\s*([0-9].*)', line) if m: - status_output['hostProcMeminfoMemTotal'] = m.group(1) + txt += "* host RAM: %s\n" % m.group(1) break - buildserver_config = 'builder/Vagrantfile' + fdroid_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) + buildserver_config = os.path.join(fdroid_path, 'makebuildserver.config.py') if os.path.isfile(buildserver_config) and os.access(buildserver_config, os.R_OK): with open(buildserver_config) as configfile: for line in configfile: m = re.search(r'cpus\s*=\s*([0-9].*)', line) if m: - status_output['guestVagrantVmCpus'] = m.group(1) + txt += "* guest processors: %s\n" % m.group(1) m = re.search(r'memory\s*=\s*([0-9].*)', line) if m: - status_output['guestVagrantVmMemory'] = m.group(1) - - if buildserverid: - status_output['buildserver'] = {'commitId': buildserverid} + txt += "* guest RAM: %s MB\n" % m.group(1) + txt += "* successful builds: %d\n" % len(build_succeeded) + txt += "* failed builds: %d\n" % len(failed_builds) + txt += "\n\n" + newpage.save(txt, summary='Run log') + newpage = site.Pages['build'] + newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect') if not options.onserver: common.write_status_json(status_output) diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index e7945910..604f4e1a 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -"""Check for updates to applications.""" # # checkupdates.py - part of the FDroid server tools # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com @@ -18,341 +17,307 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import configparser -import copy -import logging import os import re +import urllib.request +import urllib.error +import time import subprocess import sys -import time -import traceback -import urllib.error -import urllib.parse -import urllib.request from argparse import ArgumentParser -from pathlib import Path -from typing import Optional +import traceback +import html +from distutils.version import LooseVersion +import logging +import copy +import urllib.parse -import git - -from . import _, common, metadata, net -from .exception import ( - FDroidException, - MetaDataException, - NoSubmodulesException, - VCSException, -) - -# https://gitlab.com/fdroid/checkupdates-runner/-/blob/1861899262a62a4ed08fa24e5449c0368dfb7617/.gitlab-ci.yml#L36 -BOT_EMAIL = 'fdroidci@bubu1.eu' +from . import _ +from . import common +from . import metadata +from .exception import VCSException, NoSubmodulesException, FDroidException, MetaDataException -def check_http(app: metadata.App) -> tuple[Optional[str], Optional[int]]: - """Check for a new version by looking at a document retrieved via HTTP. +# Check for a new version by looking at a document retrieved via HTTP. +# The app's Update Check Data field is used to provide the information +# required. +def check_http(app): - The app's UpdateCheckData field is used to provide the information - required. + ignoreversions = app.UpdateCheckIgnore + ignoresearch = re.compile(ignoreversions).search if ignoreversions else None - Parameters - ---------- - app - The App instance to check for updates for. + try: - Returns - ------- - version - The found versionName or None if the versionName should be ignored - according to UpdateCheckIgnore. - vercode - The found versionCode or None if the versionCode should be ignored - according to UpdateCheckIgnore. + if not app.UpdateCheckData: + raise FDroidException('Missing Update Check Data') - Raises - ------ - :exc:`~fdroidserver.exception.FDroidException` - If UpdateCheckData is missing or is an invalid URL or if there is no - match for the provided versionName or versionCode regex. - """ - if not app.UpdateCheckData: - raise FDroidException('Missing Update Check Data') - - urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|') - parsed = urllib.parse.urlparse(urlcode) - if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https': - raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode)) - if urlver != '.': - parsed = urllib.parse.urlparse(urlver) + urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|') + parsed = urllib.parse.urlparse(urlcode) if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https': - raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlver)) + raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode)) + if urlver != '.': + parsed = urllib.parse.urlparse(urlver) + if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https': + raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode)) - logging.debug("...requesting {0}".format(urlcode)) - req = urllib.request.Request(urlcode, None, headers=net.HEADERS) - resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above - page = resp.read().decode('utf-8') + vercode = None + if len(urlcode) > 0: + logging.debug("...requesting {0}".format(urlcode)) + req = urllib.request.Request(urlcode, None) + resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above + page = resp.read().decode('utf-8') - m = re.search(codeex, page) - if not m: - raise FDroidException("No RE match for versionCode") - vercode = common.version_code_string_to_int(m.group(1).strip()) + m = re.search(codeex, page) + if not m: + raise FDroidException("No RE match for version code") + vercode = m.group(1).strip() - if urlver != '.': - logging.debug("...requesting {0}".format(urlver)) - req = urllib.request.Request(urlver, None) - resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above - page = resp.read().decode('utf-8') + version = "??" + if len(urlver) > 0: + if urlver != '.': + logging.debug("...requesting {0}".format(urlver)) + req = urllib.request.Request(urlver, None) + resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above + page = resp.read().decode('utf-8') - m = re.search(verex, page) - if not m: - raise FDroidException("No RE match for version") - version = m.group(1) + m = re.search(verex, page) + if not m: + raise FDroidException("No RE match for version") + version = m.group(1) - if app.UpdateCheckIgnore and re.search(app.UpdateCheckIgnore, version): - logging.info("Version {version} for {appid} is ignored".format(version=version, appid=app.id)) - return (None, None) - - return (version, vercode) + if ignoresearch and version: + if not ignoresearch(version): + return (version, vercode) + else: + return (None, ("Version {version} is ignored").format(version=version)) + else: + return (version, vercode) + except FDroidException: + msg = "Could not complete http check for app {0} due to unknown error: {1}".format(app.id, traceback.format_exc()) + return (None, msg) -def check_tags(app: metadata.App, pattern: str) -> tuple[str, int, str]: - """Check for a new version by looking at the tags in the source repo. +# Check for a new version by looking at the tags in the source repo. +# Whether this can be used reliably or not depends on +# the development procedures used by the project's developers. Use it with +# caution, because it's inappropriate for many projects. +# Returns (None, "a message") if this didn't work, or (version, vercode, tag) for +# the details of the current version. +def check_tags(app, pattern): - Whether this can be used reliably or not depends on - the development procedures used by the project's developers. Use it with - caution, because it's inappropriate for many projects. + try: - Parameters - ---------- - app - The App instance to check for updates for. - pattern - The pattern a tag needs to match to be considered. + if app.RepoType == 'srclib': + build_dir = os.path.join('build', 'srclib', app.Repo) + repotype = common.getsrclibvcs(app.Repo) + else: + build_dir = os.path.join('build', app.id) + repotype = app.RepoType - Returns - ------- - versionName - The highest found versionName. - versionCode - The highest found versionCode. - ref - The Git reference, commit hash or tag name, of the highest found - versionName, versionCode. + if repotype not in ('git', 'git-svn', 'hg', 'bzr'): + return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None) - Raises - ------ - :exc:`~fdroidserver.exception.MetaDataException` - If this function is not suitable for the RepoType of the app or - information is missing to perform this type of check. - :exc:`~fdroidserver.exception.FDroidException` - If no matching tags or no information whatsoever could be found. - """ - if app.RepoType == 'srclib': - build_dir = Path('build/srclib') / app.Repo - repotype = common.getsrclibvcs(app.Repo) - else: - build_dir = Path('build') / app.id - repotype = app.RepoType + if repotype == 'git-svn' and ';' not in app.Repo: + return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None) - if repotype not in ('git', 'git-svn', 'hg', 'bzr'): - raise MetaDataException(_('Tags update mode only works for git, hg, bzr and git-svn repositories currently')) + # Set up vcs interface and make sure we have the latest code... + vcs = common.getvcs(app.RepoType, app.Repo, build_dir) - if repotype == 'git-svn' and ';' not in app.Repo: - raise MetaDataException(_('Tags update mode used in git-svn, but the repo was not set up with tags')) + vcs.gotorevision(None) - # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app.RepoType, app.Repo, build_dir) + last_build = app.get_last_build() - vcs.gotorevision(None) - - last_build = get_last_build_from_app(app) - - try_init_submodules(app, last_build, vcs) - - htag = None - hver = None - hcode = 0 - - tags = [] - if repotype == 'git': - tags = vcs.latesttags() - else: - tags = vcs.gettags() - if not tags: - raise FDroidException(_('No tags found')) - - logging.debug("All tags: " + ','.join(tags)) - if pattern: - pat = re.compile(pattern) - tags = [tag for tag in tags if pat.match(tag)] - if not tags: - raise FDroidException(_('No matching tags found')) - logging.debug("Matching tags: " + ','.join(tags)) - - if len(tags) > 5 and repotype == 'git': - tags = tags[:5] - logging.debug("Latest tags: " + ','.join(tags)) - - for tag in tags: - logging.debug("Check tag: '{0}'".format(tag)) - vcs.gotorevision(tag) try_init_submodules(app, last_build, vcs) - if app.UpdateCheckData: - filecode, codeex, filever, verex = app.UpdateCheckData.split('|') + hpak = None + htag = None + hver = None + hcode = "0" - if filecode: - filecode = build_dir / filecode - if not filecode.is_file(): - logging.debug("UpdateCheckData file {0} not found in tag {1}".format(filecode, tag)) - continue - filecontent = filecode.read_text() - else: - filecontent = tag - - vercode = tag - if codeex: - m = re.search(codeex, filecontent) - if not m: - logging.debug(f"UpdateCheckData regex {codeex} for versionCode" - f" has no match in tag {tag}") - continue - - vercode = m.group(1).strip() - - if filever: - if filever != '.': - filever = build_dir / filever - if filever.is_file(): - filecontent = filever.read_text() - else: - logging.debug("UpdateCheckData file {0} not found in tag {1}".format(filever, tag)) - else: - filecontent = tag - - version = tag - if verex: - m = re.search(verex, filecontent) - if not m: - logging.debug(f"UpdateCheckData regex {verex} for versionName" - f" has no match in tag {tag}") - continue - - version = m.group(1) - - logging.debug("UpdateCheckData found version {0} ({1})" - .format(version, vercode)) - vercode = common.version_code_string_to_int(vercode) - if vercode > hcode: - htag = tag - hcode = vercode - hver = version + tags = [] + if repotype == 'git': + tags = vcs.latesttags() else: + tags = vcs.gettags() + if not tags: + return (None, "No tags found", None) + + logging.debug("All tags: " + ','.join(tags)) + if pattern: + pat = re.compile(pattern) + tags = [tag for tag in tags if pat.match(tag)] + if not tags: + return (None, "No matching tags found", None) + logging.debug("Matching tags: " + ','.join(tags)) + + if len(tags) > 5 and repotype == 'git': + tags = tags[:5] + logging.debug("Latest tags: " + ','.join(tags)) + + for tag in tags: + logging.debug("Check tag: '{0}'".format(tag)) + vcs.gotorevision(tag) + for subdir in possible_subdirs(app): - root_dir = build_dir / subdir + if subdir == '.': + root_dir = build_dir + else: + root_dir = os.path.join(build_dir, subdir) paths = common.manifest_paths(root_dir, last_build.gradle) - version, vercode, _package = common.parse_androidmanifests(paths, app) - if version in ('Unknown', 'Ignore'): - version = tag + version, vercode, package = common.parse_androidmanifests(paths, app) if vercode: logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})" .format(subdir, version, vercode)) - if vercode > hcode: + i_vercode = common.version_code_string_to_int(vercode) + if i_vercode > common.version_code_string_to_int(hcode): + hpak = package htag = tag - hcode = vercode + hcode = str(i_vercode) hver = version - if hver: - if htag != tags[0]: - logging.warning( - "{appid}: latest tag {tag} does not contain highest version {version}".format( - appid=app.id, tag=tags[0], version=hver - ) - ) - try: - commit = vcs.getref(htag) - if commit: - return (hver, hcode, commit) - except VCSException: - pass - return (hver, hcode, htag) - raise FDroidException(_("Couldn't find any version information")) + if not hpak: + return (None, "Couldn't find package ID", None) + if hver: + return (hver, hcode, htag) + return (None, "Couldn't find any version information", None) + + except VCSException as vcse: + msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse) + return (None, msg, None) + except Exception: + msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc()) + return (None, msg, None) -def check_repomanifest(app: metadata.App, branch: Optional[str] = None) -> tuple[str, int]: - """Check for a new version by looking at the AndroidManifest.xml at the HEAD of the source repo. +# Check for a new version by looking at the AndroidManifest.xml at the HEAD +# of the source repo. Whether this can be used reliably or not depends on +# the development procedures used by the project's developers. Use it with +# caution, because it's inappropriate for many projects. +# Returns (None, "a message") if this didn't work, or (version, vercode) for +# the details of the current version. +def check_repomanifest(app, branch=None): - Whether this can be used reliably or not depends on - the development procedures used by the project's developers. Use it with - caution, because it's inappropriate for many projects. + try: - Parameters - ---------- - app - The App instance to check for updates for. - branch - The VCS branch where to search for versionCode, versionName. + if app.RepoType == 'srclib': + build_dir = os.path.join('build', 'srclib', app.Repo) + repotype = common.getsrclibvcs(app.Repo) + else: + build_dir = os.path.join('build', app.id) + repotype = app.RepoType - Returns - ------- - versionName - The highest found versionName. - versionCode - The highest found versionCode. + # Set up vcs interface and make sure we have the latest code... + vcs = common.getvcs(app.RepoType, app.Repo, build_dir) - Raises - ------ - :exc:`~fdroidserver.exception.FDroidException` - If no package id or no version information could be found. - """ - if app.RepoType == 'srclib': - build_dir = Path('build/srclib') / app.Repo - repotype = common.getsrclibvcs(app.Repo) - else: - build_dir = Path('build') / app.id - repotype = app.RepoType + if repotype == 'git': + if branch: + branch = 'origin/' + branch + vcs.gotorevision(branch) + elif repotype == 'git-svn': + vcs.gotorevision(branch) + elif repotype == 'hg': + vcs.gotorevision(branch) + elif repotype == 'bzr': + vcs.gotorevision(None) - # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app.RepoType, app.Repo, build_dir) + last_build = metadata.Build() + if len(app.builds) > 0: + last_build = app.builds[-1] + + try_init_submodules(app, last_build, vcs) + + hpak = None + hver = None + hcode = "0" + for subdir in possible_subdirs(app): + if subdir == '.': + root_dir = build_dir + else: + root_dir = os.path.join(build_dir, subdir) + paths = common.manifest_paths(root_dir, last_build.gradle) + version, vercode, package = common.parse_androidmanifests(paths, app) + if vercode: + logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})" + .format(subdir, version, vercode)) + if int(vercode) > int(hcode): + hpak = package + hcode = str(int(vercode)) + hver = version + + if not hpak: + return (None, "Couldn't find package ID") + if hver: + return (hver, hcode) + return (None, "Couldn't find any version information") + + except VCSException as vcse: + msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse) + return (None, msg) + except Exception: + msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc()) + return (None, msg) + + +def check_repotrunk(app): + + try: + if app.RepoType == 'srclib': + build_dir = os.path.join('build', 'srclib', app.Repo) + repotype = common.getsrclibvcs(app.Repo) + else: + build_dir = os.path.join('build', app.id) + repotype = app.RepoType + + if repotype not in ('git-svn', ): + return (None, 'RepoTrunk update mode only makes sense in git-svn repositories') + + # Set up vcs interface and make sure we have the latest code... + vcs = common.getvcs(app.RepoType, app.Repo, build_dir) - if repotype == 'git': - if branch: - branch = 'origin/' + branch - vcs.gotorevision(branch) - elif repotype == 'git-svn': - vcs.gotorevision(branch) - elif repotype == 'hg': - vcs.gotorevision(branch) - elif repotype == 'bzr': vcs.gotorevision(None) - last_build = get_last_build_from_app(app) - try_init_submodules(app, last_build, vcs) - - hpak = None - hver = None - hcode = 0 - for subdir in possible_subdirs(app): - root_dir = build_dir / subdir - paths = common.manifest_paths(root_dir, last_build.gradle) - version, vercode, package = common.parse_androidmanifests(paths, app) - if vercode: - logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})" - .format(subdir, version, vercode)) - if vercode > hcode: - hpak = package - hcode = vercode - hver = version - - if not hpak: - raise FDroidException(_("Couldn't find package ID")) - if hver: - return (hver, hcode) - raise FDroidException(_("Couldn't find any version information")) + ref = vcs.getref() + return (ref, ref) + except VCSException as vcse: + msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse) + return (None, msg) + except Exception: + msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc()) + return (None, msg) -def try_init_submodules(app: metadata.App, last_build: metadata.Build, vcs: common.vcs): - """Try to init submodules if the last build entry uses them. +# Check for a new version by looking at the Google Play Store. +# Returns (None, "a message") if this didn't work, or (version, None) for +# the details of the current version. +def check_gplay(app): + time.sleep(15) + url = 'https://play.google.com/store/apps/details?id=' + app.id + headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'} + req = urllib.request.Request(url, None, headers) + try: + resp = urllib.request.urlopen(req, None, 20) # nosec B310 URL base is hardcoded above + page = resp.read().decode() + except urllib.error.HTTPError as e: + return (None, str(e.code)) + except Exception as e: + return (None, 'Failed:' + str(e)) + version = None + + m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*', page) + if m: + version = html.unescape(m.group(1)) + + if version == 'Varies with device': + return (None, 'Device-variable version, cannot use this method') + + if not version: + return (None, "Couldn't find version") + return (version.strip(), None) + + +def try_init_submodules(app, last_build, vcs): + """Try to init submodules if the last build entry used them. They might have been removed from the app's repo in the meantime, so if we can't find any submodules we continue with the updates check. If there is any other error in initializing them then we stop the check. @@ -361,101 +326,60 @@ def try_init_submodules(app: metadata.App, last_build: metadata.Build, vcs: comm try: vcs.initsubmodules() except NoSubmodulesException: - logging.info("No submodules present for {}".format(_getappname(app))) - except VCSException: - logging.info("submodule broken for {}".format(_getappname(app))) + logging.info("No submodules present for {}".format(app.Name)) -def dirs_with_manifest(startdir: str): - """Find directories containing a manifest file. - - Yield all directories under startdir that contain any of the manifest - files, and thus are probably an Android project. - - Parameters - ---------- - startdir - Directory to be walked down for search - - Yields - ------ - path : :class:`pathlib.Path` or None - A directory that contains a manifest file of an Android project, None if - no directory could be found - """ +# Return all directories under startdir that contain any of the manifest +# files, and thus are probably an Android project. +def dirs_with_manifest(startdir): for root, dirs, files in os.walk(startdir): - dirs.sort() if any(m in files for m in [ 'AndroidManifest.xml', 'pom.xml', 'build.gradle', 'build.gradle.kts']): - yield Path(root) + yield root -def possible_subdirs(app: metadata.App): - """Try to find a new subdir starting from the root build_dir. +# Tries to find a new subdir starting from the root build_dir. Returns said +# subdir relative to the build dir if found, None otherwise. +def possible_subdirs(app): - Yields said subdir relative to the build dir if found, None otherwise. - - Parameters - ---------- - app - The app to check for subdirs - - Yields - ------ - subdir : :class:`pathlib.Path` or None - A possible subdir, None if no subdir could be found - """ if app.RepoType == 'srclib': - build_dir = Path('build/srclib') / app.Repo + build_dir = os.path.join('build', 'srclib', app.Repo) else: - build_dir = Path('build') / app.id + build_dir = os.path.join('build', app.id) - last_build = get_last_build_from_app(app) + last_build = app.get_last_build() for d in dirs_with_manifest(build_dir): m_paths = common.manifest_paths(d, last_build.gradle) package = common.parse_androidmanifests(m_paths, app)[2] - if package is not None or app.UpdateCheckName == "Ignore": - subdir = d.relative_to(build_dir) + if package is not None: + subdir = os.path.relpath(d, build_dir) logging.debug("Adding possible subdir %s" % subdir) yield subdir -def _getappname(app: metadata.App) -> str: - return common.get_app_display_name(app) +def _getappname(app): + if app.Name: + return app.Name + if app.AutoName: + return app.AutoName + return app.id -def _getcvname(app: metadata.App) -> str: +def _getcvname(app): return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode) -def fetch_autoname(app: metadata.App, tag: str) -> Optional[str]: - """Fetch AutoName. +def fetch_autoname(app, tag): - Get the to be displayed name of an app from the source code and adjust the - App instance in case it is different name has been found. - - Parameters - ---------- - app - The App instance to get the AutoName for. - tag - Tag to fetch AutoName at. - - Returns - ------- - commitmsg - Commit message about the name change. None in case checking for the - name is disabled, a VCSException occured or no name could be found. - """ if not app.RepoType or app.UpdateCheckMode in ('None', 'Static') \ or app.UpdateCheckName == "Ignore": return None if app.RepoType == 'srclib': - build_dir = Path('build/srclib') / app.Repo + build_dir = os.path.join('build', 'srclib', app.Repo) else: - build_dir = Path('build') / app.id + build_dir = os.path.join('build', app.id) try: vcs = common.getvcs(app.RepoType, app.Repo, build_dir) @@ -463,12 +387,15 @@ def fetch_autoname(app: metadata.App, tag: str) -> Optional[str]: except VCSException: return None - last_build = get_last_build_from_app(app) + last_build = app.get_last_build() - logging.debug("...fetch auto name from " + str(build_dir)) + logging.debug("...fetch auto name from " + build_dir) new_name = None for subdir in possible_subdirs(app): - root_dir = build_dir / subdir + if subdir == '.': + root_dir = build_dir + else: + root_dir = os.path.join(build_dir, subdir) new_name = common.fetch_real_name(root_dir, last_build.gradle) if new_name is not None: break @@ -485,103 +412,74 @@ def fetch_autoname(app: metadata.App, tag: str) -> Optional[str]: return commitmsg -def operate_vercode(operation: str, vercode: int) -> int: - """Calculate a new versionCode from a mathematical operation. +def checkupdates_app(app): - Parameters - ---------- - operation - The operation to execute to get the new versionCode. - vercode - The versionCode for replacing "%c" in the operation. - - Returns - ------- - vercode - The new versionCode obtained by executing the operation. - - Raises - ------ - :exc:`~fdroidserver.exception.MetaDataException` - If the operation is invalid. - """ - if not common.VERCODE_OPERATION_RE.match(operation): - raise MetaDataException(_('Invalid VercodeOperation: {field}') - .format(field=operation)) - oldvercode = vercode - op = operation.replace("%c", str(oldvercode)) - vercode = common.calculate_math_string(op) - logging.debug("Applied vercode operation: %d -> %d" % (oldvercode, vercode)) - return vercode - - -def checkupdates_app(app: metadata.App, auto: bool, commit: bool = False) -> None: - """Check for new versions and updated name of a single app. - - Also write back changes to the metadata file and create a Git commit if - requested. - - Parameters - ---------- - app - The app to check for updates for. - - Raises - ------ - :exc:`~fdroidserver.exception.MetaDataException` - If the app has an invalid UpdateCheckMode or AutoUpdateMode. - :exc:`~fdroidserver.exception.FDroidException` - If no version information could be found, the current version is newer - than the found version, auto-update was requested but an app has no - CurrentVersionCode or (Git) commiting the changes failed. - """ # If a change is made, commitmsg should be set to a description of it. - # Only if this is set, changes will be written back to the metadata. + # Only if this is set will changes be written back to the metadata. commitmsg = None tag = None + msg = None + vercode = None + noverok = False mode = app.UpdateCheckMode if mode.startswith('Tags'): pattern = mode[5:] if len(mode) > 4 else None (version, vercode, tag) = check_tags(app, pattern) + if version == 'Unknown': + version = tag + msg = vercode elif mode == 'RepoManifest': (version, vercode) = check_repomanifest(app) + msg = vercode elif mode.startswith('RepoManifest/'): tag = mode[13:] (version, vercode) = check_repomanifest(app, tag) + msg = vercode + elif mode == 'RepoTrunk': + (version, vercode) = check_repotrunk(app) + msg = vercode elif mode == 'HTTP': (version, vercode) = check_http(app) + msg = vercode elif mode in ('None', 'Static'): - logging.debug('Checking disabled') - return + version = None + msg = 'Checking disabled' + noverok = True else: - raise MetaDataException(_('Invalid UpdateCheckMode: {mode}').format(mode=mode)) + version = None + msg = 'Invalid update check method' - if not version or not vercode: - raise FDroidException(_('no version information found')) + if version and vercode and app.VercodeOperation: + if not common.VERCODE_OPERATION_RE.match(app.VercodeOperation): + raise MetaDataException(_('Invalid VercodeOperation: {field}') + .format(field=app.VercodeOperation)) + oldvercode = str(int(vercode)) + op = app.VercodeOperation.replace("%c", oldvercode) + vercode = str(common.calculate_math_string(op)) + logging.debug("Applied vercode operation: %s -> %s" % (oldvercode, vercode)) - if app.VercodeOperation: - vercodes = sorted([ - operate_vercode(operation, vercode) for operation in app.VercodeOperation - ]) - else: - vercodes = [vercode] + if version and any(version.startswith(s) for s in [ + '${', # Gradle variable names + '@string/', # Strings we could not resolve + ]): + version = "Unknown" updating = False - if vercodes[-1] == app.CurrentVersionCode: - logging.debug("...up to date") - elif vercodes[-1] > app.CurrentVersionCode: - logging.debug("...updating - old vercode={0}, new vercode={1}".format( - app.CurrentVersionCode, vercodes[-1])) - app.CurrentVersion = version - app.CurrentVersionCode = vercodes[-1] - updating = True + if version is None: + logmsg = "...{0} : {1}".format(app.id, msg) + if noverok: + logging.info(logmsg) + else: + logging.warning(logmsg) + elif vercode == app.CurrentVersionCode: + logging.info("...up to date") else: - raise FDroidException( - _('current version is newer: old vercode={old}, new vercode={new}').format( - old=app.CurrentVersionCode, new=vercodes[-1] - ) - ) + logging.debug("...updating - old vercode={0}, new vercode={1}".format( + app.CurrentVersionCode, vercode)) + app.CurrentVersion = version + app.CurrentVersionCode = str(int(vercode)) + updating = True commitmsg = fetch_autoname(app, tag) @@ -591,285 +489,63 @@ def checkupdates_app(app: metadata.App, auto: bool, commit: bool = False) -> Non logging.info('...updating to version %s' % ver) commitmsg = 'Update CurrentVersion of %s to %s' % (name, ver) - if auto: + if options.auto: mode = app.AutoUpdateMode if not app.CurrentVersionCode: - raise MetaDataException( - _("Can't auto-update app with no CurrentVersionCode") - ) + logging.warning("Can't auto-update app with no CurrentVersionCode: " + app.id) elif mode in ('None', 'Static'): pass - elif mode.startswith('Version'): + elif mode.startswith('Version '): pattern = mode[8:] suffix = '' if pattern.startswith('+'): try: suffix, pattern = pattern[1:].split(' ', 1) - except ValueError as exc: - raise MetaDataException("Invalid AutoUpdateMode: " + mode) from exc + except ValueError: + raise MetaDataException("Invalid AutoUpdateMode: " + mode) gotcur = False latest = None - builds = app.get('Builds', []) - - if builds: - latest = builds[-1] - if latest.versionCode == app.CurrentVersionCode: + for build in app.builds: + if int(build.versionCode) >= int(app.CurrentVersionCode): gotcur = True - elif latest.versionCode > app.CurrentVersionCode: - raise FDroidException( - _( - 'latest build recipe is newer: ' - 'old vercode={old}, new vercode={new}' - ).format(old=latest.versionCode, new=app.CurrentVersionCode) - ) + if not latest or int(build.versionCode) > int(latest.versionCode): + latest = build + + if int(latest.versionCode) > int(app.CurrentVersionCode): + logging.info("Refusing to auto update, since the latest build is newer") if not gotcur: - newbuilds = copy.deepcopy(builds[-len(vercodes):]) - - # These are either built-in or invalid in newer system versions - bookworm_blocklist = [ - 'apt-get install -y openjdk-11-jdk', - 'apt-get install openjdk-11-jdk-headless', - 'apt-get install -y openjdk-11-jdk-headless', - 'apt-get install -t stretch-backports openjdk-11-jdk-headless openjdk-11-jre-headless', - 'apt-get install -y -t stretch-backports openjdk-11-jdk-headless openjdk-11-jre-headless', - 'apt-get install -y openjdk-17-jdk', - 'apt-get install openjdk-17-jdk-headless', - 'apt-get install -y openjdk-17-jdk-headless', - 'update-alternatives --auto java', - 'update-java-alternatives -a', - ] - - for build in newbuilds: - if "sudo" in build: - if any("openjdk-11" in line for line in build["sudo"]) or any("openjdk-17" in line for line in build["sudo"]): - build["sudo"] = [line for line in build["sudo"] if line not in bookworm_blocklist] - if build["sudo"] == ['apt-get update']: - build["sudo"] = '' - - for b, v in zip(newbuilds, vercodes): - b.disable = False - b.versionCode = v - b.versionName = app.CurrentVersion + suffix.replace( - '%c', str(v) - ) - logging.info("...auto-generating build for " + b.versionName) - if tag: - b.commit = tag - else: - commit = pattern.replace('%v', app.CurrentVersion) - commit = commit.replace('%c', str(v)) - b.commit = commit - - app['Builds'].extend(newbuilds) - + newbuild = copy.deepcopy(latest) + newbuild.disable = False + newbuild.versionCode = app.CurrentVersionCode + newbuild.versionName = app.CurrentVersion + suffix.replace('%c', newbuild.versionCode) + logging.info("...auto-generating build for " + newbuild.versionName) + commit = pattern.replace('%v', app.CurrentVersion) + commit = commit.replace('%c', newbuild.versionCode) + newbuild.commit = commit + app.builds.append(newbuild) name = _getappname(app) ver = _getcvname(app) commitmsg = "Update %s to %s" % (name, ver) else: - raise MetaDataException( - _('Invalid AutoUpdateMode: {mode}').format(mode=mode) - ) + logging.warning('Invalid auto update mode "' + mode + '" on ' + app.id) if commitmsg: metadata.write_metadata(app.metadatapath, app) - if commit: + if options.commit: logging.info("Commiting update for " + app.metadatapath) gitcmd = ["git", "commit", "-m", commitmsg] + if 'auto_author' in config: + gitcmd.extend(['--author', config['auto_author']]) gitcmd.extend(["--", app.metadatapath]) if subprocess.call(gitcmd) != 0: raise FDroidException("Git commit failed") -def get_last_build_from_app(app: metadata.App) -> metadata.Build: - """Get the last build entry of an app.""" - if app.get('Builds'): - return app['Builds'][-1] - else: - return metadata.Build() +def status_update_json(processed, failed): + """Output a JSON file with metadata about this run""" - -def get_upstream_main_branch(git_repo): - refs = list() - for ref in git_repo.remotes.upstream.refs: - if ref.name != 'upstream/HEAD': - refs.append(ref.name) - if len(refs) == 1: - return refs[0] - for name in ('upstream/main', 'upstream/master'): - if name in refs: - return name - try: - with git_repo.config_reader() as reader: - return 'upstream/%s' % reader.get_value('init', 'defaultBranch') - except configparser.NoSectionError: - return 'upstream/main' - - -def checkout_appid_branch(appid): - """Prepare the working branch named after the appid. - - This sets up everything for checkupdates_app() to run and add - commits. If there is an existing branch named after the appid, - and it has commits from users other than the checkupdates-bot, - then this will return False. Otherwise, it returns True. - - The checkupdates-runner must set the committer email address in - the git config. Then any commit with a committer or author that - does not match that will be considered to have human edits. That - email address is currently set in: - https://gitlab.com/fdroid/checkupdates-runner/-/blob/1861899262a62a4ed08fa24e5449c0368dfb7617/.gitlab-ci.yml#L36 - - """ - logging.debug(f'Creating merge request branch for {appid}') - git_repo = git.Repo.init('.') - upstream_main = get_upstream_main_branch(git_repo) - for remote in git_repo.remotes: - remote.fetch() - try: - git_repo.remotes.origin.fetch(f'{appid}:refs/remotes/origin/{appid}') - except Exception as e: - logging.debug('"%s" branch not found on origin remote:\n\t%s', appid, e) - if appid in git_repo.remotes.origin.refs: - start_point = f"origin/{appid}" - for commit in git_repo.iter_commits( - f'{upstream_main}...{start_point}', right_only=True - ): - if commit.committer.email != BOT_EMAIL or commit.author.email != BOT_EMAIL: - return False - else: - start_point = upstream_main - git_repo.git.checkout('-B', appid, start_point) - git_repo.git.rebase(upstream_main, strategy_option='ours', kill_after_timeout=120) - return True - - -def get_changes_versus_ref(git_repo, ref, f): - changes = [] - for m in re.findall( - r"^[+-].*", git_repo.git.diff(f"{ref}", '--', f), flags=re.MULTILINE - ): - if not re.match(r"^(\+\+\+|---) ", m): - changes.append(m) - return changes - - -def push_commits(branch_name='checkupdates'): - """Make git branch then push commits as merge request. - - The appid is parsed from the actual file that was changed so that - only the right branch is ever updated. - - This uses the appid as the standard branch name so that there is - only ever one open merge request per-app. If multiple apps are - included in the branch, then 'checkupdates' is used as branch - name. This is to support the old way operating, e.g. in batches. - - This uses GitLab "Push Options" to create a merge request. Git - Push Options are config data that can be sent via `git push - --push-option=... origin foo`. - - References - ---------- - * https://docs.gitlab.com/ee/user/project/push_options.html - - """ - if branch_name != "checkupdates": - if callable(getattr(git.SymbolicReference, "_check_ref_name_valid", None)): - git.SymbolicReference._check_ref_name_valid(branch_name) - - git_repo = git.Repo.init('.') - upstream_main = get_upstream_main_branch(git_repo) - files = set() - for commit in git_repo.iter_commits(f'{upstream_main}...HEAD', right_only=True): - files.update(commit.stats.files.keys()) - - files = list(files) - if len(files) == 1: - m = re.match(r'metadata/(\S+)\.yml', files[0]) - if m: - branch_name = m.group(1) # appid - if not files: - return - - # https://git-scm.com/docs/git-check-ref-format Git refname can't end with .lock - if branch_name.endswith(".lock"): - branch_name = f"{branch_name}_" - - remote = git_repo.remotes.origin - if branch_name in remote.refs: - if not get_changes_versus_ref(git_repo, f'origin/{branch_name}', files[0]): - return - - git_repo.create_head(branch_name, force=True) - push_options = [ - 'merge_request.create', - 'merge_request.remove_source_branch', - 'merge_request.title=bot: ' + git_repo.branches[branch_name].commit.summary, - 'merge_request.description=' - + '~%s checkupdates-bot run %s' % (branch_name, os.getenv('CI_JOB_URL')), - ] - - # mark as draft if there are only changes to CurrentVersion: - current_version_only = True - for m in get_changes_versus_ref(git_repo, upstream_main, files[0]): - if not re.match(r"^[-+]CurrentVersion", m): - current_version_only = False - break - if current_version_only: - push_options.append('merge_request.draft') - - progress = git.RemoteProgress() - - pushinfos = remote.push( - f"HEAD:refs/heads/{branch_name}", - progress=progress, - force=True, - set_upstream=True, - push_option=push_options, - ) - - for pushinfo in pushinfos: - logging.info(pushinfo.summary) - # Show potentially useful messages from git remote - if progress: - for line in progress.other_lines: - logging.info(line) - if pushinfo.flags & ( - git.remote.PushInfo.ERROR - | git.remote.PushInfo.REJECTED - | git.remote.PushInfo.REMOTE_FAILURE - | git.remote.PushInfo.REMOTE_REJECTED - ): - raise FDroidException( - f'{remote.url} push failed: {pushinfo.flags} {pushinfo.summary}' - ) - else: - logging.info(remote.url + ': ' + pushinfo.summary) - - -def prune_empty_appid_branches(git_repo=None, main_branch='main'): - """Remove empty branches from checkupdates-bot git remote.""" - if git_repo is None: - git_repo = git.Repo.init('.') - upstream_main = get_upstream_main_branch(git_repo) - main_branch = upstream_main.split('/')[1] - - remote = git_repo.remotes.origin - remote.update(prune=True) - merged_branches = git_repo.git().branch(remotes=True, merged=upstream_main).split() - for remote_branch in merged_branches: - if not remote_branch or '/' not in remote_branch: - continue - if remote_branch.split('/')[1] not in (main_branch, 'HEAD'): - for ref in git_repo.remotes.origin.refs: - if remote_branch == ref.name: - remote.push(':%s' % ref.remote_head, force=True) # rm remote branch - - -def status_update_json(processed: list, failed: dict) -> None: - """Output a JSON file with metadata about this run.""" logging.debug(_('Outputting JSON')) output = common.setup_status_output(start_timestamp) if processed: @@ -879,17 +555,46 @@ def status_update_json(processed: list, failed: dict) -> None: common.write_status_json(output) +def update_wiki(gplaylog, locallog): + if config.get('wiki_server') and config.get('wiki_path'): + try: + import mwclient + site = mwclient.Site((config['wiki_protocol'], config['wiki_server']), + path=config['wiki_path']) + site.login(config['wiki_user'], config['wiki_password']) + + # Write a page with the last build log for this version code + wiki_page_path = 'checkupdates_' + time.strftime('%s', start_timestamp) + newpage = site.Pages[wiki_page_path] + txt = '' + txt += "* command line: " + ' '.join(sys.argv) + "\n" + txt += common.get_git_describe_link() + txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n' + txt += "* completed at " + common.get_wiki_timestamp() + '\n' + txt += "\n\n" + txt += common.get_android_tools_version_log() + txt += "\n\n" + if gplaylog: + txt += '== --gplay check ==\n\n' + txt += gplaylog + if locallog: + txt += '== local source check ==\n\n' + txt += locallog + newpage.save(txt, summary='Run log') + newpage = site.Pages['checkupdates'] + newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect') + except Exception as e: + logging.error(_('Error while attempting to publish log: %s') % e) + + config = None +options = None start_timestamp = time.gmtime() def main(): - """Check for updates for one or more apps. - The behaviour of this function is influenced by the configuration file as - well as command line parameters. - """ - global config + global config, options # Parse command line... parser = ArgumentParser() @@ -901,15 +606,15 @@ def main(): help=_("Only process apps with auto-updates")) parser.add_argument("--commit", action="store_true", default=False, help=_("Commit changes")) - parser.add_argument("--merge-request", action="store_true", default=False, - help=_("Commit changes, push, then make a merge request")) parser.add_argument("--allow-dirty", action="store_true", default=False, help=_("Run on git repo that has uncommitted changes")) + parser.add_argument("--gplay", action="store_true", default=False, + help=_("Only print differences with the Play Store")) metadata.add_metadata_arguments(parser) - options = common.parse_args(parser) + options = parser.parse_args() metadata.warnings_action = options.W - config = common.read_config() + config = common.read_config(options) if not options.allow_dirty: status = subprocess.check_output(['git', 'status', '--porcelain']) @@ -917,15 +622,42 @@ def main(): logging.error(_('Build metadata git repo has uncommited changes!')) sys.exit(1) - if options.merge_request and not (options.appid and len(options.appid) == 1): - logging.error(_('--merge-request only runs on a single appid!')) - sys.exit(1) + # Get all apps... + allapps = metadata.read_metadata() - apps = common.read_app_args(options.appid) + apps = common.read_app_args(options.appid, allapps, False) + gplaylog = '' + if options.gplay: + for appid, app in apps.items(): + gplaylog += '* ' + appid + '\n' + version, reason = check_gplay(app) + if version is None: + if reason == '404': + logging.info("{0} is not in the Play Store".format(_getappname(app))) + else: + logging.info("{0} encountered a problem: {1}".format(_getappname(app), reason)) + if version is not None: + stored = app.CurrentVersion + if not stored: + logging.info("{0} has no Current Version but has version {1} on the Play Store" + .format(_getappname(app), version)) + elif LooseVersion(stored) < LooseVersion(version): + logging.info("{0} has version {1} on the Play Store, which is bigger than {2}" + .format(_getappname(app), version, stored)) + else: + if stored != version: + logging.info("{0} has version {1} on the Play Store, which differs from {2}" + .format(_getappname(app), version, stored)) + else: + logging.info("{0} has the same version {1} on the Play Store" + .format(_getappname(app), version)) + update_wiki(gplaylog, None) + return + + locallog = '' processed = [] failed = dict() - exit_code = 0 for appid, app in apps.items(): if options.autoonly and app.AutoUpdateMode in ('None', 'Static'): @@ -934,33 +666,20 @@ def main(): msg = _("Processing {appid}").format(appid=appid) logging.info(msg) + locallog += '* ' + msg + '\n' try: - if options.merge_request: - if not checkout_appid_branch(appid): - msg = _("...checkupdate failed for {appid} : {error}").format( - appid=appid, - error='Open merge request with human edits, skipped.', - ) - logging.warning(msg) - failed[appid] = msg - continue - - checkupdates_app(app, options.auto, options.commit or options.merge_request) + checkupdates_app(app) processed.append(appid) except Exception as e: msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e) logging.error(msg) - logging.debug(traceback.format_exc()) + locallog += msg + '\n' failed[appid] = str(e) - exit_code = 1 - - if options.appid and options.merge_request: - push_commits() - prune_empty_appid_branches() + update_wiki(None, locallog) status_update_json(processed, failed) - sys.exit(exit_code) + logging.info(_("Finished")) if __name__ == "__main__": diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 127976c3..a9f14c23 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1,16 +1,8 @@ #!/usr/bin/env python3 # # common.py - part of the FDroid server tools -# -# Copyright (C) 2010-2016, Ciaran Gultnieks, ciaran@ciarang.com -# Copyright (C) 2013-2017, Daniel Martí -# Copyright (C) 2013-2021, Hans-Christoph Steiner -# Copyright (C) 2017-2018, Torsten Grote -# Copyright (C) 2017, tobiasKaminsky -# Copyright (C) 2017-2021, Michael Pöhn -# Copyright (C) 2017,2021, mimi89999 -# Copyright (C) 2019-2021, Jochen Sprickerhof -# Copyright (C) 2021, Felix C. Stegerman +# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com +# Copyright (C) 2013-2014 Daniel Martí # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -25,106 +17,70 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +# common.py is imported by all modules, so do not import third-party +# libraries here as they will become a requirement for all commands. -"""Collection of functions shared by subcommands. - -This is basically the "shared library" for all the fdroid subcommands. -The contains core functionality and a number of utility functions. -This is imported by all modules, so do not import third-party -libraries here as they will become a requirement for all commands. - -Config ------- - -Parsing and using the configuration settings from config.yml is -handled here. The data format is YAML 1.2. The config has its own -supported data types: - -* Boolean (e.g. deploy_process_logs:) -* Integer (e.g. archive_older:, repo_maxage:) -* String-only (e.g. repo_name:, sdk_path:) -* Multi-String (string, list of strings, or list of dicts with - strings, e.g. serverwebroot:, mirrors:) - -String-only fields can also use a special value {env: varname}, which -is a dict with a single key 'env' and a value that is the name of the -environment variable to include. - -""" - -import ast -import base64 -import copy -import difflib -import filecmp -import glob -import gzip -import hashlib +import git import io -import itertools -import json -import logging -import operator import os +import sys import re +import ast +import gzip import shutil -import socket +import glob import stat import subprocess -import sys -import tempfile import time +import operator +import logging +import hashlib +import socket +import base64 +import urllib.parse +import urllib.request +import yaml import zipfile -from argparse import BooleanOptionalAction -from base64 import urlsafe_b64encode +import tempfile +import json + +# TODO change to only import defusedxml once its installed everywhere +try: + import defusedxml.ElementTree as XMLElementTree +except ImportError: + import xml.etree.ElementTree as XMLElementTree # nosec this is a fallback only + from binascii import hexlify from datetime import datetime, timedelta, timezone -from pathlib import Path +from distutils.version import LooseVersion from queue import Queue -from typing import List -from urllib.parse import urlparse, urlsplit, urlunparse from zipfile import ZipFile -import defusedxml.ElementTree as XMLElementTree -import git -from asn1crypto import cms +from pyasn1.codec.der import decoder, encoder +from pyasn1_modules import rfc2315 +from pyasn1.error import PyAsn1Error import fdroidserver.metadata +import fdroidserver.lint from fdroidserver import _ -from fdroidserver._yaml import config_dump, yaml -from fdroidserver.exception import ( - BuildException, - FDroidException, - MetaDataException, - NoSubmodulesException, - VCSException, - VerificationException, -) - -from . import apksigcopier, common +from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\ + BuildException, VerificationException, MetaDataException from .asynchronousfilereader import AsynchronousFileReader -from .looseversion import LooseVersion # The path to this fdroidserver distribution FDROID_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) -# There needs to be a default, and this is the most common for software. -DEFAULT_LOCALE = 'en-US' - # this is the build-tools version, aapt has a separate version that # has to be manually set in test_aapt_version() MINIMUM_AAPT_BUILD_TOOLS_VERSION = '26.0.0' -# 33.0.x has a bug that verifies APKs it shouldn't https://gitlab.com/fdroid/fdroidserver/-/issues/1253 -# 31.0.0 is the first version to support --v4-signing-enabled. -# we only require 30.0.0 for now as that's the version in buster-backports, see also signindex.py # 26.0.2 is the first version recognizing md5 based signatures as valid again # (as does android, so we want that) -MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION = '30.0.0' +MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION = '26.0.2' VERCODE_OPERATION_RE = re.compile(r'^([ 0-9/*+-]|%c)+$') # A signature block file with a .DSA, .RSA, or .EC extension -SIGNATURE_BLOCK_FILE_REGEX = re.compile(r'\AMETA-INF/.*\.(DSA|EC|RSA)\Z', re.DOTALL) +SIGNATURE_BLOCK_FILE_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$') APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk') APK_ID_TRIPLET_REGEX = re.compile(r"^package: name='(\w[^']*)' versionCode='([^']+)' versionName='([^']*)'") STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+') @@ -134,68 +90,48 @@ VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[ re.IGNORECASE) ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''') +SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?') +GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:([^'"]+)['"]''') + MAX_VERSION_CODE = 0x7fffffff # Java's Integer.MAX_VALUE (2147483647) XMLNS_ANDROID = '{http://schemas.android.com/apk/res/android}' -# https://docs.gitlab.com/ee/user/gitlab_com/#gitlab-pages -GITLAB_COM_PAGES_MAX_SIZE = 1000000000 - -# the names used for things that are configured per-repo -ANTIFEATURES_CONFIG_NAME = 'antiFeatures' -CATEGORIES_CONFIG_NAME = 'categories' -CONFIG_CONFIG_NAME = 'config' -MIRRORS_CONFIG_NAME = 'mirrors' -RELEASECHANNELS_CONFIG_NAME = "releaseChannels" -CONFIG_NAMES = ( - ANTIFEATURES_CONFIG_NAME, - CATEGORIES_CONFIG_NAME, - CONFIG_CONFIG_NAME, - MIRRORS_CONFIG_NAME, - RELEASECHANNELS_CONFIG_NAME, -) - -CONFIG_FILE = 'config.yml' - config = None options = None env = None orig_path = None -def get_default_cachedir(): - """Get a cachedir, using platformdirs for cross-platform, but works without. - - Once platformdirs is installed everywhere, this function can be - removed. - - """ - appname = __name__.split('.')[0] - try: - import platformdirs - - return platformdirs.user_cache_dir(appname, 'F-Droid') - except ImportError: - return str(Path.home() / '.cache' / appname) - - -# All paths in the config must be strings, never pathlib.Path instances default_config = { 'sdk_path': "$ANDROID_HOME", - 'ndk_paths': {}, - 'cachedir': get_default_cachedir(), + 'ndk_paths': { + 'r10e': None, + 'r11c': None, + 'r12b': "$ANDROID_NDK", + 'r13b': None, + 'r14b': None, + 'r15c': None, + 'r16b': None, + }, + 'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'java_paths': None, 'scan_binary': False, 'ant': "ant", 'mvn3': "mvn", - 'gradle': shutil.which('gradlew-fdroid'), + 'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'), + 'gradle_version_dir': os.path.join(os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'gradle'), 'sync_from_local_copy_dir': False, 'allow_disabled_algorithms': False, - 'keep_when_not_allowed': False, 'per_app_repos': False, 'make_current_version_link': False, 'current_version_name_source': 'Name', 'deploy_process_logs': False, + 'update_stats': False, + 'stats_ignore': [], + 'stats_server': None, + 'stats_user': None, + 'stats_to_carbon': False, 'repo_maxage': 0, 'build_server_always': False, 'keystore': 'keystore.p12', @@ -210,129 +146,33 @@ default_config = { }, 'keyaliases': {}, 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo", - 'repo_name': "My First F-Droid Repo Demo", - 'repo_icon': "icon.png", - 'repo_description': _("""This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/fdroid."""), # type: ignore - 'archive_name': 'My First F-Droid Archive Demo', - 'archive_description': _('These are the apps that have been archived from the main repo.'), # type: ignore + 'repo_name': "My First FDroid Repo Demo", + 'repo_icon': "fdroid-icon.png", + 'repo_description': _(''' + This is a repository of apps to be used with FDroid. Applications in this + repository are either official binaries built by the original application + developers, or are binaries built from source by f-droid.org using the + tools on https://gitlab.com/fdroid. + '''), + 'archive_description': _('These are the apps that have been archived from the main repo.'), 'archive_older': 0, + 'lint_licenses': fdroidserver.lint.APPROVED_LICENSES, 'git_mirror_size_limit': 10000000000, - 'scanner_signature_sources': ['suss'], } -def get_options(): - """Return options as set up by parse_args(). - - This provides an easy way to get the global instance without - having to think about very confusing import and submodule - visibility. The code should be probably refactored so it does not - need this. If each individual option value was always passed to - functions as args, for example. - - https://docs.python.org/3/reference/import.html#submodules - - """ - return fdroidserver.common.options - - -def parse_args(parser): - """Call parser.parse_args(), store result in module-level variable and return it. - - This is needed to set up the copy of the options instance in the - fdroidserver.common module. A subcommand only needs to call this - if it uses functions from fdroidserver.common that expect the - "options" variable to be initialized. - - """ - fdroidserver.common.options = parser.parse_args() - return fdroidserver.common.options - - def setup_global_opts(parser): try: # the buildserver VM might not have PIL installed from PIL import PngImagePlugin - logger = logging.getLogger(PngImagePlugin.__name__) logger.setLevel(logging.INFO) # tame the "STREAM" debug messages except ImportError: pass - parser.add_argument( - "-v", - "--verbose", - action="store_true", - default=False, - help=_("Spew out even more information than normal"), - ) - parser.add_argument( - "-q", - "--quiet", - action="store_true", - default=False, - help=_("Restrict output to warnings and errors"), - ) - parser.add_argument( - "--color", - action=BooleanOptionalAction, - default=None, - help=_("Color the log output"), - ) - - -class ColorFormatter(logging.Formatter): - - def __init__(self, msg): - logging.Formatter.__init__(self, msg) - - bright_black = "\x1b[90;20m" - yellow = "\x1b[33;20m" - red = "\x1b[31;20m" - bold_red = "\x1b[31;1m" - reset = "\x1b[0m" - - self.FORMATS = { - logging.DEBUG: bright_black + msg + reset, - logging.INFO: reset + msg + reset, # use default color - logging.WARNING: yellow + msg + reset, - logging.ERROR: red + msg + reset, - logging.CRITICAL: bold_red + msg + reset - } - - def format(self, record): - log_fmt = self.FORMATS.get(record.levelno) - formatter = logging.Formatter(log_fmt) - return formatter.format(record) - - -def set_console_logging(verbose=False, color=False): - """Globally set logging to output nicely to the console.""" - - class _StdOutFilter(logging.Filter): - def filter(self, record): - return record.levelno < logging.ERROR - - if verbose: - level = logging.DEBUG - else: - level = logging.ERROR - - if color or (color is None and sys.stdout.isatty()): - formatter = ColorFormatter - else: - formatter = logging.Formatter - - stdout_handler = logging.StreamHandler(sys.stdout) - stdout_handler.addFilter(_StdOutFilter()) - stdout_handler.setFormatter(formatter('%(message)s')) - - stderr_handler = logging.StreamHandler(sys.stderr) - stderr_handler.setLevel(logging.ERROR) - stderr_handler.setFormatter(formatter(_('ERROR: %(message)s'))) - - logging.basicConfig( - force=True, level=level, handlers=[stdout_handler, stderr_handler] - ) + parser.add_argument("-v", "--verbose", action="store_true", default=False, + help=_("Spew out even more information than normal")) + parser.add_argument("-q", "--quiet", action="store_true", default=False, + help=_("Restrict output to warnings and errors")) def _add_java_paths_to_config(pathlist, thisconfig): @@ -351,18 +191,18 @@ def _add_java_paths_to_config(pathlist, thisconfig): j = os.path.basename(d) # the last one found will be the canonical one, so order appropriately for regex in [ - r'^1\.([126-9][0-9]?)\.0\.jdk$', # OSX - r'^jdk1\.([126-9][0-9]?)\.0_[0-9]+.jdk$', # OSX and Oracle tarball - r'^jdk1\.([126-9][0-9]?)\.0_[0-9]+$', # Oracle Windows - r'^jdk([126-9][0-9]?)-openjdk$', # Arch - r'^java-([126-9][0-9]?)-openjdk$', # Arch - r'^java-([126-9][0-9]?)-jdk$', # Arch (oracle) - r'^java-1\.([126-9][0-9]?)\.0-.*$', # RedHat - r'^java-([126-9][0-9]?)-oracle$', # Debian WebUpd8 - r'^jdk-([126-9][0-9]?)-oracle-.*$', # Debian make-jpkg - r'^java-([126-9][0-9]?)-openjdk-.*$', # Debian - r'^oracle-jdk-bin-1\.([126-9][0-9]?).*$', # Gentoo (oracle) - r'^icedtea-bin-([126-9][0-9]?).*$', # Gentoo (openjdk) + r'^1\.([16-9][0-9]?)\.0\.jdk$', # OSX + r'^jdk1\.([16-9][0-9]?)\.0_[0-9]+.jdk$', # OSX and Oracle tarball + r'^jdk1\.([16-9][0-9]?)\.0_[0-9]+$', # Oracle Windows + r'^jdk([16-9][0-9]?)-openjdk$', # Arch + r'^java-([16-9][0-9]?)-openjdk$', # Arch + r'^java-([16-9][0-9]?)-jdk$', # Arch (oracle) + r'^java-1\.([16-9][0-9]?)\.0-.*$', # RedHat + r'^java-([16-9][0-9]?)-oracle$', # Debian WebUpd8 + r'^jdk-([16-9][0-9]?)-oracle-.*$', # Debian make-jpkg + r'^java-([16-9][0-9]?)-openjdk-[^c][^o][^m].*$', # Debian + r'^oracle-jdk-bin-1\.([17-9][0-9]?).*$', # Gentoo (oracle) + r'^icedtea-bin-([17-9][0-9]?).*$', # Gentoo (openjdk) ]: m = re.match(regex, j) if not m: @@ -373,24 +213,13 @@ def _add_java_paths_to_config(pathlist, thisconfig): def fill_config_defaults(thisconfig): - """Fill in the global config dict with relevant defaults. - - For config values that have a path that can be expanded, e.g. an - env var or a ~/, this will store the original value using "_orig" - appended to the key name so that if the config gets written out, - it will preserve the original, unexpanded string. - - """ for k, v in default_config.items(): if k not in thisconfig: - if isinstance(v, dict) or isinstance(v, list): - thisconfig[k] = v.copy() - else: - thisconfig[k] = v + thisconfig[k] = v # Expand paths (~users and $vars) def expand_path(path): - if not path or not isinstance(path, str): + if path is None: return None orig = path path = os.path.expanduser(path) @@ -399,7 +228,7 @@ def fill_config_defaults(thisconfig): return None return path - for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore']: + for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']: v = thisconfig[k] exp = expand_path(v) if exp is not None: @@ -410,20 +239,19 @@ def fill_config_defaults(thisconfig): if thisconfig['java_paths'] is None: thisconfig['java_paths'] = dict() pathlist = [] - pathlist += glob.glob('/usr/lib/jvm/j*[126-9]*') - pathlist += glob.glob('/usr/java/jdk1.[126-9]*') - pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[126-9][0-9]?.0.jdk') + pathlist += glob.glob('/usr/lib/jvm/j*[16-9]*') + pathlist += glob.glob('/usr/java/jdk1.[16-9]*') + pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[16-9][0-9]?.0.jdk') pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[0-9]*') pathlist += glob.glob('/opt/oracle-jdk-*1.[0-9]*') pathlist += glob.glob('/opt/icedtea-*[0-9]*') if os.getenv('JAVA_HOME') is not None: pathlist.append(os.getenv('JAVA_HOME')) if os.getenv('PROGRAMFILES') is not None: - pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[126-9][0-9]?.*')) + pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[16-9][0-9]?.*')) _add_java_paths_to_config(pathlist, thisconfig) - for java_version in range(29, 6, -1): - java_version = str(java_version) + for java_version in ('14', '13', '12', '11', '10', '9', '8', '7'): if java_version not in thisconfig['java_paths']: continue java_home = thisconfig['java_paths'][java_version] @@ -438,17 +266,6 @@ def fill_config_defaults(thisconfig): if 'keytool' not in thisconfig and shutil.which('keytool'): thisconfig['keytool'] = shutil.which('keytool') - # enable apksigner by default so v2/v3 APK signatures validate - find_apksigner(thisconfig) - if not thisconfig.get('apksigner'): - logging.warning(_('apksigner not found! Cannot sign or verify modern APKs')) - - if 'ipfs_cid' not in thisconfig and shutil.which('ipfs_cid'): - thisconfig['ipfs_cid'] = shutil.which('ipfs_cid') - cmd = sys.argv[1] if len(sys.argv) >= 2 else '' - if cmd == 'update' and not thisconfig.get('ipfs_cid'): - logging.debug(_("ipfs_cid not found, skipping CIDv1 generation")) - for k in ['ndk_paths', 'java_paths']: d = thisconfig[k] for k2 in d.copy(): @@ -458,70 +275,6 @@ def fill_config_defaults(thisconfig): thisconfig[k][k2] = exp thisconfig[k][k2 + '_orig'] = v - ndk_paths = thisconfig.get('ndk_paths', {}) - - ndk_bundle = os.path.join(thisconfig['sdk_path'], 'ndk-bundle') - if os.path.exists(ndk_bundle): - version = get_ndk_version(ndk_bundle) - if version not in ndk_paths: - ndk_paths[version] = ndk_bundle - - ndk_dir = os.path.join(thisconfig['sdk_path'], 'ndk') - if os.path.exists(ndk_dir): - for ndk in glob.glob(os.path.join(ndk_dir, '*')): - version = get_ndk_version(ndk) - if version not in ndk_paths: - ndk_paths[version] = ndk - - if 'cachedir_scanner' not in thisconfig: - thisconfig['cachedir_scanner'] = str(Path(thisconfig['cachedir']) / 'scanner') - if 'gradle_version_dir' not in thisconfig: - thisconfig['gradle_version_dir'] = str(Path(thisconfig['cachedir']) / 'gradle') - - -def get_config(): - """Get the initalized, singleton config instance. - - config and options are intertwined in read_config(), so they have - to be here too. In the current ugly state of things, there are - multiple potential instances of config and options in use: - - * global - * module-level in the subcommand module (e.g. fdroidserver/build.py) - * module-level in fdroidserver.common - - There are some insane parts of the code that are probably - referring to multiple instances of these at different points. - This can be super confusing and maddening. - - The current intermediate refactoring step is to move all - subcommands to always get/set config and options via this function - so that there is no longer a distinction between the global and - module-level instances. Then there can be only one module-level - instance in fdroidserver.common. - - """ - global config - - if config is not None: - return config - - read_config() - - # make sure these values are available in common.py even if they didn't - # declare global in a scope - common.config = config - - return config - - -def get_cachedir(): - cachedir = config and config.get('cachedir') - if cachedir and os.path.exists(cachedir): - return Path(cachedir) - else: - return Path(tempfile.mkdtemp()) - def regsub_file(pattern, repl, path): with open(path, 'rb') as f: @@ -531,126 +284,58 @@ def regsub_file(pattern, repl, path): f.write(text) -def config_type_check(path, data): - if Path(path).name == 'mirrors.yml': - expected_type = list - else: - expected_type = dict - if expected_type == dict: - if not isinstance(data, dict): - msg = _('{path} is not "key: value" dict, but a {datatype}!') - raise TypeError(msg.format(path=path, datatype=type(data).__name__)) - elif not isinstance(data, expected_type): - msg = _('{path} is not {expected_type}, but a {datatype}!') - raise TypeError( - msg.format( - path=path, - expected_type=expected_type.__name__, - datatype=type(data).__name__, - ) - ) +def read_config(opts): + """Read the repository config - -class _Config(dict): - def __init__(self, default={}): - super(_Config, self).__init__(default) - self.loaded = {} - - def lazyget(self, key): - if key not in self.loaded: - value = super(_Config, self).__getitem__(key) - - if key == 'serverwebroot': - roots = parse_list_of_dicts(value) - rootlist = [] - for d in roots: - # since this is used with rsync, where trailing slashes have - # meaning, ensure there is always a trailing slash - rootstr = d.get('url') - if not rootstr: - logging.error('serverwebroot: has blank value!') - continue - if rootstr[-1] != '/': - rootstr += '/' - d['url'] = rootstr.replace('//', '/') - rootlist.append(d) - self.loaded[key] = rootlist - - elif key == 'servergitmirrors': - self.loaded[key] = parse_list_of_dicts(value) - - elif isinstance(value, dict) and 'env' in value and len(value) == 1: - var = value['env'] - if var in os.environ: - self.loaded[key] = os.getenv(var) - else: - logging.error( - _( - 'Environment variable {var} from {configname} is not set!' - ).format(var=value['env'], configname=key) - ) - self.loaded[key] = None - else: - self.loaded[key] = value - - return self.loaded[key] - - def __getitem__(self, key): - return self.lazyget(key) - - def get(self, key, default=None, /): - try: - return self.lazyget(key) - except KeyError: - return default - - -def read_config(): - """Read the repository config. - - The config is read from config.yml, which is in the current + The config is read from config_file, which is in the current directory when any of the repo management commands are used. If there is a local metadata file in the git repo, then the config is not required, just use defaults. config.yml is the preferred form because no code is executed when - reading it. config.py is deprecated and no longer supported. - - config.yml requires ASCII or UTF-8 encoding because this code does - not auto-detect the file's encoding. That is left up to the YAML - library. YAML allows ASCII, UTF-8, UTF-16, and UTF-32 encodings. - Since it is a good idea to manage config.yml (WITHOUT PASSWORDS!) - in git, it makes sense to use a globally standard encoding. + reading it. config.py is deprecated and supported for backwards + compatibility. """ - global config + global config, options if config is not None: return config + options = opts + config = {} - - if os.path.exists(CONFIG_FILE): - logging.debug(_("Reading '{config_file}'").format(config_file=CONFIG_FILE)) - with open(CONFIG_FILE, encoding='utf-8') as fp: - config = yaml.load(fp) - if not config: - config = {} - config_type_check(CONFIG_FILE, config) - + config_file = 'config.yml' old_config_file = 'config.py' - if os.path.exists(old_config_file): - logging.warning( - _("""Ignoring deprecated {oldfile}, use {newfile}!""").format( - oldfile=old_config_file, newfile=CONFIG_FILE - ) - ) + + if os.path.exists(config_file) and os.path.exists(old_config_file): + logging.error(_("""Conflicting config files! Using {newfile}, ignoring {oldfile}!""") + .format(oldfile=old_config_file, newfile=config_file)) + + if os.path.exists(config_file): + logging.debug(_("Reading '{config_file}'").format(config_file=config_file)) + with open(config_file) as fp: + config = yaml.safe_load(fp) + elif os.path.exists(old_config_file): + logging.warning(_("""{oldfile} is deprecated, use {newfile}""") + .format(oldfile=old_config_file, newfile=config_file)) + with io.open(old_config_file, "rb") as fp: + code = compile(fp.read(), old_config_file, 'exec') + exec(code, None, config) # nosec TODO automatically migrate + else: + logging.warning(_("No config.yml found, using defaults.")) + + for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'): + if k in config: + if not type(config[k]) in (str, list, tuple): + logging.warning( + _("'{field}' will be in random order! Use () or [] brackets if order is important!") + .format(field=k)) # smartcardoptions must be a list since its command line args for Popen smartcardoptions = config.get('smartcardoptions') if isinstance(smartcardoptions, str): - sco_items = re.sub(r'\s+', r' ', config['smartcardoptions']).split(' ') - config['smartcardoptions'] = [i.strip() for i in sco_items if i] + config['smartcardoptions'] = re.sub(r'\s+', r' ', config['smartcardoptions']).split(' ') elif not smartcardoptions and 'keystore' in config and config['keystore'] == 'NONE': # keystore='NONE' means use smartcard, these are required defaults config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName', @@ -658,20 +343,46 @@ def read_config(): 'sun.security.pkcs11.SunPKCS11', '-providerArg', 'opensc-fdroid.cfg'] + if any(k in config for k in ["keystore", "keystorepass", "keypass"]): + if os.path.exists(config_file): + f = config_file + elif os.path.exists(old_config_file): + f = old_config_file + st = os.stat(f) + if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO: + logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!") + .format(config_file=f)) + fill_config_defaults(config) + if 'serverwebroot' in config: + if isinstance(config['serverwebroot'], str): + roots = [config['serverwebroot']] + elif all(isinstance(item, str) for item in config['serverwebroot']): + roots = config['serverwebroot'] + else: + raise TypeError(_('only accepts strings, lists, and tuples')) + rootlist = [] + for rootstr in roots: + # since this is used with rsync, where trailing slashes have + # meaning, ensure there is always a trailing slash + if rootstr[-1] != '/': + rootstr += '/' + rootlist.append(rootstr.replace('//', '/')) + config['serverwebroot'] = rootlist + if 'servergitmirrors' in config: + if isinstance(config['servergitmirrors'], str): + roots = [config['servergitmirrors']] + elif all(isinstance(item, str) for item in config['servergitmirrors']): + roots = config['servergitmirrors'] + else: + raise TypeError(_('only accepts strings, lists, and tuples')) + config['servergitmirrors'] = roots + limit = config['git_mirror_size_limit'] config['git_mirror_size_limit'] = parse_human_readable_size(limit) - if 'repo_url' in config: - if not config['repo_url'].endswith('/repo'): - raise FDroidException(_('repo_url needs to end with /repo')) - - if 'archive_url' in config: - if not config['archive_url'].endswith('/archive'): - raise FDroidException(_('archive_url needs to end with /archive')) - confignames_to_delete = set() for configname, dictvalue in config.items(): if configname == 'java_paths': @@ -683,163 +394,25 @@ def read_config(): continue elif isinstance(dictvalue, dict): for k, v in dictvalue.items(): - if k != 'env': + if k == 'env': + env = os.getenv(v) + if env: + config[configname] = env + else: + confignames_to_delete.add(configname) + logging.error(_('Environment variable {var} from {configname} is not set!') + .format(var=k, configname=configname)) + else: confignames_to_delete.add(configname) logging.error(_('Unknown entry {key} in {configname}') .format(key=k, configname=configname)) for configname in confignames_to_delete: - del config[configname] + del(config[configname]) - if any(k in config and config.get(k) for k in ["keystorepass", "keypass"]): - st = os.stat(CONFIG_FILE) - if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO: - logging.warning( - _("unsafe permissions on '{config_file}' (should be 0600)!").format( - config_file=CONFIG_FILE - ) - ) - - config = _Config(config) return config -def expand_env_dict(s): - """Expand env var dict to a string value. - - {env: varName} syntax can be used to replace any string value in the - config with the value of an environment variable "varName". This - allows for secrets management when commiting the config file to a - public git repo. - - """ - if not s or type(s) not in (str, dict): - return - if isinstance(s, dict): - if 'env' not in s or len(s) > 1: - raise TypeError(_('Only accepts a single key "env"')) - var = s['env'] - s = os.getenv(var) - if not s: - logging.error( - _('Environment variable {{env: {var}}} is not set!').format(var=var) - ) - return - return os.path.expanduser(s) - - -def parse_list_of_dicts(l_of_d): - """Parse config data structure that is a list of dicts of strings. - - The value can be specified as a string, list of strings, or list of dictionary maps - where the values are strings. - - """ - if isinstance(l_of_d, str): - return [{"url": expand_env_dict(l_of_d)}] - if isinstance(l_of_d, dict): - return [{"url": expand_env_dict(l_of_d)}] - if all(isinstance(item, str) for item in l_of_d): - return [{'url': expand_env_dict(i)} for i in l_of_d] - if all(isinstance(item, dict) for item in l_of_d): - for item in l_of_d: - item['url'] = expand_env_dict(item['url']) - return l_of_d - raise TypeError(_('only accepts strings, lists, and tuples')) - - -def get_mirrors(url, filename=None): - """Get list of dict entries for mirrors, appending filename if provided.""" - # TODO use cached index if it exists - if isinstance(url, str): - url = urlsplit(url) - - if url.netloc == 'f-droid.org': - mirrors = FDROIDORG_MIRRORS - else: - mirrors = parse_list_of_dicts(url.geturl()) - - if filename: - return append_filename_to_mirrors(filename, mirrors) - else: - return mirrors - - -def append_filename_to_mirrors(filename, mirrors): - """Append the filename to all "url" entries in the mirrors dict.""" - appended = copy.deepcopy(mirrors) - for mirror in appended: - parsed = urlparse(mirror['url']) - mirror['url'] = urlunparse( - parsed._replace(path=os.path.join(parsed.path, filename)) - ) - return appended - - -def file_entry(filename, hash_value=None): - meta = {} - meta["name"] = "/" + Path(filename).as_posix().split("/", 1)[1] - meta["sha256"] = hash_value or sha256sum(filename) - meta["size"] = os.stat(filename).st_size - return meta - - -def load_localized_config(name, repodir): - """Load localized config files and put them into internal dict format. - - This will maintain the order as came from the data files, e.g - YAML. The locale comes from unsorted paths on the filesystem, so - that is separately sorted. - - """ - ret = dict() - found_config_file = False - for f in Path().glob("config/**/{name}.yml".format(name=name)): - found_config_file = True - locale = f.parts[1] - if len(f.parts) == 2: - locale = DEFAULT_LOCALE - with open(f, encoding="utf-8") as fp: - elem = yaml.load(fp) - if not isinstance(elem, dict): - msg = _('{path} is not "key: value" dict, but a {datatype}!') - raise TypeError(msg.format(path=f, datatype=type(elem).__name__)) - for afname, field_dict in elem.items(): - if afname not in ret: - ret[afname] = dict() - for key, value in field_dict.items(): - if key not in ret[afname]: - ret[afname][key] = dict() - if key == "icon": - icons_dir = os.path.join(repodir, 'icons') - if not os.path.exists(icons_dir): - os.makedirs(icons_dir, exist_ok=True) - src = os.path.join("config", value) - dest = os.path.join(icons_dir, os.path.basename(src)) - if not os.path.exists(dest) or not filecmp.cmp(src, dest): - shutil.copy2(src, dest) - ret[afname][key][locale] = file_entry( - os.path.join(icons_dir, value) - ) - else: - ret[afname][key][locale] = value - - if not found_config_file: - for f in Path().glob("config/*.yml"): - if f.stem not in CONFIG_NAMES: - msg = _('{path} is not a standard config file!').format(path=f) - m = difflib.get_close_matches(f.stem, CONFIG_NAMES, 1) - if m: - msg += ' ' - msg += _('Did you mean config/{name}.yml?').format(name=m[0]) - logging.error(msg) - - for elem in ret.values(): - for afname in elem: - elem[afname] = {locale: v for locale, v in sorted(elem[afname].items())} - return ret - - def parse_human_readable_size(size): units = { 'b': 1, @@ -848,26 +421,20 @@ def parse_human_readable_size(size): } try: return int(float(size)) - except (ValueError, TypeError) as exc: + except (ValueError, TypeError): if type(size) != str: raise ValueError(_('Could not parse size "{size}", wrong type "{type}"') - .format(size=size, type=type(size))) from exc + .format(size=size, type=type(size))) s = size.lower().replace(' ', '') - m = re.match(r'^(?P[0-9][0-9.]*) *(?P' + r'|'.join(units.keys()) + r')$', s) + m = re.match(r'^(?P[0-9][0-9.]+) *(?P' + r'|'.join(units.keys()) + r')$', s) if not m: - raise ValueError(_('Not a valid size definition: "{}"').format(size)) from exc + raise ValueError(_('Not a valid size definition: "{}"').format(size)) return int(float(m.group("value")) * units[m.group("unit")]) -def get_dir_size(path_or_str): - """Get the total size of all files in the given directory.""" - if isinstance(path_or_str, str): - path_or_str = Path(path_or_str) - return sum(f.stat().st_size for f in path_or_str.glob('**/*') if f.is_file()) - - def assert_config_keystore(config): """Check weather keystore is configured correctly and raise exception if not.""" + nosigningkey = False if 'repo_keyalias' not in config: nosigningkey = True @@ -893,60 +460,39 @@ def assert_config_keystore(config): + "you can create one using: fdroid update --create-key") -def find_apksigner(config): - """Search for the best version apksigner and adds it to the config. - +def find_apksigner(): + """ Returns the best version of apksigner following this algorithm: - * use config['apksigner'] if set * try to find apksigner in path * find apksigner in build-tools starting from newest installed going down to MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION - - Returns - ------- - str - path to apksigner or None if no version is found - + :return: path to apksigner or None if no version is found """ - command = 'apksigner' - if command in config: - return - - tmp = find_command(command) - if tmp is not None: - config[command] = tmp - return - - build_tools_path = os.path.join(config.get('sdk_path', ''), 'build-tools') + if set_command_in_config('apksigner'): + return config['apksigner'] + build_tools_path = os.path.join(config['sdk_path'], 'build-tools') if not os.path.isdir(build_tools_path): - return + return None for f in sorted(os.listdir(build_tools_path), reverse=True): if not os.path.isdir(os.path.join(build_tools_path, f)): continue try: - version = LooseVersion(f) - if version >= LooseVersion('33') and version < LooseVersion('34'): - logging.warning( - _('apksigner in build-tools;{version} passes APKs with invalid v3 signatures, ignoring.').format( - version=version - ) - ) - continue - if version < LooseVersion(MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION): - logging.debug("Local Android SDK only has outdated apksigner versions") - return + if LooseVersion(f) < LooseVersion(MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION): + return None except TypeError: continue if os.path.exists(os.path.join(build_tools_path, f, 'apksigner')): apksigner = os.path.join(build_tools_path, f, 'apksigner') logging.info("Using %s " % apksigner) + # memoize result config['apksigner'] = apksigner - return + return config['apksigner'] def find_sdk_tools_cmd(cmd): - """Find a working path to a tool from the Android SDK.""" + '''find a working path to a tool from the Android SDK''' + tooldirs = [] if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']): # try to find a working path to this command, in all the recent possible paths @@ -962,26 +508,20 @@ def find_sdk_tools_cmd(cmd): sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools') if os.path.exists(sdk_platform_tools): tooldirs.append(sdk_platform_tools) - sdk_build_tools = glob.glob(os.path.join(config['sdk_path'], 'build-tools', '*.*')) - if sdk_build_tools: - tooldirs.append(sorted(sdk_build_tools)[-1]) # use most recent version - if os.path.exists('/usr/bin'): - tooldirs.append('/usr/bin') + tooldirs.append('/usr/bin') for d in tooldirs: path = os.path.join(d, cmd) - if not os.path.isfile(path): - path += '.exe' if os.path.isfile(path): if cmd == 'aapt': test_aapt_version(path) return path # did not find the command, exit with error message test_sdk_exists(config) # ignore result so None is never returned - raise FDroidException(_("Android SDK tool {cmd} not found!").format(cmd=cmd)) + raise FDroidException(_("Android SDK tool {cmd} found!").format(cmd=cmd)) def test_aapt_version(aapt): - """Check whether the version of aapt is new enough.""" + '''Check whether the version of aapt is new enough''' output = subprocess.check_output([aapt, 'version'], universal_newlines=True) if output is None or output == '': logging.error(_("'{path}' failed to execute!").format(path=aapt)) @@ -1007,16 +547,13 @@ def test_aapt_version(aapt): def test_sdk_exists(thisconfig): if 'sdk_path' not in thisconfig: - # check the 'apksigner' value in the config to see if its new enough - f = thisconfig.get('apksigner', '') - if os.path.isfile(f): - sdk_path = os.path.dirname(os.path.dirname(os.path.dirname(f))) - tmpconfig = {'sdk_path': sdk_path} - find_apksigner(tmpconfig) - if os.path.exists(tmpconfig.get('apksigner', '')): - return True - logging.error(_("'sdk_path' not set in config.yml!")) - return False + # TODO convert this to apksigner once it is required + if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']): + test_aapt_version(thisconfig['aapt']) + return True + else: + logging.error(_("'sdk_path' not set in config.yml!")) + return False if thisconfig['sdk_path'] == default_config['sdk_path']: logging.error(_('No Android SDK found!')) logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:')) @@ -1030,50 +567,36 @@ def test_sdk_exists(thisconfig): logging.critical(_("Android SDK path '{path}' is not a directory!") .format(path=thisconfig['sdk_path'])) return False - find_apksigner(thisconfig) - if not os.path.exists(thisconfig.get('apksigner', '')): - return False return True def get_local_metadata_files(): - """Get any metadata files local to an app's source repo. + '''get any metadata files local to an app's source repo This tries to ignore anything that does not count as app metdata, including emacs cruft ending in ~ - """ + ''' return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]') -def read_pkg_args(appid_versionCode_pairs, allow_version_codes=False): - """No summary. - - Parameters - ---------- - appids - arguments in the form of multiple appid:[versionCode] strings - - Returns - ------- - a dictionary with the set of vercodes specified for each package +def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False): + """ + :param appids: arguments in the form of multiple appid:[vc] strings + :returns: a dictionary with the set of vercodes specified for each package """ vercodes = {} if not appid_versionCode_pairs: return vercodes - error = False - apk_regex = re.compile(r'_(\d+)\.apk$') for p in appid_versionCode_pairs: - # Convert the apk name to a appid:versioncode pair - p = apk_regex.sub(r':\1', p) - if allow_version_codes and ':' in p: + if allow_vercodes and ':' in p: package, vercode = p.split(':') try: - vercode = version_code_string_to_int(vercode) - except ValueError as e: - logging.error('"%s": %s' % (p, str(e))) - error = True + i_vercode = int(vercode, 0) + except ValueError: + i_vercode = int(vercode) + vercode = str(i_vercode) else: package, vercode = p, None if package not in vercodes: @@ -1082,52 +605,20 @@ def read_pkg_args(appid_versionCode_pairs, allow_version_codes=False): elif vercode and vercode not in vercodes[package]: vercodes[package] += [vercode] if vercode else [] - if error: - raise FDroidException(_("Found invalid versionCodes for some apps")) - return vercodes -def get_metadata_files(vercodes): - """ - Build a list of metadata files and raise an exception for invalid appids. - - Parameters - ---------- - vercodes - versionCodes as returned by read_pkg_args() - - Returns - ------- - List - a list of corresponding metadata/*.yml files - """ - found_invalid = False - metadatafiles = [] - for appid in vercodes.keys(): - f = Path('metadata') / ('%s.yml' % appid) - if f.exists(): - metadatafiles.append(f) - else: - found_invalid = True - logging.critical(_("No such package: %s") % appid) - if found_invalid: - raise FDroidException(_("Found invalid appids in arguments")) - return metadatafiles - - -def read_app_args(appid_versionCode_pairs, allow_version_codes=False, sort_by_time=False): - """Build a list of App instances for processing. +def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False): + """Build a list of App instances for processing On top of what read_pkg_args does, this returns the whole app metadata, but limiting the builds list to the builds matching the - appid_versionCode_pairs and vercodes specified. If no - appid_versionCode_pairs are specified, then all App and Build instances are - returned. + appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then + all App and Build instances are returned. """ - vercodes = read_pkg_args(appid_versionCode_pairs, allow_version_codes) - allapps = fdroidserver.metadata.read_metadata(vercodes, sort_by_time) + + vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes) if not vercodes: return allapps @@ -1137,6 +628,11 @@ def read_app_args(appid_versionCode_pairs, allow_version_codes=False, sort_by_ti if appid in vercodes: apps[appid] = app + if len(apps) != len(vercodes): + for p in vercodes: + if p not in allapps: + logging.critical(_("No such package: %s") % p) + raise FDroidException(_("Found invalid appids in arguments")) if not apps: raise FDroidException(_("No packages specified")) @@ -1145,10 +641,10 @@ def read_app_args(appid_versionCode_pairs, allow_version_codes=False, sort_by_ti vc = vercodes[appid] if not vc: continue - app['Builds'] = [b for b in app.get('Builds', []) if b.versionCode in vc] - if len(app.get('Builds', [])) != len(vercodes[appid]): + app.builds = [b for b in app.builds if b.versionCode in vc] + if len(app.builds) != len(vercodes[appid]): error = True - allvcs = [b.versionCode for b in app.get('Builds', [])] + allvcs = [b.versionCode for b in app.builds] for v in vercodes[appid]: if v not in allvcs: logging.critical(_("No such versionCode {versionCode} for app {appid}") @@ -1161,7 +657,7 @@ def read_app_args(appid_versionCode_pairs, allow_version_codes=False, sort_by_ti def get_extension(filename): - """Get name and extension of filename, with extension always lower case.""" + """get name and extension of filename, with extension always lower case""" base, ext = os.path.splitext(filename) if not ext: return base, '' @@ -1175,9 +671,9 @@ def publishednameinfo(filename): filename = os.path.basename(filename) m = publish_name_regex.match(filename) try: - result = (m.group(1), int(m.group(2))) - except AttributeError as exc: - raise FDroidException(_("Invalid name for published file: %s") % filename) from exc + result = (m.group(1), m.group(2)) + except AttributeError: + raise FDroidException(_("Invalid name for published file: %s") % filename) return result @@ -1186,15 +682,13 @@ apk_release_filename_with_sigfp = re.compile(r'(?P[a-zA-Z0-9_\.]+)_(?P= 2.3 git_config = [ @@ -1570,7 +976,7 @@ class vcs_git(vcs): '-c', 'core.sshCommand=/bin/false', '-c', 'url.https://.insteadOf=ssh://', ] - for domain in ('bitbucket.org', 'github.com', 'gitlab.com', 'codeberg.org'): + for domain in ('bitbucket.org', 'github.com', 'gitlab.com'): git_config.append('-c') git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':') git_config.append('-c') @@ -1587,28 +993,22 @@ class vcs_git(vcs): envs=envs, cwd=cwd, output=output) def checkrepo(self): - """No summary. - - If the local directory exists, but is somehow not a git repository, + """If the local directory exists, but is somehow not a git repository, git will traverse up the directory tree until it finds one that is (i.e. fdroidserver) and then we'll proceed to destroy it! This is called as a safety check. """ - cmd = ['git', 'rev-parse', '--show-toplevel'] - p = FDroidPopen(cmd, cwd=self.local, output=False) + + p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False) result = p.output.rstrip() - if p.returncode > 0: - raise VCSException( - f"`{' '.join(cmd)}` failed, (in '{os.path.abspath(self.local)}') {result}" - ) - if Path(result) != Path(self.local).resolve(): - raise VCSException(f"Repository mismatch ('{self.local}' != '{result}')") + if not result.endswith(self.local): + raise VCSException('Repository mismatch') def gotorevisionx(self, rev): if not os.path.exists(self.local): # Brand new checkout - p = self.git(['clone', '--', self.remote, str(self.local)]) + p = self.git(['clone', '--', self.remote, self.local]) if p.returncode != 0: self.clone_failed = True raise VCSException("Git clone failed", p.output) @@ -1618,23 +1018,17 @@ class vcs_git(vcs): # Discard any working tree changes p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive', 'git', 'reset', '--hard'], cwd=self.local, output=False) - if p.returncode != 0: - logging.debug("Git submodule reset failed (ignored) {output}".format(output=p.output)) - p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False) if p.returncode != 0: raise VCSException(_("Git reset failed"), p.output) # Remove untracked files now, in case they're tracked in the target # revision (it happens!) p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive', 'git', 'clean', '-dffx'], cwd=self.local, output=False) - if p.returncode != 0: - logging.debug("Git submodule cleanup failed (ignored) {output}".format(output=p.output)) - p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False) if p.returncode != 0: raise VCSException(_("Git clean failed"), p.output) if not self.refreshed: # Get latest commits and tags from remote - p = self.git(['fetch', '--prune', '--prune-tags', '--force', 'origin'], cwd=self.local) + p = self.git(['fetch', 'origin'], cwd=self.local) if p.returncode != 0: raise VCSException(_("Git fetch failed"), p.output) p = self.git(['fetch', '--prune', '--tags', '--force', 'origin'], output=False, cwd=self.local) @@ -1687,33 +1081,23 @@ class vcs_git(vcs): if p.returncode != 0: raise VCSException(_("Git submodule update failed"), p.output) - def deinitsubmodules(self): - self.checkrepo() - p = FDroidPopen(['git', 'submodule', 'deinit', '--all', '--force'], cwd=self.local, output=False) - if p.returncode != 0: - raise VCSException(_("Git submodule deinit failed"), p.output) - def _gettags(self): self.checkrepo() p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False) return p.output.splitlines() - def latesttags(self): - """Return a list of latest tags.""" - self.checkrepo() - return [tag.name for tag in sorted( - git.Repo(self.local).tags, - key=lambda t: t.commit.committed_date, - reverse=True - )] + tag_format = re.compile(r'tag: ([^),]*)') - def getref(self, revname='HEAD'): + def latesttags(self): self.checkrepo() - repo = git.Repo(self.local) - try: - return repo.commit(revname).hexsha - except git.BadName: - return None + p = FDroidPopen(['git', 'log', '--tags', + '--simplify-by-decoration', '--pretty=format:%d'], + cwd=self.local, output=False) + tags = [] + for line in p.output.splitlines(): + for tag in self.tag_format.findall(line): + tags.append(tag) + return tags class vcs_gitsvn(vcs): @@ -1725,9 +1109,7 @@ class vcs_gitsvn(vcs): return ['git', 'svn', '--version'] def checkrepo(self): - """No summary. - - If the local directory exists, but is somehow not a git repository, + """If the local directory exists, but is somehow not a git repository, git will traverse up the directory tree until it finds one that is (i.e. fdroidserver) and then we'll proceed to destory it! This is called as a safety check. @@ -1735,11 +1117,11 @@ class vcs_gitsvn(vcs): """ p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False) result = p.output.rstrip() - if Path(result) != Path(self.local).resolve(): + if not result.endswith(self.local): raise VCSException('Repository mismatch') def git(self, args, envs=dict(), cwd=None, output=True): - """Prevent git fetch/clone/submodule from hanging at the username/password prompt. + '''Prevent git fetch/clone/submodule from hanging at the username/password prompt AskPass is set to /bin/true to let the process try to connect without a username/password. @@ -1748,7 +1130,7 @@ class vcs_gitsvn(vcs): (supported in git >= 2.3). This protects against CVE-2017-1000117. - """ + ''' git_config = [ '-c', 'core.askpass=/bin/true', '-c', 'core.sshCommand=/bin/false', @@ -1787,16 +1169,16 @@ class vcs_gitsvn(vcs): # git-svn sucks at certificate validation, this throws useful errors: try: import requests - r = requests.head(remote, timeout=300) + r = requests.head(remote) r.raise_for_status() except Exception as e: - raise VCSException('SVN certificate pre-validation failed: ' + str(e)) from e + raise VCSException('SVN certificate pre-validation failed: ' + str(e)) location = r.headers.get('location') if location and not location.startswith('https://'): raise VCSException(_('Invalid redirect to non-HTTPS: {before} -> {after} ') .format(before=remote, after=location)) - gitsvn_args.extend(['--', remote, str(self.local)]) + gitsvn_args.extend(['--', remote, self.local]) p = self.git(gitsvn_args) if p.returncode != 0: self.clone_failed = True @@ -1878,9 +1260,9 @@ class vcs_gitsvn(vcs): if os.path.isdir(d): return os.listdir(d) - def getref(self, revname='HEAD'): + def getref(self): self.checkrepo() - p = FDroidPopen(['git', 'svn', 'find-rev', revname], cwd=self.local, output=False) + p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False) if p.returncode != 0: return None return p.output.strip() @@ -1896,17 +1278,17 @@ class vcs_hg(vcs): def gotorevisionx(self, rev): if not os.path.exists(self.local): - p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, str(self.local)], + p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, self.local], output=False) if p.returncode != 0: self.clone_failed = True raise VCSException("Hg clone failed", p.output) else: - p = FDroidPopen(['hg', 'status', '-uiS'], cwd=self.local, output=False) + p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False) if p.returncode != 0: raise VCSException("Hg status failed", p.output) for line in p.output.splitlines(): - if not line.startswith('? ') and not line.startswith('I '): + if not line.startswith('? '): raise VCSException("Unexpected output from hg status -uS: " + line) FDroidPopen(['rm', '-rf', '--', line[2:]], cwd=self.local, output=False) if not self.refreshed: @@ -1921,6 +1303,16 @@ class vcs_hg(vcs): p = FDroidPopen(['hg', 'update', '-C', '--', rev], cwd=self.local, output=False) if p.returncode != 0: raise VCSException("Hg checkout of '%s' failed" % rev, p.output) + p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False) + # Also delete untracked files, we have to enable purge extension for that: + if "'purge' is provided by the following extension" in p.output: + with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile: + myfile.write("\n[extensions]\nhgext.purge=\n") + p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False) + if p.returncode != 0: + raise VCSException("HG purge failed", p.output) + elif p.returncode != 0: + raise VCSException("HG purge failed", p.output) def _gettags(self): p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False) @@ -1936,7 +1328,7 @@ class vcs_bzr(vcs): return ['bzr', '--version'] def bzr(self, args, envs=dict(), cwd=None, output=True): - """Prevent bzr from ever using SSH to avoid security vulns.""" + '''Prevent bzr from ever using SSH to avoid security vulns''' envs.update({ 'BZR_SSH': 'false', }) @@ -1944,7 +1336,7 @@ class vcs_bzr(vcs): def gotorevisionx(self, rev): if not os.path.exists(self.local): - p = self.bzr(['branch', self.remote, str(self.local)], output=False) + p = self.bzr(['branch', self.remote, self.local], output=False) if p.returncode != 0: self.clone_failed = True raise VCSException("Bzr branch failed", p.output) @@ -1980,11 +1372,7 @@ def unescape_string(string): def retrieve_string(app_dir, string, xmlfiles=None): - if string.startswith('@string/'): - name = string[len('@string/'):] - elif string.startswith('${'): - return '' # Gradle variable - else: + if not string.startswith('@string/'): return unescape_string(string) if xmlfiles is None: @@ -1997,20 +1385,18 @@ def retrieve_string(app_dir, string, xmlfiles=None): if os.path.basename(root) == 'values': xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')] + name = string[len('@string/'):] + def element_content(element): if element.text is None: return "" s = XMLElementTree.tostring(element, encoding='utf-8', method='text') return s.decode('utf-8').strip() - for path in sorted(xmlfiles): + for path in xmlfiles: if not os.path.isfile(path): continue - try: - xml = parse_xml(path) - except (XMLElementTree.ParseError, ValueError): - logging.warning(_("Problem with xml at '{path}'").format(path=path)) - continue + xml = parse_xml(path) element = xml.find('string[@name="' + name + '"]') if element is not None: content = element_content(element) @@ -2023,36 +1409,33 @@ def retrieve_string_singleline(app_dir, string, xmlfiles=None): return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip() -def manifest_paths(app_dir, flavors): - """Return list of existing files that will be used to find the highest vercode.""" - possible_manifests = \ - [Path(app_dir) / 'AndroidManifest.xml', - Path(app_dir) / 'src/main/AndroidManifest.xml', - Path(app_dir) / 'src/AndroidManifest.xml', - Path(app_dir) / 'build.gradle', - Path(app_dir) / 'build-extras.gradle', - Path(app_dir) / 'build.gradle.kts'] +def manifest_paths(app_dir, flavours): + '''Return list of existing files that will be used to find the highest vercode''' - for flavor in flavors: - if flavor == 'yes': + possible_manifests = \ + [os.path.join(app_dir, 'AndroidManifest.xml'), + os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'), + os.path.join(app_dir, 'src', 'AndroidManifest.xml'), + os.path.join(app_dir, 'build.gradle'), + os.path.join(app_dir, 'build-extras.gradle'), + os.path.join(app_dir, 'build.gradle.kts')] + + for flavour in flavours: + if flavour == 'yes': continue possible_manifests.append( - Path(app_dir) / 'src' / flavor / 'AndroidManifest.xml') + os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml')) - return [path for path in possible_manifests if path.is_file()] + return [path for path in possible_manifests if os.path.isfile(path)] -def fetch_real_name(app_dir, flavors): - """Retrieve the package name. Returns the name, or None if not found.""" - for path in manifest_paths(app_dir, flavors): - if not path.suffix == '.xml' or not path.is_file(): - continue - logging.debug("fetch_real_name: Checking manifest at %s" % path) - try: - xml = parse_xml(path) - except (XMLElementTree.ParseError, ValueError): - logging.warning(_("Problem with xml at '{path}'").format(path=path)) +def fetch_real_name(app_dir, flavours): + '''Retrieve the package name. Returns the name, or None if not found.''' + for path in manifest_paths(app_dir, flavours): + if not path.endswith('.xml') or not os.path.isfile(path): continue + logging.debug("fetch_real_name: Checking manifest at " + path) + xml = parse_xml(path) app = xml.find('application') if app is None: continue @@ -2105,10 +1488,10 @@ def remove_debuggable_flags(root_dir): os.path.join(root, 'AndroidManifest.xml')) -vcsearch_g = re.compile(r'''\b[Vv]ersionCode\s*=?\s*["'(]*([0-9][0-9_]*)["')]*''').search -vnsearch_g = re.compile(r'''\b[Vv]ersionName\s*=?\s*\(?(["'])((?:(?=(\\?))\3.)*?)\1''').search +vcsearch_g = re.compile(r'''\b[Vv]ersionCode\s*=?\s*["']*([0-9_]+)["']*''').search +vnsearch_g = re.compile(r'''\b[Vv]ersionName\s*=?\s*(["'])((?:(?=(\\?))\3.)*?)\1''').search vnssearch_g = re.compile(r'''\b[Vv]ersionNameSuffix\s*=?\s*(["'])((?:(?=(\\?))\3.)*?)\1''').search -psearch_g = re.compile(r'''\b(packageName|applicationId|namespace)\s*=*\s*["']([^"']+)["']''').search +psearch_g = re.compile(r'''\b(packageName|applicationId)\s*=*\s*["']([^"']+)["']''').search fsearch_g = re.compile(r'''\b(applicationIdSuffix)\s*=*\s*["']([^"']+)["']''').search @@ -2122,15 +1505,12 @@ def app_matches_packagename(app, package): def parse_androidmanifests(paths, app): - """Extract some information from the AndroidManifest.xml at the given path. - + """ + Extract some information from the AndroidManifest.xml at the given path. Returns (version, vercode, package), any or all of which might be None. All values returned are strings. - - Android Studio recommends "you use UTF-8 encoding whenever possible", so - this code assumes the files use UTF-8. - https://sites.google.com/a/android.com/tools/knownissues/encoding """ + ignoreversions = app.UpdateCheckIgnore ignoresearch = re.compile(ignoreversions).search if ignoreversions else None @@ -2141,20 +1521,9 @@ def parse_androidmanifests(paths, app): max_vercode = None max_package = None - def vnsearch(line): - matches = vnsearch_g(line) - if matches and not any( - matches.group(2).startswith(s) - for s in [ - '${', # Gradle variable names - '@string/', # Strings we could not resolve - ] - ): - return matches.group(2) - return None - for path in paths: - if not path.is_file(): + + if not os.path.isfile(path): continue logging.debug(_("Parsing manifest at '{path}'").format(path=path)) @@ -2162,17 +1531,17 @@ def parse_androidmanifests(paths, app): vercode = None package = None - flavors = None + flavour = None temp_app_id = None temp_version_name = None - if len(app.get('Builds', [])) > 0 and 'gradle' in app['Builds'][-1] and app['Builds'][-1].gradle: - flavors = app['Builds'][-1].gradle + if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle: + flavour = app.builds[-1].gradle[-1] - if path.suffix == '.gradle' or path.name.endswith('.gradle.kts'): - with open(path, 'r', encoding='utf-8') as f: + if path.endswith('.gradle') or path.endswith('.gradle.kts'): + with open(path, 'r') as f: android_plugin_file = False - inside_flavor_group = 0 - inside_required_flavor = 0 + inside_flavour_group = 0 + inside_required_flavour = 0 for line in f: if gradle_comment.match(line): continue @@ -2183,12 +1552,12 @@ def parse_androidmanifests(paths, app): temp_app_id = matches.group(2) if "versionName" in line and not temp_version_name: - matches = vnsearch(line) + matches = vnsearch_g(line) if matches: - temp_version_name = matches + temp_version_name = matches.group(2) - if inside_flavor_group > 0: - if inside_required_flavor > 1: + if inside_flavour_group > 0: + if inside_required_flavour > 0: matches = psearch_g(line) if matches: s = matches.group(2) @@ -2203,12 +1572,11 @@ def parse_androidmanifests(paths, app): if app_matches_packagename(app, temp_app_id): package = temp_app_id - matches = vnsearch(line) + matches = vnsearch_g(line) if matches: - version = matches - + version = matches.group(2) else: - # If build.gradle contains applicationNameSuffix add it to the end of versionName + # If build.gradle contains applicationNameSuffix add it to the end of version name matches = vnssearch_g(line) if matches and temp_version_name: name_suffix = matches.group(2) @@ -2216,31 +1584,23 @@ def parse_androidmanifests(paths, app): matches = vcsearch_g(line) if matches: - vercode = version_code_string_to_int(matches.group(1)) + vercode = matches.group(1) - if inside_required_flavor > 0: if '{' in line: - inside_required_flavor += 1 + inside_required_flavour += 1 if '}' in line: - inside_required_flavor -= 1 - if inside_required_flavor == 1: - inside_required_flavor -= 1 - elif flavors: - for flavor in flavors: - if re.match(r'.*[\'"\s]{flavor}[\'"\s].*\{{.*'.format(flavor=flavor), line): - inside_required_flavor = 2 - break - if re.match(r'.*[\'"\s]{flavor}[\'"\s].*'.format(flavor=flavor), line): - inside_required_flavor = 1 - break + inside_required_flavour -= 1 + else: + if flavour and (flavour in line): + inside_required_flavour = 1 if '{' in line: - inside_flavor_group += 1 + inside_flavour_group += 1 if '}' in line: - inside_flavor_group -= 1 + inside_flavour_group -= 1 else: if "productFlavors" in line: - inside_flavor_group = 1 + inside_flavour_group = 1 if not package: matches = psearch_g(line) if matches: @@ -2248,13 +1608,13 @@ def parse_androidmanifests(paths, app): if app_matches_packagename(app, s): package = s if not version: - matches = vnsearch(line) + matches = vnsearch_g(line) if matches: - version = matches + version = matches.group(2) if not vercode: matches = vcsearch_g(line) if matches: - vercode = version_code_string_to_int(matches.group(1)) + vercode = matches.group(1) if not android_plugin_file and ANDROID_PLUGIN_REGEX.match(line): android_plugin_file = True if android_plugin_file: @@ -2269,20 +1629,20 @@ def parse_androidmanifests(paths, app): else: try: xml = parse_xml(path) - except (XMLElementTree.ParseError, ValueError): + if "package" in xml.attrib: + s = xml.attrib["package"] + if app_matches_packagename(app, s): + package = s + if XMLNS_ANDROID + "versionName" in xml.attrib: + version = xml.attrib[XMLNS_ANDROID + "versionName"] + base_dir = os.path.dirname(path) + version = retrieve_string_singleline(base_dir, version) + if XMLNS_ANDROID + "versionCode" in xml.attrib: + a = xml.attrib[XMLNS_ANDROID + "versionCode"] + if string_is_integer(a): + vercode = a + except Exception: logging.warning(_("Problem with xml at '{path}'").format(path=path)) - continue - if "package" in xml.attrib: - s = xml.attrib["package"] - if app_matches_packagename(app, s): - package = s - if XMLNS_ANDROID + "versionName" in xml.attrib: - version = xml.attrib[XMLNS_ANDROID + "versionName"] - base_dir = os.path.dirname(path) - version = retrieve_string_singleline(base_dir, version) - if XMLNS_ANDROID + "versionCode" in xml.attrib: - vercode = version_code_string_to_int( - xml.attrib[XMLNS_ANDROID + "versionCode"]) # Remember package name, may be defined separately from version+vercode if package is None: @@ -2291,8 +1651,8 @@ def parse_androidmanifests(paths, app): logging.debug("..got package={0}, version={1}, vercode={2}" .format(package, version, vercode)) - # Always grab the package name and versionName in case they are not - # together with the highest versionCode + # Always grab the package name and version name in case they are not + # together with the highest version code if max_package is None and package is not None: max_package = package if max_version is None and version is not None: @@ -2324,7 +1684,7 @@ def parse_androidmanifests(paths, app): def is_valid_package_name(name): - """Check whether name is a valid fdroid package name. + """Check whether name is a valid fdroid package name APKs and manually defined package names must use a valid Java Package Name. Automatically generated package names for non-APK @@ -2336,7 +1696,7 @@ def is_valid_package_name(name): def is_strict_application_id(name): - """Check whether name is a valid Android Application ID. + """Check whether name is a valid Android Application ID The Android ApplicationID is basically a Java Package Name, but with more restrictive naming rules: @@ -2345,8 +1705,6 @@ def is_strict_application_id(name): * Each segment must start with a letter. * All characters must be alphanumeric or an underscore [a-zA-Z0-9_]. - References - ---------- https://developer.android.com/studio/build/application-id """ @@ -2354,6 +1712,156 @@ def is_strict_application_id(name): and '.' in name +def get_all_gradle_and_manifests(build_dir): + paths = [] + for root, dirs, files in os.walk(build_dir): + for f in sorted(files): + if f == 'AndroidManifest.xml' \ + or f.endswith('.gradle') or f.endswith('.gradle.kts'): + full = os.path.join(root, f) + paths.append(full) + return paths + + +def get_gradle_subdir(build_dir, paths): + """get the subdir where the gradle build is based""" + first_gradle_dir = None + for path in paths: + if not first_gradle_dir: + first_gradle_dir = os.path.relpath(os.path.dirname(path), build_dir) + if os.path.exists(path) and SETTINGS_GRADLE_REGEX.match(os.path.basename(path)): + with open(path) as fp: + for m in GRADLE_SUBPROJECT_REGEX.finditer(fp.read()): + for f in glob.glob(os.path.join(os.path.dirname(path), m.group(1), 'build.gradle*')): + with open(f) as fp: + while True: + line = fp.readline() + if not line: + break + if ANDROID_PLUGIN_REGEX.match(line): + return os.path.relpath(os.path.dirname(f), build_dir) + if first_gradle_dir and first_gradle_dir != '.': + return first_gradle_dir + + return '' + + +def getrepofrompage(url): + """Get the repo type and address from the given web page. + + The page is scanned in a rather naive manner for 'git clone xxxx', + 'hg clone xxxx', etc, and when one of these is found it's assumed + that's the information we want. Returns repotype, address, or + None, reason + + """ + if not url.startswith('http'): + return (None, _('{url} does not start with "http"!'.format(url=url))) + req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out + if req.getcode() != 200: + return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode())) + page = req.read().decode(req.headers.get_content_charset()) + + # Works for BitBucket + m = re.search('data-fetch-url="(.*)"', page) + if m is not None: + repo = m.group(1) + + if repo.endswith('.git'): + return ('git', repo) + + return ('hg', repo) + + # Works for BitBucket (obsolete) + index = page.find('hg clone') + if index != -1: + repotype = 'hg' + repo = page[index + 9:] + index = repo.find('<') + if index == -1: + return (None, _("Error while getting repo address")) + repo = repo[:index] + repo = repo.split('"')[0] + return (repotype, repo) + + # Works for BitBucket (obsolete) + index = page.find('git clone') + if index != -1: + repotype = 'git' + repo = page[index + 10:] + index = repo.find('<') + if index == -1: + return (None, _("Error while getting repo address")) + repo = repo[:index] + repo = repo.split('"')[0] + return (repotype, repo) + + return (None, _("No information found.") + page) + + +def get_app_from_url(url): + """Guess basic app metadata from the URL. + + The URL must include a network hostname, unless it is an lp:, + file:, or git/ssh URL. This throws ValueError on bad URLs to + match urlparse(). + + """ + + parsed = urllib.parse.urlparse(url) + invalid_url = False + if not parsed.scheme or not parsed.path: + invalid_url = True + + app = fdroidserver.metadata.App() + app.Repo = url + if url.startswith('git://') or url.startswith('git@'): + app.RepoType = 'git' + elif parsed.netloc == 'github.com': + app.RepoType = 'git' + app.SourceCode = url + app.IssueTracker = url + '/issues' + elif parsed.netloc == 'gitlab.com' or parsed.netloc == 'framagit.org': + # git can be fussy with gitlab URLs unless they end in .git + if url.endswith('.git'): + url = url[:-4] + app.Repo = url + '.git' + app.RepoType = 'git' + app.SourceCode = url + app.IssueTracker = url + '/issues' + elif parsed.netloc == 'notabug.org': + if url.endswith('.git'): + url = url[:-4] + app.Repo = url + '.git' + app.RepoType = 'git' + app.SourceCode = url + app.IssueTracker = url + '/issues' + elif parsed.netloc == 'bitbucket.org': + if url.endswith('/'): + url = url[:-1] + app.SourceCode = url + '/src' + app.IssueTracker = url + '/issues' + # Figure out the repo type and adddress... + app.RepoType, app.Repo = getrepofrompage(url) + elif parsed.netloc == 'codeberg.org': + app.RepoType = 'git' + app.SourceCode = url + app.IssueTracker = url + '/issues' + elif url.startswith('https://') and url.endswith('.git'): + app.RepoType = 'git' + + if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'): + invalid_url = True + + if invalid_url: + raise ValueError(_('"{url}" is not a valid URL!'.format(url=url))) + + if not app.RepoType: + raise FDroidException("Unable to determine vcs type. " + app.Repo) + + return app + + def parse_srclib_spec(spec): if type(spec) != str: @@ -2361,15 +1869,15 @@ def parse_srclib_spec(spec): "(not a string): '{}'") .format(spec)) - tokens = spec.split('@', 1) - if not tokens[0]: - raise MetaDataException( - _("could not parse srclib spec (no name specified): '{}'").format(spec) - ) - if len(tokens) < 2 or not tokens[1]: - raise MetaDataException( - _("could not parse srclib spec (no ref specified): '{}'").format(spec) - ) + tokens = spec.split('@') + if len(tokens) > 2: + raise MetaDataException(_("could not parse srclib spec " + "(too many '@' signs): '{}'") + .format(spec)) + elif len(tokens) < 2: + raise MetaDataException(_("could not parse srclib spec " + "(no ref specified): '{}'") + .format(spec)) name = tokens[0] ref = tokens[1] @@ -2389,23 +1897,22 @@ def getsrclib(spec, srclib_dir, basepath=False, build=None): """Get the specified source library. - Return the path to it. Normally this is the path to be used when + Returns the path to it. Normally this is the path to be used when referencing it, which may be a subdirectory of the actual project. If you want the base directory of the project, pass 'basepath=True'. - spec and srclib_dir are both strings, not pathlib.Path. """ number = None subdir = None - if not isinstance(spec, str): - spec = str(spec) - if not isinstance(srclib_dir, str): - spec = str(srclib_dir) if raw: name = spec ref = None else: - name, ref, number, subdir = parse_srclib_spec(spec) + name, ref = spec.split('@') + if ':' in name: + number, name = name.split(':', 1) + if '/' in name: + name, subdir = name.split('/', 1) if name not in fdroidserver.metadata.srclibs: raise VCSException('srclib ' + name + ' not found.') @@ -2442,9 +1949,9 @@ def getsrclib(spec, srclib_dir, basepath=False, if prepare: if srclib["Prepare"]: - cmd = replace_config_vars("; ".join(srclib["Prepare"]), build) + cmd = replace_config_vars(srclib["Prepare"], build) - p = FDroidPopen(['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', '--', cmd], cwd=libdir) + p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=libdir) if p.returncode != 0: raise BuildException("Error running prepare command for srclib %s" % name, p.output) @@ -2459,31 +1966,21 @@ gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True): - """Prepare the source code for a particular build. + """ Prepare the source code for a particular build - Parameters - ---------- - vcs - the appropriate vcs object for the application - app - the application details from the metadata - build - the build details from the metadata - build_dir - the path to the build directory, usually 'build/app.id' - srclib_dir - the path to the source libraries directory, usually 'build/srclib' - extlib_dir - the path to the external libraries directory, usually 'build/extlib' + :param vcs: the appropriate vcs object for the application + :param app: the application details from the metadata + :param build: the build details from the metadata + :param build_dir: the path to the build directory, usually 'build/app.id' + :param srclib_dir: the path to the source libraries directory, usually 'build/srclib' + :param extlib_dir: the path to the external libraries directory, usually 'build/extlib' - Returns - ------- - root - is the root directory, which may be the same as 'build_dir' or may - be a subdirectory of it. - srclibpaths - is information on the srclibs being used + Returns the (root, srclibpaths) where: + :param root: is the root directory, which may be the same as 'build_dir' or may + be a subdirectory of it. + :param srclibpaths: is information on the srclibs being used """ + # Optionally, the actual app source can be in a subdirectory if build.subdir: root_dir = os.path.join(build_dir, build.subdir) @@ -2498,8 +1995,6 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= if build.submodules: logging.info(_("Initialising submodules")) vcs.initsubmodules() - else: - vcs.deinitsubmodules() # Check that a subdir (if we're using one) exists. This has to happen # after the checkout, since it might not exist elsewhere @@ -2508,10 +2003,10 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= # Run an init command if one is required if build.init: - cmd = replace_config_vars("; ".join(build.init), build) + cmd = replace_config_vars(build.init, build) logging.info("Running 'init' commands in %s" % root_dir) - p = FDroidPopen(['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', '--', cmd], cwd=root_dir) + p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running init command for %s:%s" % (app.id, build.versionName), p.output) @@ -2584,9 +2079,9 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= with open(path, 'w', encoding='iso-8859-1') as f: f.write(props) - flavors = [] + flavours = [] if build.build_method() == 'gradle': - flavors = build.gradle + flavours = build.gradle if build.target: n = build.target.split('-')[1] @@ -2596,8 +2091,6 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= gradlefile = build_gradle elif os.path.exists(build_gradle_kts): gradlefile = build_gradle_kts - else: - raise BuildException("No gradle file found") regsub_file(r'compileSdkVersion[ =]+[0-9]+', r'compileSdkVersion %s' % n, gradlefile) @@ -2605,31 +2098,31 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= # Remove forced debuggable flags remove_debuggable_flags(root_dir) - # Insert versionCode and number into the manifest if necessary + # Insert version code and number into the manifest if necessary if build.forceversion: - logging.info("Changing the versionName") - for path in manifest_paths(root_dir, flavors): + logging.info("Changing the version name") + for path in manifest_paths(root_dir, flavours): if not os.path.isfile(path): continue - if path.suffix == '.xml': + if path.endswith('.xml'): regsub_file(r'android:versionName="[^"]*"', r'android:versionName="%s"' % build.versionName, path) - elif path.suffix == '.gradle': + elif path.endswith('.gradle'): regsub_file(r"""(\s*)versionName[\s'"=]+.*""", r"""\1versionName '%s'""" % build.versionName, path) if build.forcevercode: - logging.info("Changing the versionCode") - for path in manifest_paths(root_dir, flavors): - if not path.is_file(): + logging.info("Changing the version code") + for path in manifest_paths(root_dir, flavours): + if not os.path.isfile(path): continue - if path.suffix == '.xml': + if path.endswith('.xml'): regsub_file(r'android:versionCode="[^"]*"', r'android:versionCode="%s"' % build.versionCode, path) - elif path.suffix == '.gradle': + elif path.endswith('.gradle'): regsub_file(r'versionCode[ =]+[0-9]+', r'versionCode %s' % build.versionCode, path) @@ -2677,13 +2170,13 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= if build.prebuild: logging.info("Running 'prebuild' commands in %s" % root_dir) - cmd = replace_config_vars("; ".join(build.prebuild), build) + cmd = replace_config_vars(build.prebuild, build) # Substitute source library paths into prebuild commands for name, number, libpath in srclibpaths: cmd = cmd.replace('$$' + name + '$$', os.path.join(os.getcwd(), libpath)) - p = FDroidPopen(['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', '--', cmd], cwd=root_dir) + p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running prebuild command for %s:%s" % (app.id, build.versionName), p.output) @@ -2724,27 +2217,21 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= def getpaths_map(build_dir, globpaths): - """Extend via globbing the paths from a field and return them as a map from original path to resulting paths.""" + """Extend via globbing the paths from a field and return them as a map from original path to resulting paths""" paths = dict() - not_found_paths = [] for p in globpaths: p = p.strip() full_path = os.path.join(build_dir, p) full_path = os.path.normpath(full_path) - paths[p] = [r[len(str(build_dir)) + 1:] for r in glob.glob(full_path)] + paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)] if not paths[p]: - not_found_paths.append(p) - return paths, not_found_paths + raise FDroidException("glob path '%s' did not match any files/dirs" % p) + return paths def getpaths(build_dir, globpaths): - """Extend via globbing the paths from a field and return them as a set.""" - paths_map, not_found_paths = getpaths_map(build_dir, globpaths) - if not_found_paths: - raise FDroidException( - "Some glob paths did not match any files/dirs:\n" - + "\n".join(not_found_paths) - ) + """Extend via globbing the paths from a field and return them as a set""" + paths_map = getpaths_map(build_dir, globpaths) paths = set() for k, v in paths_map.items(): for p in v: @@ -2757,7 +2244,7 @@ def natural_key(s): def check_system_clock(dt_obj, path): - """Check if system clock is updated based on provided date. + """Check if system clock is updated based on provided date If an APK has files newer than the system time, suggest updating the system clock. This is useful for offline systems, used for @@ -2774,105 +2261,136 @@ def check_system_clock(dt_obj, path): class KnownApks: - """Permanent store of existing APKs with the date they were added. + """permanent store of existing APKs with the date they were added This is currently the only way to permanently store the "updated" date of APKs. """ def __init__(self): - """Load filename/date info about previously seen APKs. + '''Load filename/date info about previously seen APKs Since the appid and date strings both will never have spaces, this is parsed as a list from the end to allow the filename to have any combo of spaces. - """ + ''' + + self.path = os.path.join('stats', 'known_apks.txt') self.apks = {} - for part in ('repo', 'archive'): - path = os.path.join(part, 'index-v2.json') - if os.path.isfile(path): - with open(path, 'r', encoding='utf-8') as f: - index = json.load(f) - for appid, data in index["packages"].items(): - for version in data["versions"].values(): - filename = version["file"]["name"][1:] - date = datetime.fromtimestamp(version["added"] // 1000, tz=timezone.utc) - self.apks[filename] = date + if os.path.isfile(self.path): + with open(self.path, 'r') as f: + for line in f: + t = line.rstrip().split(' ') + if len(t) == 2: + self.apks[t[0]] = (t[1], None) + else: + appid = t[-2] + date = datetime.strptime(t[-1], '%Y-%m-%d') + filename = line[0:line.rfind(appid) - 1] + self.apks[filename] = (appid, date) + check_system_clock(date, self.path) + self.changed = False - def recordapk(self, apkName, default_date=None): - """ - Record an APK (if it's new, otherwise does nothing). + def writeifchanged(self): + if not self.changed: + return - Returns - ------- - datetime - the date it was added as a datetime instance. - """ + if not os.path.exists('stats'): + os.mkdir('stats') + + lst = [] + for apk, app in self.apks.items(): + appid, added = app + line = apk + ' ' + appid + if added: + line += ' ' + added.strftime('%Y-%m-%d') + lst.append(line) + + with open(self.path, 'w') as f: + for line in sorted(lst, key=natural_key): + f.write(line + '\n') + + def recordapk(self, apkName, app, default_date=None): + ''' + Record an apk (if it's new, otherwise does nothing) + Returns the date it was added as a datetime instance + ''' if apkName not in self.apks: if default_date is None: - default_date = datetime.now(timezone.utc) - self.apks[apkName] = default_date - return self.apks[apkName] + default_date = datetime.utcnow() + self.apks[apkName] = (app, default_date) + self.changed = True + _ignored, added = self.apks[apkName] + return added + + def getapp(self, apkname): + """Look up information - given the 'apkname', returns (app id, date added/None). + + Or returns None for an unknown apk. + """ + if apkname in self.apks: + return self.apks[apkname] + return None + + def getlatest(self, num): + """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first""" + apps = {} + for apk, app in self.apks.items(): + appid, added = app + if added: + if appid in apps: + if apps[appid] > added: + apps[appid] = added + else: + apps[appid] = added + sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:] + lst = [app for app, _ignored in sortedapps] + lst.reverse() + return lst def get_file_extension(filename): - """Get the normalized file extension, can be blank string but never None.""" + """get the normalized file extension, can be blank string but never None""" if isinstance(filename, bytes): filename = filename.decode('utf-8') return os.path.splitext(filename)[1].lower()[1:] -def _androguard_logging_level(level=logging.ERROR): - """Tames androguard's default debug output. +def use_androguard(): + """Report if androguard is available, and config its debug logging""" - There should be no debug output when the functions are being used - via the API. Otherwise, the output is controlled by the --verbose - flag. - - To get coverage across the full range of androguard >= 3.3.5, this - includes all known logger names that are relevant. So some of - these names might not be present in the version of androguard - currently in use. - - """ - if options and options.verbose: - level = logging.WARNING - - for name in ( - 'androguard.apk', - 'androguard.axml', - 'androguard.core.api_specific_resources', - 'androguard.core.apk', - 'androguard.core.axml', - ): - logging.getLogger(name).setLevel(level) - - # some parts of androguard 4.x use loguru instead of logging try: - from loguru import logger - logger.remove() + import androguard + if use_androguard.show_path: + logging.debug(_('Using androguard from "{path}"').format(path=androguard.__file__)) + use_androguard.show_path = False + if options and options.verbose: + logging.getLogger("androguard.axml").setLevel(logging.INFO) + return True except ImportError: - pass + return False -def get_androguard_APK(apkfile, skip_analysis=False): +use_androguard.show_path = True + + +def _get_androguard_APK(apkfile): try: - # these were moved in androguard 4.0 - from androguard.core.apk import APK - except ImportError: from androguard.core.bytecodes.apk import APK - _androguard_logging_level() + except ImportError: + raise FDroidException("androguard library is not installed") - return APK(apkfile, skip_analysis=skip_analysis) + return APK(apkfile) def ensure_final_value(packageName, arsc, value): - """Ensure incoming value is always the value, not the resid. + """Ensure incoming value is always the value, not the resid androguard will sometimes return the Android "resId" aka Resource ID instead of the actual value. This checks whether the value is actually a resId, then performs the Android Resource lookup as needed. + """ if value: returnValue = value @@ -2887,29 +2405,16 @@ def ensure_final_value(packageName, arsc, value): return '' -def is_debuggable_or_testOnly(apkfile): - """Return True if the given file is an APK and is debuggable or testOnly. +def is_apk_and_debuggable(apkfile): + """Returns True if the given file is an APK and is debuggable - These two settings should never be enabled in release builds. This - parses - from the APK and nothing else to run fast, since it is run on - every APK as part of update. + Parse only from the APK. - Parameters - ---------- - apkfile - full path to the APK to check + :param apkfile: full path to the apk to check""" - """ if get_file_extension(apkfile) != 'apk': return False - try: - # these were moved in androguard 4.0 - from androguard.core.axml import START_TAG, AXMLParser, format_value - except ImportError: - from androguard.core.bytecodes.axml import START_TAG, AXMLParser, format_value - _androguard_logging_level() - + from androguard.core.bytecodes.axml import AXMLParser, format_value, START_TAG with ZipFile(apkfile) as apk: with apk.open('AndroidManifest.xml') as manifest: axml = AXMLParser(manifest.read()) @@ -2918,7 +2423,7 @@ def is_debuggable_or_testOnly(apkfile): if _type == START_TAG and axml.getName() == 'application': for i in range(0, axml.getAttributeCount()): name = axml.getAttributeName(i) - if name in ('debuggable', 'testOnly'): + if name == 'debuggable': _type = axml.getAttributeValueType(i) _data = axml.getAttributeValueData(i) value = format_value(_type, _data, lambda _: axml.getAttributeValue(i)) @@ -2938,30 +2443,20 @@ def get_apk_id(apkfile): APK, aapt still can. So aapt is also used as the final fallback method. - Parameters - ---------- - apkfile - path to an APK file. - - Returns - ------- - appid - versionCode - versionName + :param apkfile: path to an APK file. + :returns: triplet (appid, version code, version name) """ try: return get_apk_id_androguard(apkfile) except zipfile.BadZipFile as e: - if config and 'aapt' in config: - logging.error(apkfile + ': ' + str(e)) + logging.error(apkfile + ': ' + str(e)) + if 'aapt' in config: return get_apk_id_aapt(apkfile) - else: - raise e def get_apk_id_androguard(apkfile): - """Read (appid, versionCode, versionName) from an APK. + """Read (appid, versionCode, versionName) from an APK This first tries to do quick binary XML parsing to just get the values that are needed. It will fallback to full androguard @@ -2969,34 +2464,12 @@ def get_apk_id_androguard(apkfile): versionName is set to a Android String Resource (e.g. an integer hex value that starts with @). - This function is part of androguard as get_apkid(), so this - vendored and modified to return versionCode as an integer. - """ if not os.path.exists(apkfile): raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'") .format(apkfilename=apkfile)) - try: - # these were moved in androguard 4.0 - from androguard.core.axml import ( - END_DOCUMENT, - END_TAG, - START_TAG, - TEXT, - AXMLParser, - format_value, - ) - except ImportError: - from androguard.core.bytecodes.axml import ( - END_DOCUMENT, - END_TAG, - START_TAG, - TEXT, - AXMLParser, - format_value, - ) - _androguard_logging_level() + from androguard.core.bytecodes.axml import AXMLParser, format_value, START_TAG, END_TAG, TEXT, END_DOCUMENT appid = None versionCode = None @@ -3018,20 +2491,20 @@ def get_apk_id_androguard(apkfile): appid = value elif versionCode is None and name == 'versionCode': if value.startswith('0x'): - versionCode = int(value, 16) + versionCode = str(int(value, 16)) else: - versionCode = int(value) + versionCode = value elif versionName is None and name == 'versionName': versionName = value if axml.getName() == 'manifest': break - elif _type in (END_TAG, TEXT, END_DOCUMENT): + elif _type == END_TAG or _type == TEXT or _type == END_DOCUMENT: raise RuntimeError('{path}: must be the first element in AndroidManifest.xml' .format(path=apkfile)) if not versionName or versionName[0] == '@': - a = get_androguard_APK(apkfile) + a = _get_androguard_APK(apkfile) versionName = ensure_final_value(a.package, a.get_android_resources(), a.get_androidversion_name()) if not versionName: versionName = '' # versionName is expected to always be a str @@ -3040,22 +2513,17 @@ def get_apk_id_androguard(apkfile): def get_apk_id_aapt(apkfile): - """Read (appid, versionCode, versionName) from an APK.""" p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False) m = APK_ID_TRIPLET_REGEX.match(p.output[0:p.output.index('\n')]) if m: - return m.group(1), int(m.group(2)), m.group(3) - raise FDroidException(_( - "Reading packageName/versionCode/versionName failed," - "APK invalid: '{apkfilename}'" - ).format(apkfilename=apkfile)) + return m.group(1), m.group(2), m.group(3) + raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'") + .format(apkfilename=apkfile)) def get_native_code(apkfile): - """Aapt checks if there are architecture folders under the lib/ folder. - - We are simulating the same behaviour. - """ + """aapt checks if there are architecture folders under the lib/ folder + so we are simulating the same behaviour""" arch_re = re.compile("^lib/(.*)/.*$") archset = set() with ZipFile(apkfile) as apk: @@ -3067,9 +2535,9 @@ def get_native_code(apkfile): class PopenResult: - def __init__(self, returncode=None, output=None): - self.returncode = returncode - self.output = output + def __init__(self): + self.returncode = None + self.output = None def SdkToolsPopen(commands, cwd=None, output=True): @@ -3089,19 +2557,12 @@ def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdou """ Run a command and capture the possibly huge output as bytes. - Parameters - ---------- - commands - command and argument list like in subprocess.Popen - cwd - optionally specifies a working directory - envs - a optional dictionary of environment variables and their values - - Returns - ------- - A PopenResult. + :param commands: command and argument list like in subprocess.Popen + :param cwd: optionally specifies a working directory + :param envs: a optional dictionary of environment variables and their values + :returns: A PopenResult. """ + global env if env is None: set_FDroidPopen_env() @@ -3124,7 +2585,7 @@ def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdou stderr=stderr_param) except OSError as e: raise BuildException("OSError while trying to execute " - + ' '.join(commands) + ': ' + str(e)) from e + + ' '.join(commands) + ': ' + str(e)) # TODO are these AsynchronousFileReader threads always exiting? if not stderr_to_stdout and options.verbose: @@ -3147,7 +2608,7 @@ def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdou while not stdout_reader.eof(): while not stdout_queue.empty(): line = stdout_queue.get() - if output and options and options.verbose: + if output and options.verbose: # Output directly to console sys.stderr.buffer.write(line) sys.stderr.flush() @@ -3171,18 +2632,10 @@ def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=Tru """ Run a command and capture the possibly huge output as a str. - Parameters - ---------- - commands - command and argument list like in subprocess.Popen - cwd - optionally specifies a working directory - envs - a optional dictionary of environment variables and their values - - Returns - ------- - A PopenResult. + :param commands: command and argument list like in subprocess.Popen + :param cwd: optionally specifies a working directory + :param envs: a optional dictionary of environment variables and their values + :returns: A PopenResult. """ result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout) result.output = result.output.decode('utf-8', 'ignore') @@ -3271,28 +2724,23 @@ def remove_signing_keys(build_dir): logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path)) -def set_FDroidPopen_env(app=None, build=None): - """Set up the environment variables for the build environment. +def set_FDroidPopen_env(build=None): + ''' + set up the environment variables for the build environment There is only a weak standard, the variables used by gradle, so also set up the most commonly used environment variables for SDK and NDK. Also, if there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8. - - If an App instance is provided, then the SOURCE_DATE_EPOCH - environment variable will be set based on that app's source repo. - - """ + ''' global env, orig_path if env is None: env = os.environ orig_path = env['PATH'] - if config: - if config.get('sdk_path'): - for n in ['ANDROID_HOME', 'ANDROID_SDK', 'ANDROID_SDK_ROOT']: - env[n] = config['sdk_path'] - for k, v in config.get('java_paths', {}).items(): - env['JAVA%s_HOME' % k] = v + for n in ['ANDROID_HOME', 'ANDROID_SDK']: + env[n] = config['sdk_path'] + for k, v in config['java_paths'].items(): + env['JAVA%s_HOME' % k] = v missinglocale = True for k, v in env.items(): @@ -3303,12 +2751,10 @@ def set_FDroidPopen_env(app=None, build=None): if missinglocale: env['LANG'] = 'en_US.UTF-8' - if app: - env['SOURCE_DATE_EPOCH'] = get_source_date_epoch(get_build_dir(app)) if build is not None: path = build.ndk_path() paths = orig_path.split(os.pathsep) - if path and path not in paths: + if path not in paths: paths = [path] + paths env['PATH'] = os.pathsep.join(paths) for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']: @@ -3325,6 +2771,7 @@ def replace_build_vars(cmd, build): def replace_config_vars(cmd, build): cmd = cmd.replace('$$SDK$$', config['sdk_path']) cmd = cmd.replace('$$NDK$$', build.ndk_path()) + cmd = cmd.replace('$$MVN3$$', config['mvn3']) if build is not None: cmd = replace_build_vars(cmd, build) return cmd @@ -3362,170 +2809,99 @@ def signer_fingerprint_short(cert_encoded): Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint for a given pkcs7 signature. - Parameters - ---------- - cert_encoded - Contents of an APK signing certificate. - - Returns - ------- - shortened signing-key fingerprint. + :param cert_encoded: Contents of an APK signing certificate. + :returns: shortened signing-key fingerprint. """ return signer_fingerprint(cert_encoded)[:7] def signer_fingerprint(cert_encoded): - """Return SHA-256 signer fingerprint for PKCS#7 DER-encoded signature. + """Obtain sha256 signing-key fingerprint for pkcs7 DER certificate. - Parameters - ---------- - Contents of an APK signature. - - Returns - ------- - Standard SHA-256 signer fingerprint. + Extracts hexadecimal sha256 signing-key fingerprint string + for a given pkcs7 signature. + :param: Contents of an APK signature. + :returns: shortened signature fingerprint. """ return hashlib.sha256(cert_encoded).hexdigest() def get_first_signer_certificate(apkpath): - """Get the first signing certificate from the APK, DER-encoded. - - JAR and APK Signatures allow for multiple signers, though it is - rarely used, and this is poorly documented. So this method only - fetches the first certificate, and errors out if there are more. - - Starting with targetSdkVersion 30, APK v2 Signatures are required. - https://developer.android.com/about/versions/11/behavior-changes-11#minimum-signature-scheme - - When a APK v2+ signature is present, the JAR signature is not - verified. The verifier parses the signers from the v2+ signature - and does not seem to look at the JAR signature. - https://source.android.com/docs/security/features/apksigning/v2#apk-signature-scheme-v2-block - https://android.googlesource.com/platform/tools/apksig/+/refs/tags/android-13.0.0_r3/src/main/java/com/android/apksig/ApkVerifier.java#270 - - apksigner checks that the signers from all the APK signatures match: - https://android.googlesource.com/platform/tools/apksig/+/refs/tags/android-13.0.0_r3/src/main/java/com/android/apksig/ApkVerifier.java#383 - - apksigner verifies each signer's signature block file - .(RSA|DSA|EC) against the corresponding signature file .SF - https://android.googlesource.com/platform/tools/apksig/+/refs/tags/android-13.0.0_r3/src/main/java/com/android/apksig/internal/apk/v1/V1SchemeVerifier.java#280 - - NoOverwriteDict is a workaround for: - https://github.com/androguard/androguard/issues/1030 - - Lots more discusion here: - https://gitlab.com/fdroid/fdroidserver/-/issues/1128 - - """ - - class NoOverwriteDict(dict): - def __setitem__(self, k, v): - if k not in self: - super().__setitem__(k, v) - + """Get the first signing certificate from the APK, DER-encoded""" + certs = None cert_encoded = None - found_certs = [] - apkobject = get_androguard_APK(apkpath) - apkobject._v2_blocks = NoOverwriteDict() - certs_v3 = apkobject.get_certificates_der_v3() - if certs_v3: - cert_v3 = certs_v3[0] - found_certs.append(cert_v3) - if not cert_encoded: - logging.debug(_('Using APK Signature v3')) - cert_encoded = cert_v3 + with zipfile.ZipFile(apkpath, 'r') as apk: + cert_files = [n for n in apk.namelist() if SIGNATURE_BLOCK_FILE_REGEX.match(n)] + if len(cert_files) > 1: + logging.error(_("Found multiple JAR Signature Block Files in {path}").format(path=apkpath)) + return None + elif len(cert_files) == 1: + cert_encoded = get_certificate(apk.read(cert_files[0])) - certs_v2 = apkobject.get_certificates_der_v2() - if certs_v2: - cert_v2 = certs_v2[0] - found_certs.append(cert_v2) - if not cert_encoded: + if not cert_encoded and use_androguard(): + apkobject = _get_androguard_APK(apkpath) + certs = apkobject.get_certificates_der_v2() + if len(certs) > 0: logging.debug(_('Using APK Signature v2')) - cert_encoded = cert_v2 - - if get_min_sdk_version(apkobject) < 24 or ( - not (certs_v3 or certs_v2) and get_effective_target_sdk_version(apkobject) < 30 - ): - with zipfile.ZipFile(apkpath, 'r') as apk: - cert_files = [ - n for n in apk.namelist() if SIGNATURE_BLOCK_FILE_REGEX.match(n) - ] - if len(cert_files) > 1: - logging.error( - _("Found multiple JAR Signature Block Files in {path}").format( - path=apkpath - ) - ) - return - elif len(cert_files) == 1: - signature_block_file = cert_files[0] - signature_file = ( - cert_files[0][: signature_block_file.rindex('.')] + '.SF' - ) - cert_v1 = get_certificate( - apk.read(signature_block_file), - apk.read(signature_file), - ) - found_certs.append(cert_v1) - if not cert_encoded: - logging.debug(_('Using JAR Signature')) - cert_encoded = cert_v1 + cert_encoded = certs[0] + if not cert_encoded: + certs = apkobject.get_certificates_der_v3() + if len(certs) > 0: + logging.debug(_('Using APK Signature v3')) + cert_encoded = certs[0] if not cert_encoded: logging.error(_("No signing certificates found in {path}").format(path=apkpath)) - return - - if not all(cert == found_certs[0] for cert in found_certs): - logging.error( - _("APK signatures have different certificates in {path}:").format( - path=apkpath - ) - ) - return - + return None return cert_encoded def apk_signer_fingerprint(apk_path): - """Get SHA-256 fingerprint string for the first signer from given APK. + """Obtain sha256 signing-key fingerprint for APK. - Parameters - ---------- - apk_path - path to APK - - Returns - ------- - Standard SHA-256 signer fingerprint + Extracts hexadecimal sha256 signing-key fingerprint string + for a given APK. + :param apk_path: path to APK + :returns: signature fingerprint """ + cert_encoded = get_first_signer_certificate(apk_path) if not cert_encoded: return None return signer_fingerprint(cert_encoded) +def apk_signer_fingerprint_short(apk_path): + """Obtain shortened sha256 signing-key fingerprint for APK. + + Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint + for a given pkcs7 APK. + + :param apk_path: path to APK + :returns: shortened signing-key fingerprint + """ + return apk_signer_fingerprint(apk_path)[:7] + + def metadata_get_sigdir(appid, vercode=None): - """Get signature directory for app.""" + """Get signature directory for app""" if vercode: - return os.path.join('metadata', appid, 'signatures', str(vercode)) + return os.path.join('metadata', appid, 'signatures', vercode) else: return os.path.join('metadata', appid, 'signatures') def metadata_find_developer_signature(appid, vercode=None): - """Try to find the developer signature for given appid. + """Tires to find the developer signature for given appid. This picks the first signature file found in metadata an returns its signature. - Returns - ------- - sha256 signing key fingerprint of the developer signing key. - None in case no signature can not be found. - """ + :returns: sha256 signing key fingerprint of the developer signing key. + None in case no signature can not be found.""" + # fetch list of dirs for all versions of signatures appversigdirs = [] if vercode: @@ -3540,75 +2916,45 @@ def metadata_find_developer_signature(appid, vercode=None): appversigdirs.append(appversigdir) for sigdir in appversigdirs: - signature_block_files = ( - glob.glob(os.path.join(sigdir, '*.DSA')) - + glob.glob(os.path.join(sigdir, '*.EC')) - + glob.glob(os.path.join(sigdir, '*.RSA')) - ) - if len(signature_block_files) > 1: + sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \ + glob.glob(os.path.join(sigdir, '*.EC')) + \ + glob.glob(os.path.join(sigdir, '*.RSA')) + if len(sigs) > 1: raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir)) - for signature_block_file in signature_block_files: - with open(signature_block_file, 'rb') as f: + for sig in sigs: + with open(sig, 'rb') as f: return signer_fingerprint(get_certificate(f.read())) return None def metadata_find_signing_files(appid, vercode): - """Get a list of signed manifests and signatures. + """Gets a list of singed manifests and signatures. - Parameters - ---------- - appid - app id string - vercode - app versionCode - - Returns - ------- - List - of 4-tuples for each signing key with following paths: - (signature_file, signature_block_file, manifest, v2_files), where v2_files - is either a (apk_signing_block_offset_file, apk_signing_block_file) pair or None - - References - ---------- - * https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html - * https://source.android.com/security/apksigning/v2 - * https://source.android.com/security/apksigning/v3 + :param appid: app id string + :param vercode: app version code + :returns: a list of triplets for each signing key with following paths: + (signature_file, singed_file, manifest_file) """ ret = [] sigdir = metadata_get_sigdir(appid, vercode) - signature_block_files = ( - glob.glob(os.path.join(sigdir, '*.DSA')) - + glob.glob(os.path.join(sigdir, '*.EC')) - + glob.glob(os.path.join(sigdir, '*.RSA')) - ) - signature_block_pat = re.compile(r'(\.DSA|\.EC|\.RSA)$') - apk_signing_block = os.path.join(sigdir, "APKSigningBlock") - apk_signing_block_offset = os.path.join(sigdir, "APKSigningBlockOffset") - if os.path.isfile(apk_signing_block) and os.path.isfile(apk_signing_block_offset): - v2_files = apk_signing_block, apk_signing_block_offset - else: - v2_files = None - for signature_block_file in signature_block_files: - signature_file = signature_block_pat.sub('.SF', signature_block_file) - if os.path.isfile(signature_file): - manifest = os.path.join(sigdir, 'MANIFEST.MF') - if os.path.isfile(manifest): - ret.append((signature_block_file, signature_file, manifest, v2_files)) + sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \ + glob.glob(os.path.join(sigdir, '*.EC')) + \ + glob.glob(os.path.join(sigdir, '*.RSA')) + extre = re.compile(r'(\.DSA|\.EC|\.RSA)$') + for sig in sigs: + sf = extre.sub('.SF', sig) + if os.path.isfile(sf): + mf = os.path.join(sigdir, 'MANIFEST.MF') + if os.path.isfile(mf): + ret.append((sig, sf, mf)) return ret def metadata_find_developer_signing_files(appid, vercode): """Get developer signature files for specified app from metadata. - Returns - ------- - List - of 4-tuples for each signing key with following paths: - (signature_file, signature_block_file, manifest, v2_files), where v2_files - is either a (apk_signing_block_offset_file, apk_signing_block_file) pair or None - + :returns: A triplet of paths for signing files from metadata: + (signature_file, singed_file, manifest_file) """ allsigningfiles = metadata_find_signing_files(appid, vercode) if allsigningfiles and len(allsigningfiles) == 1: @@ -3617,47 +2963,12 @@ def metadata_find_developer_signing_files(appid, vercode): return None -class ClonedZipInfo(zipfile.ZipInfo): - """Hack to allow fully cloning ZipInfo instances. - - The zipfile library has some bugs that prevent it from fully - cloning ZipInfo entries. https://bugs.python.org/issue43547 - - """ - - def __init__(self, zinfo): - super().__init__() - self.original = zinfo - for k in self.__slots__: - try: - setattr(self, k, getattr(zinfo, k)) - except AttributeError: - pass - - def __getattribute__(self, name): - if name in ("date_time", "external_attr", "flag_bits"): - return getattr(self.original, name) - return object.__getattribute__(self, name) - - -def apk_has_v1_signatures(apkfile): - """Test whether an APK has v1 signature files.""" - with ZipFile(apkfile, 'r') as apk: - for info in apk.infolist(): - if APK_SIGNATURE_FILES.match(info.filename): - return True - return False - - def apk_strip_v1_signatures(signed_apk, strip_manifest=False): - """Remove signatures from APK. + """Removes signatures from APK. - Parameters - ---------- - signed_apk - path to APK file. - strip_manifest - when set to True also the manifest file will be removed from the APK. + :param signed_apk: path to apk file. + :param strip_manifest: when set to True also the manifest file will + be removed from the APK. """ with tempfile.TemporaryDirectory() as tmpdir: tmp_apk = os.path.join(tmpdir, 'tmp.apk') @@ -3669,70 +2980,75 @@ def apk_strip_v1_signatures(signed_apk, strip_manifest=False): if strip_manifest: if info.filename != 'META-INF/MANIFEST.MF': buf = in_apk.read(info.filename) - out_apk.writestr(ClonedZipInfo(info), buf) + out_apk.writestr(info, buf) else: buf = in_apk.read(info.filename) - out_apk.writestr(ClonedZipInfo(info), buf) + out_apk.writestr(info, buf) -def apk_implant_signatures(apkpath, outpath, manifest): - """Implant a signature from metadata into an APK. +def _zipalign(unsigned_apk, aligned_apk): + """run 'zipalign' using standard flags used by Gradle Android Plugin + + -p was added in build-tools-23.0.0 + + https://developer.android.com/studio/publish/app-signing#sign-manually + """ + p = SdkToolsPopen(['zipalign', '-v', '-p', '4', unsigned_apk, aligned_apk]) + if p.returncode != 0: + raise BuildException("Failed to align application") + + +def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest): + """Implats a signature from metadata into an APK. Note: this changes there supplied APK in place. So copy it if you need the original to be preserved. - Parameters - ---------- - apkpath - location of the unsigned apk - outpath - location of the output apk - - References - ---------- - * https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html - * https://source.android.com/security/apksigning/v2 - * https://source.android.com/security/apksigning/v3 - + :param apkpath: location of the apk """ - sigdir = os.path.dirname(manifest) # FIXME - apksigcopier.do_patch(sigdir, apkpath, outpath, v1_only=None, - exclude=apksigcopier.exclude_meta) + # get list of available signature files in metadata + with tempfile.TemporaryDirectory() as tmpdir: + apkwithnewsig = os.path.join(tmpdir, 'newsig.apk') + with ZipFile(apkpath, 'r') as in_apk: + with ZipFile(apkwithnewsig, 'w') as out_apk: + for sig_file in [signaturefile, signedfile, manifest]: + with open(sig_file, 'rb') as fp: + buf = fp.read() + info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file)) + info.compress_type = zipfile.ZIP_DEFLATED + info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses + out_apk.writestr(info, buf) + for info in in_apk.infolist(): + if not APK_SIGNATURE_FILES.match(info.filename): + if info.filename != 'META-INF/MANIFEST.MF': + buf = in_apk.read(info.filename) + out_apk.writestr(info, buf) + os.remove(apkpath) + _zipalign(apkwithnewsig, apkpath) -def apk_extract_signatures(apkpath, outdir): - """Extract a signature files from APK and puts them into target directory. - - Parameters - ---------- - apkpath - location of the apk - outdir - older where the extracted signature files will be stored - - References - ---------- - * https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html - * https://source.android.com/security/apksigning/v2 - * https://source.android.com/security/apksigning/v3 +def apk_extract_signatures(apkpath, outdir, manifest=True): + """Extracts a signature files from APK and puts them into target directory. + :param apkpath: location of the apk + :param outdir: folder where the extracted signature files will be stored + :param manifest: (optionally) disable extracting manifest file """ - apksigcopier.do_extract(apkpath, outdir, v1_only=None) + with ZipFile(apkpath, 'r') as in_apk: + for f in in_apk.infolist(): + if APK_SIGNATURE_FILES.match(f.filename) or \ + (manifest and f.filename == 'META-INF/MANIFEST.MF'): + newpath = os.path.join(outdir, os.path.basename(f.filename)) + with open(newpath, 'wb') as out_file: + out_file.write(in_apk.read(f.filename)) def get_min_sdk_version(apk): - """Wrap the androguard function to always return an integer. - - Fall back to 1 if we can't get a valid minsdk version. - - Parameters - ---------- - apk - androguard APK object - - Returns - ------- - minSdkVersion: int + """ + This wraps the androguard function to always return and int and fall back to 1 + if we can't get a valid minsdk version + :param apk: androguard apk object + :return: minsdk as int """ try: return int(apk.get_min_sdk_version()) @@ -3740,185 +3056,208 @@ def get_min_sdk_version(apk): return 1 -def get_effective_target_sdk_version(apk): - """Wrap the androguard function to always return an integer. - - Parameters - ---------- - apk - androguard APK object - - Returns - ------- - targetSdkVersion: int - """ - try: - return int(apk.get_effective_target_sdk_version()) - except TypeError: - return get_min_sdk_version(apk) - - -def get_apksigner_smartcardoptions(smartcardoptions): - if '-providerName' in smartcardoptions.copy(): - pos = smartcardoptions.index('-providerName') - # remove -providerName and it's argument - del smartcardoptions[pos] - del smartcardoptions[pos] - replacements = {'-storetype': '--ks-type', - '-providerClass': '--provider-class', - '-providerArg': '--provider-arg'} - return [replacements.get(n, n) for n in smartcardoptions] - - def sign_apk(unsigned_path, signed_path, keyalias): - """Sign an unsigned APK, then save to a new file, deleting the unsigned. + """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned - NONE is a Java keyword used to configure smartcards as the - keystore. Otherwise, the keystore is a local file. - https://docs.oracle.com/javase/7/docs/technotes/guides/security/p11guide.html#KeyToolJarSigner + Use apksigner for making v2 and v3 signature for apks with targetSDK >=30 as + otherwise they won't be installable on Android 11/R. - When using smartcards, apksigner does not use the same options has - Java/keytool/jarsigner (-providerName, -providerClass, - -providerArg, -storetype). apksigner documents the options as - --ks-provider-class and --ks-provider-arg. Those seem to be - accepted but fail when actually making a signature with weird - internal exceptions. We use the options that actually work. From: - https://geoffreymetais.github.io/code/key-signing/#scripting + Otherwise use jarsigner for v1 only signatures until we have apksig v2/v3 + signature transplantig support. + + When using jarsigner we need to manually select the hash algorithm, + apksigner does this automatically. Apksigner also does the zipalign for us. + + SHA-256 support was added in android-18 (4.3), before then, the only options were MD5 + and SHA1. This aims to use SHA-256 when the APK does not target + older Android versions, and is therefore safe to do so. + + https://issuetracker.google.com/issues/36956587 + https://android-review.googlesource.com/c/platform/libcore/+/44491 """ - if config['keystore'] == 'NONE': - signing_args = get_apksigner_smartcardoptions(config['smartcardoptions']) + apk = _get_androguard_APK(unsigned_path) + if apk.get_effective_target_sdk_version() >= 30: + if config['keystore'] == 'NONE': + # NOTE: apksigner doesn't like -providerName/--provider-name at all, don't use that. + # apksigner documents the options as --ks-provider-class and --ks-provider-arg + # those seem to be accepted but fail when actually making a signature with + # weird internal exceptions. Those options actually work. + # From: https://geoffreymetais.github.io/code/key-signing/#scripting + apksigner_smartcardoptions = config['smartcardoptions'].copy() + if '-providerName' in apksigner_smartcardoptions: + pos = config['smartcardoptions'].index('-providerName') + # remove -providerName and it's argument + del apksigner_smartcardoptions[pos] + del apksigner_smartcardoptions[pos] + replacements = {'-storetype': '--ks-type', + '-providerClass': '--provider-class', + '-providerArg': '--provider-arg'} + signing_args = [replacements.get(n, n) for n in apksigner_smartcardoptions] + else: + signing_args = ['--key-pass', 'env:FDROID_KEY_PASS'] + if not find_apksigner(): + raise BuildException(_("apksigner not found, it's required for signing!")) + cmd = [find_apksigner(), 'sign', + '--ks', config['keystore'], + '--ks-pass', 'env:FDROID_KEY_STORE_PASS'] + cmd += signing_args + cmd += ['--ks-key-alias', keyalias, + '--in', unsigned_path, + '--out', signed_path] + p = FDroidPopen(cmd, envs={ + 'FDROID_KEY_STORE_PASS': config['keystorepass'], + 'FDROID_KEY_PASS': config.get('keypass', "")}) + if p.returncode != 0: + raise BuildException(_("Failed to sign application"), p.output) + os.remove(unsigned_path) else: - signing_args = ['--key-pass', 'env:FDROID_KEY_PASS'] - apksigner = config.get('apksigner', '') - if not shutil.which(apksigner): - raise BuildException(_("apksigner not found, it's required for signing!")) - cmd = [apksigner, 'sign', - '--ks', config['keystore'], - '--ks-pass', 'env:FDROID_KEY_STORE_PASS'] - cmd += signing_args - cmd += ['--ks-key-alias', keyalias, - '--in', unsigned_path, - '--out', signed_path] - p = FDroidPopen(cmd, envs={ - 'FDROID_KEY_STORE_PASS': config['keystorepass'], - 'FDROID_KEY_PASS': config.get('keypass', "")}) - if p.returncode != 0: - if os.path.exists(signed_path): - os.remove(signed_path) - raise BuildException(_("Failed to sign application"), p.output) - os.remove(unsigned_path) + + if get_min_sdk_version(apk) < 18: + signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1'] + else: + signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256'] + if config['keystore'] == 'NONE': + signing_args = config['smartcardoptions'] + else: + signing_args = ['-keypass:env', 'FDROID_KEY_PASS'] + + cmd = [config['jarsigner'], '-keystore', config['keystore'], + '-storepass:env', 'FDROID_KEY_STORE_PASS'] + cmd += signing_args + cmd += signature_algorithm + cmd += [unsigned_path, keyalias] + p = FDroidPopen(cmd, envs={ + 'FDROID_KEY_STORE_PASS': config['keystorepass'], + 'FDROID_KEY_PASS': config.get('keypass', "")}) + if p.returncode != 0: + raise BuildException(_("Failed to sign application"), p.output) + + _zipalign(unsigned_path, signed_path) + os.remove(unsigned_path) -def verify_apks( - signed_apk, unsigned_apk, tmp_dir, v1_only=None, clean_up_verified=False -): - """Verify that two apks are the same. +def verify_apks(signed_apk, unsigned_apk, tmp_dir): + """Verify that two apks are the same One of the inputs is signed, the other is unsigned. The signature metadata - is transferred from the signed to the unsigned apk, and then apksigner is - used to verify that the signature from the signed APK is also valid for + is transferred from the signed to the unsigned apk, and then jarsigner is + used to verify that the signature from the signed apk is also valid for the unsigned one. If the APK given as unsigned actually does have a signature, it will be stripped out and ignored. - Parameters - ---------- - signed_apk - Path to a signed APK file - unsigned_apk - Path to an unsigned APK file expected to match it - tmp_dir - Path to directory for temporary files - v1_only - True for v1-only signatures, False for v1 and v2 signatures, - or None for autodetection - clean_up_verified - Remove any files created here if the verification succeeded. - - Returns - ------- - None if the verification is successful, otherwise a string describing what went wrong. + :param signed_apk: Path to a signed apk file + :param unsigned_apk: Path to an unsigned apk file expected to match it + :param tmp_dir: Path to directory for temporary files + :returns: None if the verification is successful, otherwise a string + describing what went wrong. """ - if not verify_apk_signature(signed_apk): - logging.info('...NOT verified - {0}'.format(signed_apk)) - return 'verification of signed APK failed' if not os.path.isfile(signed_apk): return 'can not verify: file does not exists: {}'.format(signed_apk) + if not os.path.isfile(unsigned_apk): return 'can not verify: file does not exists: {}'.format(unsigned_apk) - tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk)) + with ZipFile(signed_apk, 'r') as signed: + meta_inf_files = ['META-INF/MANIFEST.MF'] + for f in signed.namelist(): + if APK_SIGNATURE_FILES.match(f): + meta_inf_files.append(f) + if len(meta_inf_files) < 3: + return "Signature files missing from {0}".format(signed_apk) + tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk)) + with ZipFile(unsigned_apk, 'r') as unsigned: + # only read the signature from the signed APK, everything else from unsigned + with ZipFile(tmp_apk, 'w') as tmp: + for filename in meta_inf_files: + tmp.writestr(signed.getinfo(filename), signed.read(filename)) + for info in unsigned.infolist(): + if info.filename in meta_inf_files: + logging.warning('Ignoring %s from %s', info.filename, unsigned_apk) + continue + if info.filename in tmp.namelist(): + return "duplicate filename found: " + info.filename + tmp.writestr(info, unsigned.read(info.filename)) + + # Use jarsigner to verify the v1 signature on the reproduced APK, as + # apksigner will reject the reproduced APK if the original also had a v2 + # signature try: - apksigcopier.do_copy(signed_apk, unsigned_apk, tmp_apk, v1_only=v1_only, - exclude=apksigcopier.exclude_meta) - except apksigcopier.APKSigCopierError as e: - logging.info('...NOT verified - {0}'.format(tmp_apk)) - error = 'signature copying failed: {}'.format(str(e)) - result = compare_apks(signed_apk, unsigned_apk, tmp_dir, - os.path.dirname(unsigned_apk)) - if result is not None: - error += '\nComparing reference APK to unsigned APK...\n' + result - return error + verify_jar_signature(tmp_apk) + verified = True + except Exception: + verified = False - if not verify_apk_signature(tmp_apk): - logging.info('...NOT verified - {0}'.format(tmp_apk)) - error = 'verification of APK with copied signature failed' - result = compare_apks(signed_apk, tmp_apk, tmp_dir, - os.path.dirname(unsigned_apk)) - if result is not None: - error += '\nComparing reference APK to APK with copied signature...\n' + result - return error - if clean_up_verified and os.path.exists(tmp_apk): - logging.info(f"...cleaned up {tmp_apk} after successful verification") - os.remove(tmp_apk) + if not verified: + logging.info("...NOT verified - {0}".format(tmp_apk)) + return compare_apks(signed_apk, tmp_apk, tmp_dir, + os.path.dirname(unsigned_apk)) - logging.info('...successfully verified') + logging.info("...successfully verified") return None def verify_jar_signature(jar): - """Verify the signature of a given JAR file. + """Verifies the signature of a given JAR file. jarsigner is very shitty: unsigned JARs pass as "verified"! So this has to turn on -strict then check for result 4, since this does not expect the signature to be from a CA-signed certificate. - Raises - ------ - VerificationException - If the JAR's signature could not be verified. + :raises: VerificationException() if the JAR's signature could not be verified """ + error = _('JAR signature failed to verify: {path}').format(path=jar) try: - output = subprocess.check_output( - [config['jarsigner'], '-strict', '-verify', jar], stderr=subprocess.STDOUT - ) + output = subprocess.check_output([config['jarsigner'], '-strict', '-verify', jar], + stderr=subprocess.STDOUT) raise VerificationException(error + '\n' + output.decode('utf-8')) except subprocess.CalledProcessError as e: if e.returncode == 4: logging.debug(_('JAR signature verified: {path}').format(path=jar)) else: - raise VerificationException(error + '\n' + e.output.decode('utf-8')) from e + raise VerificationException(error + '\n' + e.output.decode('utf-8')) -def verify_deprecated_jar_signature(jar): - """Verify the signature of a given JAR file, allowing deprecated algorithms. +def verify_apk_signature(apk, min_sdk_version=None): + """verify the signature on an APK - index.jar (v0) and index-v1.jar are both signed by MD5/SHA1 by - definition, so this method provides a way to verify those. Also, - apksigner has different deprecation rules than jarsigner, so this - is our current hack to try to represent the apksigner rules when - executing jarsigner. + Try to use apksigner whenever possible since jarsigner is very + shitty: unsigned APKs pass as "verified"! Warning, this does + not work on JARs with apksigner >= 0.7 (build-tools 26.0.1) - jarsigner is very shitty: unsigned JARs pass as "verified"! So - this has to turn on -strict then check for result 4, since this - does not expect the signature to be from a CA-signed certificate. + :returns: boolean whether the APK was verified + """ + if set_command_in_config('apksigner'): + args = [config['apksigner'], 'verify'] + if min_sdk_version: + args += ['--min-sdk-version=' + min_sdk_version] + if options.verbose: + args += ['--verbose'] + try: + output = subprocess.check_output(args + [apk]) + if options.verbose: + logging.debug(apk + ': ' + output.decode('utf-8')) + return True + except subprocess.CalledProcessError as e: + logging.error('\n' + apk + ': ' + e.output.decode('utf-8')) + else: + if not config.get('jarsigner_warning_displayed'): + config['jarsigner_warning_displayed'] = True + logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")) + try: + verify_jar_signature(apk) + return True + except Exception as e: + logging.error(e) + return False - Also used to verify the signature on an archived APK, supporting deprecated - algorithms. + +def verify_old_apk_signature(apk): + """verify the signature on an archived APK, supporting deprecated algorithms F-Droid aims to keep every single binary that it ever published. Therefore, it needs to be able to verify APK signatures that include deprecated/removed @@ -3931,68 +3270,37 @@ def verify_deprecated_jar_signature(jar): file permissions while in use. That should prevent a bad actor from changing the settings during operation. - Raises - ------ - VerificationException - If the JAR's signature could not be verified. + :returns: boolean whether the APK was verified """ - error = _('JAR signature failed to verify: {path}').format(path=jar) - with tempfile.TemporaryDirectory() as tmpdir: - java_security = os.path.join(tmpdir, 'java.security') - with open(java_security, 'w') as fp: - fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024') - os.chmod(java_security, 0o400) - try: - cmd = [ - config['jarsigner'], - '-J-Djava.security.properties=' + java_security, - '-strict', '-verify', jar - ] - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - raise VerificationException(error + '\n' + output.decode('utf-8')) - except subprocess.CalledProcessError as e: - if e.returncode == 4: - logging.debug(_('JAR signature verified: {path}').format(path=jar)) - else: - raise VerificationException(error + '\n' + e.output.decode('utf-8')) from e + _java_security = os.path.join(os.getcwd(), '.java.security') + if os.path.exists(_java_security): + os.remove(_java_security) + with open(_java_security, 'w') as fp: + fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024') + os.chmod(_java_security, 0o400) - -def verify_apk_signature(apk, min_sdk_version=None): - """Verify the signature on an APK. - - Try to use apksigner whenever possible since jarsigner is very - shitty: unsigned APKs pass as "verified"! Warning, this does - not work on JARs with apksigner >= 0.7 (build-tools 26.0.1) - - Returns - ------- - Boolean - whether the APK was verified - """ - if set_command_in_config('apksigner'): - args = [config['apksigner'], 'verify'] - if min_sdk_version: - args += ['--min-sdk-version=' + min_sdk_version] - if options and options.verbose: - args += ['--verbose'] - try: - output = subprocess.check_output(args + [apk]) - if options and options.verbose: - logging.debug(apk + ': ' + output.decode('utf-8')) + try: + cmd = [ + config['jarsigner'], + '-J-Djava.security.properties=' + _java_security, + '-strict', '-verify', apk + ] + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + if e.returncode != 4: + output = e.output + else: + logging.debug(_('JAR signature verified: {path}').format(path=apk)) return True - except subprocess.CalledProcessError as e: - logging.error('\n' + apk + ': ' + e.output.decode('utf-8')) - else: - if not config.get('jarsigner_warning_displayed'): - config['jarsigner_warning_displayed'] = True - logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")) - try: - verify_deprecated_jar_signature(apk) - return True - except Exception as e: - logging.error(e) + finally: + if os.path.exists(_java_security): + os.chmod(_java_security, 0o600) + os.remove(_java_security) + + logging.error(_('Old APK signature failed to verify: {path}').format(path=apk) + + '\n' + output.decode('utf-8')) return False @@ -4000,14 +3308,13 @@ apk_badchars = re.compile('''[/ :;'"]''') def compare_apks(apk1, apk2, tmp_dir, log_dir=None): - """Compare two apks. + """Compare two apks - Returns - ------- - None if the APK content is the same (apart from the signing key), + Returns None if the apk content is the same (apart from the signing key), otherwise a string describing what's different, or what went wrong when trying to do the comparison. """ + if not log_dir: log_dir = tmp_dir @@ -4022,7 +3329,7 @@ def compare_apks(apk1, apk2, tmp_dir, log_dir=None): '--max-report-size', '12345678', '--max-diff-block-lines', '128', '--html', htmlfile, '--text', textfile, absapk1, absapk2]) != 0: - return "Failed to run diffoscope " + apk1 + return("Failed to run diffoscope " + apk1) apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk @@ -4039,21 +3346,19 @@ def compare_apks(apk1, apk2, tmp_dir, log_dir=None): f.extractall(path=os.path.join(apk2dir, 'content')) if set_command_in_config('apktool'): - if subprocess.call( - [config['apktool'], 'd', absapk1, '--output', 'apktool'], cwd=apk1dir - ): - return "Failed to run apktool " + apk1 - if subprocess.call( - [config['apktool'], 'd', absapk2, '--output', 'apktool'], cwd=apk2dir - ): - return "Failed to run apktool " + apk2 + if subprocess.call([config['apktool'], 'd', absapk1, '--output', 'apktool'], + cwd=apk1dir) != 0: + return("Failed to run apktool " + apk1) + if subprocess.call([config['apktool'], 'd', absapk2, '--output', 'apktool'], + cwd=apk2dir) != 0: + return("Failed to run apktool " + apk2) p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False) lines = p.output.splitlines() if len(lines) != 1 or 'META-INF' not in lines[0]: if set_command_in_config('meld'): p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False) - return "Unexpected diff output:\n" + p.output.replace("\r", "^M") + return("Unexpected diff output:\n" + p.output) # since everything verifies, delete the comparison to keep cruft down shutil.rmtree(apk1dir) @@ -4064,12 +3369,11 @@ def compare_apks(apk1, apk2, tmp_dir, log_dir=None): def set_command_in_config(command): - """Try to find specified command in the path, if it hasn't been manually set in config.yml. - - If found, it is added to the config + '''Try to find specified command in the path, if it hasn't been + manually set in config.py. If found, it is added to the config dict. The return value says whether the command is available. - """ + ''' if command in config: return True else: @@ -4081,7 +3385,8 @@ def set_command_in_config(command): def find_command(command): - """Find the full path of a command, or None if it can't be found in the PATH.""" + '''find the full path of a command, or None if it can't be found in the PATH''' + def is_exe(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) @@ -4100,7 +3405,7 @@ def find_command(command): def genpassword(): - """Generate a random password for when generating keys.""" + '''generate a random password for when generating keys''' h = hashlib.sha256() h.update(os.urandom(16)) # salt h.update(socket.getfqdn().encode('utf-8')) @@ -4109,15 +3414,9 @@ def genpassword(): def genkeystore(localconfig): - """Generate a new key with password provided in localconfig and add it to new keystore. - - Parameters - ---------- - localconfig - - Returns - ------- - hexed public key, public key fingerprint + """ + Generate a new key with password provided in :param localconfig and add it to new keystore + :return: hexed public key, public key fingerprint """ logging.info('Generating a new key in "' + localconfig['keystore'] + '"...') keystoredir = os.path.dirname(localconfig['keystore']) @@ -4172,238 +3471,131 @@ def genkeystore(localconfig): def get_cert_fingerprint(pubkey): - """Generate a certificate fingerprint the same way keytool does it (but with slightly different formatting).""" + """ + Generate a certificate fingerprint the same way keytool does it + (but with slightly different formatting) + """ digest = hashlib.sha256(pubkey).digest() ret = [' '.join("%02X" % b for b in bytearray(digest))] return " ".join(ret) -def get_certificate(signature_block_file, signature_file=None): - """Extract a single DER certificate from JAR Signature's "Signature Block File". +def get_certificate(signature_block_file): + """Extracts a DER certificate from JAR Signature's "Signature Block File". - If there is more than one signer certificate, this exits with an - error, unless the signature_file is provided. If that is set, it - will return the certificate that matches the Signature File, for - example, if there is a certificate chain, like TLS does. In the - fdroidserver use cases, there should always be a single signer. - But rarely, some APKs include certificate chains. + :param signature_block_file: file bytes (as string) representing the + certificate, as read directly out of the APK/ZIP - This could be replaced by androguard's APK.get_certificate_der() - provided the cert chain fix was merged there. Maybe in 4.1.2? - https://github.com/androguard/androguard/pull/1038 - - https://docs.oracle.com/en/java/javase/21/docs/specs/man/jarsigner.html#the-signed-jar-file - - Parameters - ---------- - signature_block_file - Bytes representing the PKCS#7 signer certificate and - signature, as read directly out of the JAR/APK, e.g. CERT.RSA. - - signature_file - Bytes representing the manifest signed by the Signature Block - File, e.g. CERT.SF. If this is not given, the assumption is - there will be only a single certificate in - signature_block_file, otherwise it is an error. - - Returns - ------- - A binary representation of the certificate's public key, + :return: A binary representation of the certificate's public key, or None in case of error """ - pkcs7obj = cms.ContentInfo.load(signature_block_file) - certificates = pkcs7obj['content']['certificates'] - if len(certificates) == 1: - return certificates[0].chosen.dump() - elif not signature_file: - logging.error(_('Found multiple Signer Certificates!')) - return - certificate = get_jar_signer_certificate(pkcs7obj, signature_file) - if certificate: - return certificate.chosen.dump() + content = decoder.decode(signature_block_file, asn1Spec=rfc2315.ContentInfo())[0] + if content.getComponentByName('contentType') != rfc2315.signedData: + return None + content = decoder.decode(content.getComponentByName('content'), + asn1Spec=rfc2315.SignedData())[0] + try: + certificates = content.getComponentByName('certificates') + cert = certificates[0].getComponentByName('certificate') + except PyAsn1Error: + logging.error("Certificates not found.") + return None + return encoder.encode(cert) -def _find_matching_certificate(signer_info, certificate): - """Find the certificates that matches signer_info using issuer and serial number. - - https://android.googlesource.com/platform/tools/apksig/+/refs/tags/android-13.0.0_r3/src/main/java/com/android/apksig/internal/apk/v1/V1SchemeVerifier.java#590 - https://android.googlesource.com/platform/tools/apksig/+/refs/tags/android-13.0.0_r3/src/main/java/com/android/apksig/internal/x509/Certificate.java#55 +def load_stats_fdroid_signing_key_fingerprints(): + """Load list of signing-key fingerprints stored by fdroid publish from file. + :returns: list of dictionanryies containing the singing-key fingerprints. """ - certificate_serial = certificate.chosen['tbs_certificate']['serial_number'] - expected_issuer_serial = signer_info['sid'].chosen - return ( - expected_issuer_serial['issuer'] == certificate.chosen.issuer - and expected_issuer_serial['serial_number'] == certificate_serial - ) - - -def get_jar_signer_certificate(pkcs7obj: cms.ContentInfo, signature_file: bytes): - """Return the one certificate in a chain that actually signed the manifest. - - PKCS#7-signed data can include certificate chains for use cases - where an Certificate Authority (CA) is used. Android does not - validate the certificate chain on APK signatures, so neither does - this. - https://android.googlesource.com/platform/tools/apksig/+/refs/tags/android-13.0.0_r3/src/main/java/com/android/apksig/internal/apk/v1/V1SchemeVerifier.java#512 - - Some useful fodder for understanding all this: - https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html - https://technotes.shemyak.com/posts/jar-signature-block-file-format/ - https://docs.oracle.com/en/java/javase/21/docs/specs/man/jarsigner.html#the-signed-jar-file - https://qistoph.blogspot.com/2012/01/manual-verify-pkcs7-signed-data-with.html - - """ - import oscrypto.asymmetric - import oscrypto.errors - - # Android attempts to verify all SignerInfos and then picks the first verified SignerInfo. - first_verified_signer_info = None - first_verified_signer_info_signing_certificate = None - for signer_info in pkcs7obj['content']['signer_infos']: - signature = signer_info['signature'].contents - digest_algorithm = signer_info["digest_algorithm"]["algorithm"].native - public_key = None - for certificate in pkcs7obj['content']['certificates']: - if _find_matching_certificate(signer_info, certificate): - public_key = oscrypto.asymmetric.load_public_key(certificate.chosen.public_key) - break - if public_key is None: - logging.info('No certificate found that matches signer info!') - continue - - signature_algo = signer_info['signature_algorithm'].signature_algo - if signature_algo == 'rsassa_pkcs1v15': - # ASN.1 - 1.2.840.113549.1.1.1 - verify_func = oscrypto.asymmetric.rsa_pkcs1v15_verify - elif signature_algo == 'rsassa_pss': - # ASN.1 - 1.2.840.113549.1.1.10 - verify_func = oscrypto.asymmetric.rsa_pss_verify - elif signature_algo == 'dsa': - # ASN.1 - 1.2.840.10040.4.1 - verify_func = oscrypto.asymmetric.dsa_verify - elif signature_algo == 'ecdsa': - # ASN.1 - 1.2.840.10045.4 - verify_func = oscrypto.asymmetric.ecdsa_verify - else: - logging.error( - 'Unknown signature algorithm %s:\n %s\n %s' - % ( - signature_algo, - hexlify(certificate.chosen.sha256).decode(), - certificate.chosen.subject.human_friendly, - ), - ) - return - - try: - verify_func(public_key, signature, signature_file, digest_algorithm) - if not first_verified_signer_info: - first_verified_signer_info = signer_info - first_verified_signer_info_signing_certificate = certificate - - except oscrypto.errors.SignatureError as e: - logging.error( - '"%s", skipping:\n %s\n %s' % ( - e, - hexlify(certificate.chosen.sha256).decode(), - certificate.chosen.subject.human_friendly), - ) - - if first_verified_signer_info_signing_certificate: - return first_verified_signer_info_signing_certificate - - -def load_publish_signer_fingerprints(): - """Load signing-key fingerprints stored in file generated by fdroid publish. - - Returns - ------- - dict - containing the signing-key fingerprints. - """ - jar_file = os.path.join('repo', 'signer-index.jar') + jar_file = os.path.join('stats', 'publishsigkeys.jar') if not os.path.isfile(jar_file): return {} - try: - verify_deprecated_jar_signature(jar_file) - except VerificationException as e: + cmd = [config['jarsigner'], '-strict', '-verify', jar_file] + p = FDroidPopen(cmd, output=False) + if p.returncode != 4: raise FDroidException("Signature validation of '{}' failed! " - "Please run publish again to rebuild this file.".format(jar_file)) from e + "Please run publish again to rebuild this file.".format(jar_file)) jar_sigkey = apk_signer_fingerprint(jar_file) repo_key_sig = config.get('repo_key_sha256') if repo_key_sig: if jar_sigkey != repo_key_sig: - raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.yml (found fingerprint: '{}')".format(jar_file, jar_sigkey)) + raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey)) else: - logging.warning("repo_key_sha256 not in config.yml, setting it to the signature key fingerprint of '{}'".format(jar_file)) + logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file)) config['repo_key_sha256'] = jar_sigkey write_to_config(config, 'repo_key_sha256') with zipfile.ZipFile(jar_file, 'r') as f: - return json.loads(str(f.read('signer-index.json'), 'utf-8')) + return json.loads(str(f.read('publishsigkeys.json'), 'utf-8')) -def write_config_file(config): - """Write the provided string to config.yml with the right path and encoding.""" - Path(CONFIG_FILE).write_text(config, encoding='utf-8') - - -def write_to_config(thisconfig, key, value=None): - """Write a key/value to the local config.yml. - - The config.yml is defined as YAML 1.2 in UTF-8 encoding on all - platforms. +def write_to_config(thisconfig, key, value=None, config_file=None): + '''write a key/value to the local config.yml or config.py NOTE: only supports writing string variables. - Parameters - ---------- - thisconfig - config dictionary - key - variable name in config to be overwritten/added - value - optional value to be written, instead of fetched + :param thisconfig: config dictionary + :param key: variable name in config to be overwritten/added + :param value: optional value to be written, instead of fetched from 'thisconfig' dictionary. - - """ + ''' if value is None: origkey = key + '_orig' value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key] + if config_file: + cfg = config_file + elif os.path.exists('config.py') and not os.path.exists('config.yml'): + cfg = 'config.py' + else: + cfg = 'config.yml' # load config file, create one if it doesn't exist - if not os.path.exists(CONFIG_FILE): - write_config_file('') - logging.info(_("Creating empty {config_file}").format(config_file=CONFIG_FILE)) - with open(CONFIG_FILE) as fp: - lines = fp.readlines() + if not os.path.exists(cfg): + open(cfg, 'a').close() + logging.info("Creating empty " + cfg) + with open(cfg, 'r') as f: + lines = f.readlines() # make sure the file ends with a carraige return if len(lines) > 0: if not lines[-1].endswith('\n'): lines[-1] += '\n' - pattern = re.compile(r'^[\s#]*' + key + r':.*\n') - repl = config_dump({key: value}) + # regex for finding and replacing python string variable + # definitions/initializations + if cfg.endswith('.py'): + pattern = re.compile(r'^[\s#]*' + key + r'\s*=\s*"[^"]*"') + repl = key + ' = "' + value + '"' + pattern2 = re.compile(r'^[\s#]*' + key + r"\s*=\s*'[^']*'") + repl2 = key + " = '" + value + "'" + else: + # assume .yml as default + pattern = re.compile(r'^[\s#]*' + key + r':.*') + repl = yaml.dump({key: value}, default_flow_style=False) + pattern2 = pattern + repl2 = repl # If we replaced this line once, we make sure won't be a # second instance of this line for this key in the document. didRepl = False # edit config file - with open(CONFIG_FILE, 'w', encoding='utf-8') as f: + with open(cfg, 'w') as f: for line in lines: - if pattern.match(line): + if pattern.match(line) or pattern2.match(line): if not didRepl: line = pattern.sub(repl, line) + line = pattern2.sub(repl2, line) f.write(line) didRepl = True else: f.write(line) if not didRepl: + f.write('\n') f.write(repl) + f.write('\n') def parse_xml(path): @@ -4423,44 +3615,22 @@ def string_is_integer(string): def version_code_string_to_int(vercode): - """Convert an versionCode string of any base into an int.""" - # TODO: Python 3.6 allows underscores in numeric literals - vercode = vercode.replace('_', '') + """Convert an version code string of any base into an int""" try: return int(vercode, 0) except ValueError: return int(vercode) -def get_app_display_name(app): - """Get a human readable name for the app for logging and sorting. - - When trying to find a localized name, this first tries en-US since - that his the historical language used for sorting. - - """ - if app.get('Name'): - return app['Name'] - if app.get('localized'): - localized = app['localized'].get(DEFAULT_LOCALE) - if not localized: - for v in app['localized'].values(): - localized = v - break - if localized.get('name'): - return localized['name'] - return app.get('AutoName') or app['id'] - - -def local_rsync(options, from_paths: List[str], todir: str): - """Rsync method for local to local copying of things. +def local_rsync(options, fromdir, todir): + '''Rsync method for local to local copying of things This is an rsync wrapper with all the settings for safe use within the various fdroidserver use cases. This uses stricter rsync checking on all files since people using offline mode are already prioritizing security above ease and speed. - """ + ''' rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms', '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w'] if not options.no_checksum: @@ -4469,25 +3639,21 @@ def local_rsync(options, from_paths: List[str], todir: str): rsyncargs += ['--verbose'] if options.quiet: rsyncargs += ['--quiet'] - logging.debug(' '.join(rsyncargs + from_paths + [todir])) - if subprocess.call(rsyncargs + from_paths + [todir]) != 0: + logging.debug(' '.join(rsyncargs + [fromdir, todir])) + if subprocess.call(rsyncargs + [fromdir, todir]) != 0: raise FDroidException() def deploy_build_log_with_rsync(appid, vercode, log_content): """Upload build log of one individual app build to an fdroid repository. - Parameters - ---------- - appid - package name for dientifying to which app this log belongs. - vercode - version of the app to which this build belongs. - log_content - Content of the log which is about to be posted. - Should be either a string or bytes. (bytes will - be decoded as 'utf-8') + :param appid: package name for dientifying to which app this log belongs. + :param vercode: version of the app to which this build belongs. + :param log_content: Content of the log which is about to be posted. + Should be either a string or bytes. (bytes will + be decoded as 'utf-8') """ + if not log_content: logging.warning(_('skip deploying full build logs: log content is empty')) return @@ -4509,13 +3675,13 @@ def deploy_build_log_with_rsync(appid, vercode, log_content): def rsync_status_file_to_repo(path, repo_subdir=None): - """Copy a build log or status JSON to the repo using rsync.""" + """Copy a build log or status JSON to the repo using rsync""" + if not config.get('deploy_process_logs', False): logging.debug(_('skip deploying full build logs: not enabled in config')) return - for d in config.get('serverwebroot', []): - webroot = d['url'] + for webroot in config.get('serverwebroot', []): cmd = ['rsync', '--archive', '--delete-after', @@ -4544,7 +3710,8 @@ def rsync_status_file_to_repo(path, repo_subdir=None): def get_per_app_repos(): - """Per-app repos are dirs named with the packageName of a single app.""" + '''per-app repos are dirs named with the packageName of a single app''' + # Android packageNames are Java packages, they may contain uppercase or # lowercase letters ('A' through 'Z'), numbers, and underscores # ('_'). However, individual package name parts may only start with @@ -4565,54 +3732,28 @@ def get_per_app_repos(): return repos -# list of index files that are never gpg-signed -NO_GPG_INDEX_FILES = [ - "entry.jar", - "index-v1.jar", - "index.css", - "index.html", - "index.jar", - "index.png", - "index.xml", - "signer-index.jar", -] - -# list of index files that are signed by gpgsign.py to make a .asc file -GPG_INDEX_FILES = [ - "altstore-index.json", - "entry.json", - "index-v1.json", - "index-v2.json", - "signer-index.json", -] - - -INDEX_FILES = sorted( - NO_GPG_INDEX_FILES + GPG_INDEX_FILES + [i + '.asc' for i in GPG_INDEX_FILES] -) - - -def is_repo_file(filename, for_gpg_signing=False): - """Whether the file in a repo is a build product to be delivered to users.""" +def is_repo_file(filename): + '''Whether the file in a repo is a build product to be delivered to users''' if isinstance(filename, str): filename = filename.encode('utf-8', errors="surrogateescape") - ignore_files = [i.encode() for i in NO_GPG_INDEX_FILES] - ignore_files.append(b'index_unsigned.jar') - if not for_gpg_signing: - ignore_files += [i.encode() for i in GPG_INDEX_FILES] - - return ( - os.path.isfile(filename) - and not filename.endswith(b'.asc') - and not filename.endswith(b'.sig') - and not filename.endswith(b'.idsig') - and not filename.endswith(b'.log.gz') - and os.path.basename(filename) not in ignore_files - ) + return os.path.isfile(filename) \ + and not filename.endswith(b'.asc') \ + and not filename.endswith(b'.sig') \ + and not filename.endswith(b'.idsig') \ + and not filename.endswith(b'.log.gz') \ + and os.path.basename(filename) not in [ + b'index.jar', + b'index_unsigned.jar', + b'index.xml', + b'index.html', + b'index-v1.jar', + b'index-v1.json', + b'categories.txt', + ] def get_examples_dir(): - """Return the dir where the fdroidserver example files are available.""" + '''Return the dir where the fdroidserver example files are available''' examplesdir = None tmp = os.path.dirname(sys.argv[0]) if os.path.basename(tmp) == 'bin': @@ -4634,46 +3775,74 @@ def get_examples_dir(): return examplesdir -def get_android_tools_versions(): - """Get a list of the versions of all installed Android SDK/NDK components.""" +def get_wiki_timestamp(timestamp=None): + """Return current time in the standard format for posting to the wiki""" + + if timestamp is None: + timestamp = time.gmtime() + return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp) + + +def get_android_tools_versions(ndk_path=None): + '''get a list of the versions of all installed Android SDK/NDK components''' + global config sdk_path = config['sdk_path'] if sdk_path[-1] != '/': sdk_path += '/' - components = set() - for ndk_path in config.get('ndk_paths', {}).values(): - version = get_ndk_version(ndk_path) - components.add((os.path.relpath(ndk_path, sdk_path), str(version))) + components = [] + if ndk_path: + ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT') + if os.path.isfile(ndk_release_txt): + with open(ndk_release_txt, 'r') as fp: + components.append((os.path.basename(ndk_path), fp.read()[:-1])) - pattern = re.compile(r'^Pkg.Revision *= *(.+)', re.MULTILINE) + pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE) for root, dirs, files in os.walk(sdk_path): if 'source.properties' in files: source_properties = os.path.join(root, 'source.properties') with open(source_properties, 'r') as fp: m = pattern.search(fp.read()) if m: - components.add((os.path.relpath(root, sdk_path), m.group(1))) + components.append((root[len(sdk_path):], m.group(1))) - return sorted(components) + return components -def get_android_tools_version_log(): - """Get a list of the versions of all installed Android SDK/NDK components.""" +def get_android_tools_version_log(ndk_path=None): + '''get a list of the versions of all installed Android SDK/NDK components''' log = '== Installed Android Tools ==\n\n' - components = get_android_tools_versions() + components = get_android_tools_versions(ndk_path) for name, version in sorted(components): log += '* ' + name + ' (' + version + ')\n' return log +def get_git_describe_link(): + """Get a link to the current fdroiddata commit, to post to the wiki + + """ + try: + output = subprocess.check_output(['git', 'describe', '--always', '--dirty', '--abbrev=0'], + universal_newlines=True).strip() + except subprocess.CalledProcessError: + pass + if output: + commit = output.replace('-dirty', '') + return ('* fdroiddata: [https://gitlab.com/fdroid/fdroiddata/commit/{commit} {id}]\n' + .format(commit=commit, id=output)) + else: + logging.error(_("'{path}' failed to execute!").format(path='git describe')) + return '' + + def calculate_math_string(expr): ops = { ast.Add: operator.add, ast.Mult: operator.mul, ast.Sub: operator.sub, ast.USub: operator.neg, - ast.Pow: operator.pow, } def execute_ast(node): @@ -4691,14 +3860,14 @@ def calculate_math_string(expr): if '#' in expr: raise SyntaxError('no comments allowed') return execute_ast(ast.parse(expr, mode='eval').body) - except SyntaxError as exc: + except SyntaxError: raise SyntaxError("could not parse expression '{expr}', " "only basic math operations are allowed (+, -, *)" - .format(expr=expr)) from exc + .format(expr=expr)) def force_exit(exitvalue=0): - """Force exit when thread operations could block the exit. + """force exit when thread operations could block the exit The build command has to use some threading stuff to handle the timeout and locks. This seems to prevent the command from @@ -4717,7 +3886,7 @@ YAML_LINT_CONFIG = {'extends': 'default', def run_yamllint(path, indent=0): - path = Path(path) + try: import yamllint.config import yamllint.linter @@ -4725,235 +3894,8 @@ def run_yamllint(path, indent=0): return '' result = [] - with path.open('r', encoding='utf-8') as f: + with open(path, 'r', encoding='utf-8') as f: problems = yamllint.linter.run(f, yamllint.config.YamlLintConfig(json.dumps(YAML_LINT_CONFIG))) for problem in problems: - result.append(' ' * indent + str(path) + ':' + str(problem.line) + ': ' + problem.message) + result.append(' ' * indent + path + ':' + str(problem.line) + ': ' + problem.message) return '\n'.join(result) - - -def calculate_IPFS_cid(filename): - """Calculate the IPFS CID of a file and add it to the index. - - uses ipfs_cid package at https://packages.debian.org/sid/ipfs-cid - Returns CIDv1 of a file as per IPFS recommendation - """ - cmd = config and config.get('ipfs_cid') - if not cmd: - return - file_cid = subprocess.run([cmd, filename], capture_output=True) - - if file_cid.returncode == 0: - cid_output = file_cid.stdout.decode() - cid_output_dict = json.loads(cid_output) - return cid_output_dict['CIDv1'] - - -def sha256sum(filename): - """Calculate the sha256 of the given file.""" - sha = hashlib.sha256() - with open(filename, 'rb') as f: - while True: - t = f.read(16384) - if len(t) == 0: - break - sha.update(t) - return sha.hexdigest() - - -def sha256base64(filename): - """Calculate the sha256 of the given file as URL-safe base64.""" - hasher = hashlib.sha256() - with open(filename, 'rb') as f: - while True: - t = f.read(16384) - if len(t) == 0: - break - hasher.update(t) - return urlsafe_b64encode(hasher.digest()).decode() - - -def get_ndk_version(ndk_path): - """Get the version info from the metadata in the NDK package. - - Since r11, the info is nice and easy to find in - sources.properties. Before, there was a kludgey format in - RELEASE.txt. This is only needed for r10e. - - """ - source_properties = os.path.join(ndk_path, 'source.properties') - release_txt = os.path.join(ndk_path, 'RELEASE.TXT') - if os.path.exists(source_properties): - with open(source_properties) as fp: - m = re.search(r'^Pkg.Revision *= *(.+)', fp.read(), flags=re.MULTILINE) - if m: - return m.group(1) - elif os.path.exists(release_txt): - with open(release_txt) as fp: - return fp.read().split('-')[0] - - -def auto_install_ndk(build): - """Auto-install the NDK in the build, this assumes its in a buildserver guest VM. - - Download, verify, and install the NDK version as specified via the - "ndk:" field in the build entry. As it uncompresses the zipball, - this forces the permissions to work for all users, since this - might uncompress as root and then be used from a different user. - - This needs to be able to install multiple versions of the NDK, - since this is also used in CI builds, where multiple `fdroid build - --onserver` calls can run in a single session. The production - buildserver is reset between every build. - - The default ANDROID_SDK_ROOT base dir of /opt/android-sdk is hard-coded in - buildserver/Vagrantfile. The $ANDROID_HOME/ndk subdir is where Android - Studio will install the NDK into versioned subdirs. - https://developer.android.com/studio/projects/configure-agp-ndk#agp_version_41 - - Also, r10e and older cannot be handled via this mechanism because - they are packaged differently. - - """ - import sdkmanager - - global config - if build.get('disable'): - return - ndk = build.get('ndk') - if not ndk: - return - if isinstance(ndk, str): - sdkmanager.build_package_list(use_net=True) - _install_ndk(ndk) - elif isinstance(ndk, list): - sdkmanager.build_package_list(use_net=True) - for n in ndk: - _install_ndk(n) - else: - raise BuildException(_('Invalid ndk: entry in build: "{ndk}"') - .format(ndk=str(ndk))) - - -def _install_ndk(ndk): - """Install specified NDK if it is not already installed. - - Parameters - ---------- - ndk - The NDK version to install, either in "release" form (r21e) or - "revision" form (21.4.7075529). - """ - import sdkmanager - - sdk_path = config['sdk_path'] - sdkmanager.install(f'ndk;{ndk}', sdk_path) - for found in glob.glob(f'{sdk_path}/ndk/*'): - version = get_ndk_version(found) - if 'ndk_paths' not in config: - config['ndk_paths'] = dict() - config['ndk_paths'][ndk] = found - config['ndk_paths'][version] = found - logging.info( - _('Set NDK {release} ({version}) up').format(release=ndk, version=version) - ) - - -def calculate_archive_policy(app, default): - """Calculate the archive policy from the metadata and default config.""" - if app.get('ArchivePolicy') is not None: - archive_policy = app['ArchivePolicy'] - else: - archive_policy = default - if app.get('VercodeOperation'): - archive_policy *= len(app['VercodeOperation']) - builds = [build for build in app.Builds if not build.disable] - if app.Builds and archive_policy > len(builds): - archive_policy = len(builds) - return archive_policy - - -def calculate_gradle_flavor_combination(flavors): - """Calculate all combinations of gradle flavors.""" - combination_lists = itertools.product(*[[flavor, ''] for flavor in flavors]) - combinations = [ - re.sub( - r' +\w', - lambda pat: pat.group(0)[-1].upper(), - ' '.join(combination_list).strip(), - ) - for combination_list in combination_lists - ] - return combinations - - -FDROIDORG_MIRRORS = [ - { - 'isPrimary': True, - 'url': 'https://f-droid.org/repo', - 'dnsA': ['65.21.79.229', '136.243.44.143'], - 'dnsAAAA': ['2a01:4f8:212:c98::2', '2a01:4f9:3b:546d::2'], - 'worksWithoutSNI': True, - }, - { - 'url': 'http://fdroidorg6cooksyluodepej4erfctzk7rrjpjbbr6wx24jh3lqyfwyd.onion/fdroid/repo' - }, - { - 'url': 'http://dotsrccccbidkzg7oc7oj4ugxrlfbt64qebyunxbrgqhxiwj3nl6vcad.onion/fdroid/repo' - }, - { - 'url': 'http://ftpfaudev4triw2vxiwzf4334e3mynz7osqgtozhbc77fixncqzbyoyd.onion/fdroid/repo' - }, - { - 'url': 'http://lysator7eknrfl47rlyxvgeamrv7ucefgrrlhk7rouv3sna25asetwid.onion/pub/fdroid/repo' - }, - { - 'url': 'http://mirror.ossplanetnyou5xifr6liw5vhzwc2g2fmmlohza25wwgnnaw65ytfsad.onion/fdroid/repo' - }, - {'url': 'https://fdroid.tetaneutral.net/fdroid/repo', 'countryCode': 'FR'}, - { - 'url': 'https://ftp.agdsn.de/fdroid/repo', - 'countryCode': 'DE', - "dnsA": ["141.30.235.39"], - "dnsAAAA": ["2a13:dd85:b00:12::1"], - "worksWithoutSNI": True, - }, - { - 'url': 'https://ftp.fau.de/fdroid/repo', - 'countryCode': 'DE', - "dnsA": ["131.188.12.211"], - "dnsAAAA": ["2001:638:a000:1021:21::1"], - "worksWithoutSNI": True, - }, - {'url': 'https://ftp.gwdg.de/pub/android/fdroid/repo', 'countryCode': 'DE'}, - { - 'url': 'https://ftp.lysator.liu.se/pub/fdroid/repo', - 'countryCode': 'SE', - "dnsA": ["130.236.254.251", "130.236.254.253"], - "dnsAAAA": ["2001:6b0:17:f0a0::fb", "2001:6b0:17:f0a0::fd"], - "worksWithoutSNI": True, - }, - {'url': 'https://mirror.cyberbits.eu/fdroid/repo', 'countryCode': 'FR'}, - { - 'url': 'https://mirror.fcix.net/fdroid/repo', - 'countryCode': 'US', - "dnsA": ["23.152.160.16"], - "dnsAAAA": ["2620:13b:0:1000::16"], - "worksWithoutSNI": True, - }, - {'url': 'https://mirror.kumi.systems/fdroid/repo', 'countryCode': 'AT'}, - {'url': 'https://mirror.level66.network/fdroid/repo', 'countryCode': 'DE'}, - {'url': 'https://mirror.ossplanet.net/fdroid/repo', 'countryCode': 'TW'}, - {'url': 'https://mirrors.dotsrc.org/fdroid/repo', 'countryCode': 'DK'}, - {'url': 'https://opencolo.mm.fcix.net/fdroid/repo', 'countryCode': 'US'}, - { - 'url': 'https://plug-mirror.rcac.purdue.edu/fdroid/repo', - 'countryCode': 'US', - "dnsA": ["128.211.151.252"], - "dnsAAAA": ["2001:18e8:804:35::1337"], - "worksWithoutSNI": True, - }, -] -FDROIDORG_FINGERPRINT = ( - '43238D512C1E5EB2D6569F4A3AFBF5523418B82E0A3ED1552770ABB9A9C9CCAB' -) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index f1dcce21..87dfce8b 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -16,366 +16,284 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import configparser -import glob -import json -import logging -import os -import pathlib -import re -import shutil -import subprocess import sys +import glob +import hashlib +import json +import os +import re +import subprocess import time import urllib from argparse import ArgumentParser -from typing import Dict, List +import logging +import shutil -import git -import yaml -from git import Repo - -import fdroidserver.github - -from . import _, common, index +from . import _ +from . import common +from . import index +from . import update from .exception import FDroidException config = None +options = None start_timestamp = time.gmtime() -GIT_BRANCH = 'master' - BINARY_TRANSPARENCY_DIR = 'binary_transparency' +AUTO_S3CFG = '.fdroid-deploy-s3cfg' +USER_S3CFG = 's3cfg' REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*') -EMBEDDED_RCLONE_CONF = 'rclone.conf' +def update_awsbucket(repo_section): + ''' + Upload the contents of the directory `repo_section` (including + subdirectories) to the AWS S3 "bucket". The contents of that subdir of the + bucket will first be deleted. -def _get_index_file_paths(base_dir): - """Return the list of files to be synced last, since they finalize the deploy. + Requires AWS credentials set in config.yml: awsaccesskeyid, awssecretkey + ''' - The process of pushing all the new packages to the various - services can take a while. So the index files should be updated - last. That ensures that the package files are available when the - client learns about them from the new index files. + logging.debug('Syncing "' + repo_section + '" to Amazon S3 bucket "' + + config['awsbucket'] + '"') - signer-index.* are only published in the repo/ section. - - """ - return [ - os.path.join(base_dir, filename) - for filename in common.INDEX_FILES - if not (filename.startswith('signer-index.') and base_dir.endswith('archive')) - ] - - -def _get_index_excludes(base_dir): - indexes = _get_index_file_paths(base_dir) - index_excludes = [] - for f in indexes: - index_excludes.append('--exclude') - index_excludes.append(f) - return index_excludes - - -def _get_index_includes(base_dir): - indexes = _get_index_file_paths(base_dir) - index_includes = [] - for f in indexes: - index_includes.append('--include') - index_includes.append(f) - return index_includes - - -def _remove_missing_files(files: List[str]) -> List[str]: - """Remove files that are missing from the file system.""" - existing = [] - for f in files: - if os.path.exists(f): - existing.append(f) - return existing - - -def _generate_rclone_include_pattern(files): - """Generate a pattern for rclone's --include flag (https://rclone.org/filtering/).""" - return "{" + ",".join(sorted(set(files))) + "}" - - -def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=False): - """Sync the directory `repo_section` (including subdirectories) to AWS S3 US East. - - This is a shim function for public API compatibility. - - Requires AWS credentials set as environment variables: - https://rclone.org/s3/#authentication - - """ - update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet) - - -def update_remote_storage_with_rclone( - repo_section, - awsbucket, - is_index_only=False, - verbose=False, - quiet=False, - checksum=False, -): - """Sync the directory `repo_section` (including subdirectories) to configed cloud services. - - Rclone sync can send the files to any supported remote storage - service once without numerous polling. If remote storage is S3 e.g - AWS S3, Wasabi, Filebase, etc, then path will be - bucket_name/fdroid/repo where bucket_name will be an S3 bucket. If - remote storage is storage drive/sftp e.g google drive, rsync.net the - new path will be bucket_name/fdroid/repo where bucket_name will be a - folder - - See https://rclone.org/docs/#config-config-file - - rclone filtering works differently than rsync. For example, - "--include" implies "--exclude **" at the end of an rclone internal - filter list. - - If rclone.conf is in the root of the repo, then it will be preferred - over the rclone default config paths. - - """ - logging.debug(_('Using rclone to sync to "{name}"').format(name=awsbucket)) - - rclone_config = config.get('rclone_config', []) - if rclone_config and isinstance(rclone_config, str): - rclone_config = [rclone_config] - - path = config.get('path_to_custom_rclone_config') - if path: - if not os.path.exists(path): - logging.error( - _('path_to_custom_rclone_config: "{path}" does not exist!').format( - path=path - ) - ) - sys.exit(1) - configfilename = path - elif os.path.exists(EMBEDDED_RCLONE_CONF): - path = EMBEDDED_RCLONE_CONF # in this case, only for display - configfilename = EMBEDDED_RCLONE_CONF - if not rclone_config: - raise FDroidException(_("'rclone_config' must be set in config.yml!")) + if common.set_command_in_config('s3cmd'): + update_awsbucket_s3cmd(repo_section) else: - configfilename = None - output = subprocess.check_output(['rclone', 'config', 'file'], text=True) - default_config_path = output.split('\n')[-2] - if os.path.exists(default_config_path): - path = default_config_path - if path: - logging.info(_('Using "{path}" for rclone config.').format(path=path)) + update_awsbucket_libcloud(repo_section) + + +def update_awsbucket_s3cmd(repo_section): + '''upload using the CLI tool s3cmd, which provides rsync-like sync + + The upload is done in multiple passes to reduce the chance of + interfering with an existing client-server interaction. In the + first pass, only new files are uploaded. In the second pass, + changed files are uploaded, overwriting what is on the server. On + the third/last pass, the indexes are uploaded, and any removed + files are deleted from the server. The last pass is the only pass + to use a full MD5 checksum of all files to detect changes. + ''' + + logging.debug(_('Using s3cmd to sync with: {url}') + .format(url=config['awsbucket'])) + + if os.path.exists(USER_S3CFG): + logging.info(_('Using "{path}" for configuring s3cmd.').format(path=USER_S3CFG)) + configfilename = USER_S3CFG + else: + fd = os.open(AUTO_S3CFG, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600) + logging.debug(_('Creating "{path}" for configuring s3cmd.').format(path=AUTO_S3CFG)) + os.write(fd, '[default]\n'.encode('utf-8')) + os.write(fd, ('access_key = ' + config['awsaccesskeyid'] + '\n').encode('utf-8')) + os.write(fd, ('secret_key = ' + config['awssecretkey'] + '\n').encode('utf-8')) + os.close(fd) + configfilename = AUTO_S3CFG + + s3bucketurl = 's3://' + config['awsbucket'] + s3cmd = [config['s3cmd'], '--config=' + configfilename] + if subprocess.call(s3cmd + ['info', s3bucketurl]) != 0: + logging.warning(_('Creating new S3 bucket: {url}') + .format(url=s3bucketurl)) + if subprocess.call(s3cmd + ['mb', s3bucketurl]) != 0: + logging.error(_('Failed to create S3 bucket: {url}') + .format(url=s3bucketurl)) + raise FDroidException() + + s3cmd_sync = s3cmd + ['sync', '--acl-public'] + if options.verbose: + s3cmd_sync += ['--verbose'] + if options.quiet: + s3cmd_sync += ['--quiet'] + indexxml = os.path.join(repo_section, 'index.xml') + indexjar = os.path.join(repo_section, 'index.jar') + indexv1jar = os.path.join(repo_section, 'index-v1.jar') + + s3url = s3bucketurl + '/fdroid/' + logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url) + logging.debug(_('Running first pass with MD5 checking disabled')) + if subprocess.call(s3cmd_sync + + ['--no-check-md5', '--skip-existing', + '--exclude', indexxml, + '--exclude', indexjar, + '--exclude', indexv1jar, + repo_section, s3url]) != 0: + raise FDroidException() + logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url) + if subprocess.call(s3cmd_sync + + ['--no-check-md5', + '--exclude', indexxml, + '--exclude', indexjar, + '--exclude', indexv1jar, + repo_section, s3url]) != 0: + raise FDroidException() + + logging.debug(_('s3cmd sync indexes {path} to {url} and delete') + .format(path=repo_section, url=s3url)) + s3cmd_sync.append('--delete-removed') + s3cmd_sync.append('--delete-after') + if options.no_checksum: + s3cmd_sync.append('--no-check-md5') + else: + s3cmd_sync.append('--check-md5') + if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0: + raise FDroidException() + + +def update_awsbucket_libcloud(repo_section): + ''' + Upload the contents of the directory `repo_section` (including + subdirectories) to the AWS S3 "bucket". The contents of that subdir of the + bucket will first be deleted. + + Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey + ''' + + logging.debug(_('using Apache libcloud to sync with {url}') + .format(url=config['awsbucket'])) + + import libcloud.security + libcloud.security.VERIFY_SSL_CERT = True + from libcloud.storage.types import Provider, ContainerDoesNotExistError + from libcloud.storage.providers import get_driver + + if not config.get('awsaccesskeyid') or not config.get('awssecretkey'): + raise FDroidException( + _('To use awsbucket, awssecretkey and awsaccesskeyid must also be set in config.yml!')) + awsbucket = config['awsbucket'] + + if os.path.exists(USER_S3CFG): + raise FDroidException(_('"{path}" exists but s3cmd is not installed!') + .format(path=USER_S3CFG)) + + cls = get_driver(Provider.S3) + driver = cls(config['awsaccesskeyid'], config['awssecretkey']) + try: + container = driver.get_container(container_name=awsbucket) + except ContainerDoesNotExistError: + container = driver.create_container(container_name=awsbucket) + logging.info(_('Created new container "{name}"') + .format(name=container.name)) upload_dir = 'fdroid/' + repo_section + objs = dict() + for obj in container.list_objects(): + if obj.name.startswith(upload_dir + '/'): + objs[obj.name] = obj - if not rclone_config: - env = os.environ - # Check both canonical and backup names, but only tell user about canonical. - if not env.get("AWS_SECRET_ACCESS_KEY") and not env.get("AWS_SECRET_KEY"): - raise FDroidException( - _( - """"AWS_SECRET_ACCESS_KEY" must be set as an environmental variable!""" - ) - ) - if not env.get("AWS_ACCESS_KEY_ID") and not env.get('AWS_ACCESS_KEY'): - raise FDroidException( - _(""""AWS_ACCESS_KEY_ID" must be set as an environmental variable!""") - ) + for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)): + for name in files: + upload = False + file_to_upload = os.path.join(root, name) + object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd()) + if object_name not in objs: + upload = True + else: + obj = objs.pop(object_name) + if obj.size != os.path.getsize(file_to_upload): + upload = True + else: + # if the sizes match, then compare by MD5 + md5 = hashlib.md5() # nosec AWS uses MD5 + with open(file_to_upload, 'rb') as f: + while True: + data = f.read(8192) + if not data: + break + md5.update(data) + if obj.hash != md5.hexdigest(): + s3url = 's3://' + awsbucket + '/' + obj.name + logging.info(' deleting ' + s3url) + if not driver.delete_object(obj): + logging.warning('Could not delete ' + s3url) + upload = True - default_remote = "AWS-S3-US-East-1" - env_rclone_config = configparser.ConfigParser() - env_rclone_config.add_section(default_remote) - env_rclone_config.set( - default_remote, - '; = This file is auto-generated by fdroid deploy, do not edit!', - '', - ) - env_rclone_config.set(default_remote, "type", "s3") - env_rclone_config.set(default_remote, "provider", "AWS") - env_rclone_config.set(default_remote, "region", "us-east-1") - env_rclone_config.set(default_remote, "env_auth", "true") - - configfilename = ".fdroid-deploy-rclone.conf" - with open(configfilename, "w", encoding="utf-8") as autoconfigfile: - env_rclone_config.write(autoconfigfile) - rclone_config = [default_remote] - - rclone_sync_command = ['rclone', 'sync', '--delete-after'] - if configfilename: - rclone_sync_command += ['--config', configfilename] - - if checksum: - rclone_sync_command.append('--checksum') - - if verbose: - rclone_sync_command += ['--verbose'] - elif quiet: - rclone_sync_command += ['--quiet'] - - # TODO copying update_serverwebroot rsync algo - for remote_config in rclone_config: - complete_remote_path = f'{remote_config}:{awsbucket}/{upload_dir}' - logging.info(f'rclone sync to {complete_remote_path}') - if is_index_only: - index_only_files = common.INDEX_FILES + ['diff/*.*'] - include_pattern = _generate_rclone_include_pattern(index_only_files) - cmd = rclone_sync_command + [ - '--include', - include_pattern, - '--delete-excluded', - repo_section, - complete_remote_path, - ] - logging.info(cmd) - if subprocess.call(cmd) != 0: - raise FDroidException() + if upload: + logging.debug(' uploading "' + file_to_upload + '"...') + extra = {'acl': 'public-read'} + if file_to_upload.endswith('.sig'): + extra['content_type'] = 'application/pgp-signature' + elif file_to_upload.endswith('.asc'): + extra['content_type'] = 'application/pgp-signature' + logging.info(' uploading ' + os.path.relpath(file_to_upload) + + ' to s3://' + awsbucket + '/' + object_name) + with open(file_to_upload, 'rb') as iterator: + obj = driver.upload_object_via_stream(iterator=iterator, + container=container, + object_name=object_name, + extra=extra) + # delete the remnants in the bucket, they do not exist locally + while objs: + object_name, obj = objs.popitem() + s3url = 's3://' + awsbucket + '/' + object_name + if object_name.startswith(upload_dir): + logging.warning(' deleting ' + s3url) + driver.delete_object(obj) else: - cmd = ( - rclone_sync_command - + _get_index_excludes(repo_section) - + [ - repo_section, - complete_remote_path, - ] - ) - if subprocess.call(cmd) != 0: - raise FDroidException() - cmd = rclone_sync_command + [ - repo_section, - complete_remote_path, - ] - if subprocess.call(cmd) != 0: - raise FDroidException() + logging.info(' skipping ' + s3url) def update_serverwebroot(serverwebroot, repo_section): - """Deploy the index files to the serverwebroot using rsync. - - Upload the first time without the index files and delay the - deletion as much as possible. That keeps the repo functional - while this update is running. Then once it is complete, rerun the - command again to upload the index files. Always using the same - target with rsync allows for very strict settings on the receiving - server, you can literally specify the one rsync command that is - allowed to run in ~/.ssh/authorized_keys. (serverwebroot is - guaranteed to have a trailing slash in common.py) - - It is possible to optionally use a checksum comparison for - accurate comparisons on different filesystems, for example, FAT - has a low resolution timestamp - - """ - try: - subprocess.run(['rsync', '--version'], capture_output=True, check=True) - except Exception as e: - raise FDroidException( - _('rsync is missing or broken: {error}').format(error=e) - ) from e + # use a checksum comparison for accurate comparisons on different + # filesystems, for example, FAT has a low resolution timestamp rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links'] - options = common.get_options() - if not options or not options.no_checksum: + if not options.no_checksum: rsyncargs.append('--checksum') - if options and options.verbose: + if options.verbose: rsyncargs += ['--verbose'] - if options and options.quiet: + if options.quiet: rsyncargs += ['--quiet'] - if options and options.identity_file: - rsyncargs += [ - '-e', - 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file, - ] - elif config and config.get('identity_file'): - rsyncargs += [ - '-e', - 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file'], - ] - url = serverwebroot['url'] - is_index_only = serverwebroot.get('index_only', False) - logging.info('rsyncing ' + repo_section + ' to ' + url) - if is_index_only: - files_to_upload = _get_index_file_paths(repo_section) - files_to_upload = _remove_missing_files(files_to_upload) - - rsyncargs += files_to_upload - rsyncargs += [f'{url}/{repo_section}/'] - logging.info(rsyncargs) - if subprocess.call(rsyncargs) != 0: - raise FDroidException() - else: - excludes = _get_index_excludes(repo_section) - if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0: - raise FDroidException() - if subprocess.call(rsyncargs + [repo_section, url]) != 0: - raise FDroidException() - # upload "current version" symlinks if requested - if ( - config - and config.get('make_current_version_link') - and repo_section == 'repo' - ): - links_to_upload = [] - for f in ( - glob.glob('*.apk') + glob.glob('*.apk.asc') + glob.glob('*.apk.sig') - ): - if os.path.islink(f): - links_to_upload.append(f) - if len(links_to_upload) > 0: - if subprocess.call(rsyncargs + links_to_upload + [url]) != 0: - raise FDroidException() - - -def update_serverwebroots(serverwebroots, repo_section, standardwebroot=True): - for d in serverwebroots: - # this supports both an ssh host:path and just a path - serverwebroot = d['url'] - s = serverwebroot.rstrip('/').split(':') - if len(s) == 1: - fdroiddir = s[0] - elif len(s) == 2: - host, fdroiddir = s - else: - logging.error(_('Malformed serverwebroot line:') + ' ' + serverwebroot) - sys.exit(1) - repobase = os.path.basename(fdroiddir) - if standardwebroot and repobase != 'fdroid': - logging.error( - _( - 'serverwebroot: path does not end with "fdroid", perhaps you meant one of these:' - ) - + '\n\t' - + serverwebroot.rstrip('/') - + '/fdroid\n\t' - + serverwebroot.rstrip('/').rstrip(repobase) - + 'fdroid' - ) - sys.exit(1) - update_serverwebroot(d, repo_section) + if options.identity_file is not None: + rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file] + elif 'identity_file' in config: + rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']] + indexxml = os.path.join(repo_section, 'index.xml') + indexjar = os.path.join(repo_section, 'index.jar') + indexv1jar = os.path.join(repo_section, 'index-v1.jar') + # Upload the first time without the index files and delay the deletion as + # much as possible, that keeps the repo functional while this update is + # running. Then once it is complete, rerun the command again to upload + # the index files. Always using the same target with rsync allows for + # very strict settings on the receiving server, you can literally specify + # the one rsync command that is allowed to run in ~/.ssh/authorized_keys. + # (serverwebroot is guaranteed to have a trailing slash in common.py) + logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot) + if subprocess.call(rsyncargs + + ['--exclude', indexxml, + '--exclude', indexjar, + '--exclude', indexv1jar, + repo_section, serverwebroot]) != 0: + raise FDroidException() + if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0: + raise FDroidException() + # upload "current version" symlinks if requested + if config['make_current_version_link'] and repo_section == 'repo': + links_to_upload = [] + for f in glob.glob('*.apk') \ + + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'): + if os.path.islink(f): + links_to_upload.append(f) + if len(links_to_upload) > 0: + if subprocess.call(rsyncargs + links_to_upload + [serverwebroot]) != 0: + raise FDroidException() def sync_from_localcopy(repo_section, local_copy_dir): - """Sync the repo from "local copy dir" filesystem to this box. + '''Syncs the repo from "local copy dir" filesystem to this box In setups that use offline signing, this is the last step that syncs the repo from the "local copy dir" e.g. a thumb drive to the repo on the local filesystem. That local repo is then used to push to all the servers that are configured. - """ + ''' logging.info('Syncing from local_copy_dir to this repo.') - # trailing slashes have a meaning in rsync which is not needed here, so # make sure both paths have exactly one trailing slash - common.local_rsync( - common.get_options(), - [os.path.join(local_copy_dir, repo_section).rstrip('/') + '/'], - repo_section.rstrip('/') + '/', - ) + common.local_rsync(options, + os.path.join(local_copy_dir, repo_section).rstrip('/') + '/', + repo_section.rstrip('/') + '/') offline_copy = os.path.join(local_copy_dir, BINARY_TRANSPARENCY_DIR) if os.path.exists(os.path.join(offline_copy, '.git')): @@ -384,15 +302,15 @@ def sync_from_localcopy(repo_section, local_copy_dir): def update_localcopy(repo_section, local_copy_dir): - """Copy data from offline to the "local copy dir" filesystem. + '''copy data from offline to the "local copy dir" filesystem This updates the copy of this repo used to shuttle data from an offline signing machine to the online machine, e.g. on a thumb drive. - """ + ''' # local_copy_dir is guaranteed to have a trailing slash in main() below - common.local_rsync(common.get_options(), [repo_section], local_copy_dir) + common.local_rsync(options, repo_section, local_copy_dir) offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR) if os.path.isdir(os.path.join(offline_copy, '.git')): @@ -401,7 +319,7 @@ def update_localcopy(repo_section, local_copy_dir): def _get_size(start_path='.'): - """Get size of all files in a dir https://stackoverflow.com/a/1392549.""" + '''get size of all files in a dir https://stackoverflow.com/a/1392549''' total_size = 0 for root, dirs, files in os.walk(start_path): for f in files: @@ -411,7 +329,7 @@ def _get_size(start_path='.'): def update_servergitmirrors(servergitmirrors, repo_section): - """Update repo mirrors stored in git repos. + '''update repo mirrors stored in git repos This is a hack to use public git repos as F-Droid repos. It recreates the git repo from scratch each time, so that there is no @@ -421,31 +339,19 @@ def update_servergitmirrors(servergitmirrors, repo_section): For history, there is the archive section, and there is the binary transparency log. - This will attempt to use the existing remote branch so that it does - not have to push all of the files in the repo each time. Old setups - or runs of `fdroid nightly` might use the "master" branch. For the - "index only" mode, it will recreate the branch from scratch each - time since usually all the files are changed. In any case, the - index files are small compared to the full repo. - - """ + ''' + import git from clint.textui import progress - - if config.get('local_copy_dir') and not config.get('sync_from_local_copy_dir'): - logging.debug( - _('Offline machine, skipping git mirror generation until `fdroid deploy`') - ) + if config.get('local_copy_dir') \ + and not config.get('sync_from_local_copy_dir'): + logging.debug(_('Offline machine, skipping git mirror generation until `fdroid deploy`')) return - options = common.get_options() - workspace_dir = pathlib.Path(os.getcwd()) - # right now we support only 'repo' git-mirroring if repo_section == 'repo': - git_mirror_path = workspace_dir / 'git-mirror' + git_mirror_path = 'git-mirror' dotgit = os.path.join(git_mirror_path, '.git') - git_fdroiddir = os.path.join(git_mirror_path, 'fdroid') - git_repodir = os.path.join(git_fdroiddir, repo_section) + git_repodir = os.path.join(git_mirror_path, 'fdroid', repo_section) if not os.path.isdir(git_repodir): os.makedirs(git_repodir) # github/gitlab use bare git repos, so only count the .git folder @@ -454,21 +360,20 @@ def update_servergitmirrors(servergitmirrors, repo_section): dotgit_size = _get_size(dotgit) dotgit_over_limit = dotgit_size > config['git_mirror_size_limit'] if os.path.isdir(dotgit) and dotgit_over_limit: - logging.warning( - _( - 'Deleting git-mirror history, repo is too big ({size} max {limit})' - ).format(size=dotgit_size, limit=config['git_mirror_size_limit']) - ) + logging.warning(_('Deleting git-mirror history, repo is too big ({size} max {limit})') + .format(size=dotgit_size, limit=config['git_mirror_size_limit'])) shutil.rmtree(dotgit) if options.no_keep_git_mirror_archive and dotgit_over_limit: - logging.warning( - _('Deleting archive, repo is too big ({size} max {limit})').format( - size=dotgit_size, limit=config['git_mirror_size_limit'] - ) - ) + logging.warning(_('Deleting archive, repo is too big ({size} max {limit})') + .format(size=dotgit_size, limit=config['git_mirror_size_limit'])) archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive') shutil.rmtree(archive_path, ignore_errors=True) + # rsync is very particular about trailing slashes + common.local_rsync(options, + repo_section.rstrip('/') + '/', + git_repodir.rstrip('/') + '/') + # use custom SSH command if identity_file specified ssh_cmd = 'ssh -oBatchMode=yes' if options.identity_file is not None: @@ -476,174 +381,85 @@ def update_servergitmirrors(servergitmirrors, repo_section): elif 'identity_file' in config: ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file'] + repo = git.Repo.init(git_mirror_path) + + enabled_remotes = [] + for remote_url in servergitmirrors: + name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) + enabled_remotes.append(name) + r = git.remote.Remote(repo, name) + if r in repo.remotes: + r = repo.remote(name) + if 'set_url' in dir(r): # force remote URL if using GitPython 2.x + r.set_url(remote_url) + else: + repo.create_remote(name, remote_url) + logging.info('Mirroring to: ' + remote_url) + + # sadly index.add don't allow the --all parameter + logging.debug('Adding all files to git mirror') + repo.git.add(all=True) + logging.debug('Committing all files into git mirror') + repo.index.commit("fdroidserver git-mirror") + if options.verbose: - progressbar = progress.Bar() + bar = progress.Bar() class MyProgressPrinter(git.RemoteProgress): def update(self, op_code, current, maximum=None, message=None): if isinstance(maximum, float): - progressbar.show(current, maximum) - + bar.show(current, maximum) progress = MyProgressPrinter() else: progress = None - repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH) + # push for every remote. This will overwrite the git history + for remote in repo.remotes: + if remote.name not in enabled_remotes: + repo.delete_remote(remote) + continue + if remote.name == 'gitlab': + logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages') + with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as out_file: + out_file.write("""pages: + script: + - mkdir .public + - cp -r * .public/ + - mv .public public + artifacts: + paths: + - public +""") - enabled_remotes = [] - for d in servergitmirrors: - is_index_only = d.get('index_only', False) + repo.git.add(all=True) + repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages") - # Use a separate branch for the index only mode as it needs a different set of files to commit - if is_index_only: - local_branch_name = 'index_only' - else: - local_branch_name = GIT_BRANCH - if local_branch_name in repo.heads: - repo.git.switch(local_branch_name) - else: - repo.git.switch('--orphan', local_branch_name) + logging.debug(_('Pushing to {url}').format(url=remote.url)) + with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): + pushinfos = remote.push('master', force=True, set_upstream=True, progress=progress) + for pushinfo in pushinfos: + if pushinfo.flags & (git.remote.PushInfo.ERROR + | git.remote.PushInfo.REJECTED + | git.remote.PushInfo.REMOTE_FAILURE + | git.remote.PushInfo.REMOTE_REJECTED): + # Show potentially useful messages from git remote + for line in progress.other_lines: + if line.startswith('remote:'): + logging.debug(line) + raise FDroidException(remote.url + ' push failed: ' + str(pushinfo.flags) + + ' ' + pushinfo.summary) + else: + logging.debug(remote.url + ': ' + pushinfo.summary) - # trailing slashes have a meaning in rsync which is not needed here, so - # make sure both paths have exactly one trailing slash - if is_index_only: - files_to_sync = _get_index_file_paths(str(workspace_dir / repo_section)) - files_to_sync = _remove_missing_files(files_to_sync) - else: - files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/'] - common.local_rsync( - common.get_options(), files_to_sync, git_repodir.rstrip('/') + '/' - ) - - upload_to_servergitmirror( - mirror_config=d, - local_repo=repo, - enabled_remotes=enabled_remotes, - repo_section=repo_section, - is_index_only=is_index_only, - fdroid_dir=git_fdroiddir, - git_mirror_path=str(git_mirror_path), - ssh_cmd=ssh_cmd, - progress=progress, - ) if progress: - progressbar.done() - - -def upload_to_servergitmirror( - mirror_config: Dict[str, str], - local_repo: Repo, - enabled_remotes: List[str], - repo_section: str, - is_index_only: bool, - fdroid_dir: str, - git_mirror_path: str, - ssh_cmd: str, - progress: git.RemoteProgress, -) -> None: - remote_branch_name = GIT_BRANCH - local_branch_name = local_repo.active_branch.name - - remote_url = mirror_config['url'] - name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) - enabled_remotes.append(name) - r = git.remote.Remote(local_repo, name) - if r in local_repo.remotes: - r = local_repo.remote(name) - if 'set_url' in dir(r): # force remote URL if using GitPython 2.x - r.set_url(remote_url) - else: - local_repo.create_remote(name, remote_url) - logging.info('Mirroring to: ' + remote_url) - - if is_index_only: - files_to_upload = _get_index_file_paths( - os.path.join(local_repo.working_tree_dir, 'fdroid', repo_section) - ) - files_to_upload = _remove_missing_files(files_to_upload) - local_repo.index.add(files_to_upload) - else: - # sadly index.add don't allow the --all parameter - logging.debug('Adding all files to git mirror') - local_repo.git.add(all=True) - - logging.debug('Committing files into git mirror') - local_repo.index.commit("fdroidserver git-mirror") - - # only deploy to GitLab Artifacts if too big for GitLab Pages - if ( - is_index_only - or common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE - ): - gitlab_ci_job_name = 'pages' - else: - gitlab_ci_job_name = 'GitLab Artifacts' - logging.warning( - _('Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!') - % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) - ) - - # push. This will overwrite the git history - remote = local_repo.remote(name) - if remote.name == 'gitlab': - logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages') - with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp: - yaml.dump( - { - gitlab_ci_job_name: { - 'script': [ - 'mkdir .public', - 'cp -r * .public/', - 'mv .public public', - ], - 'artifacts': {'paths': ['public']}, - 'variables': {'GIT_DEPTH': 1}, - } - }, - fp, - default_flow_style=False, - ) - - local_repo.index.add(['.gitlab-ci.yml']) - local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages") - - logging.debug(_('Pushing to {url}').format(url=remote.url)) - with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): - pushinfos = remote.push( - f"{local_branch_name}:{remote_branch_name}", - force=True, - set_upstream=True, - progress=progress, - ) - for pushinfo in pushinfos: - if pushinfo.flags & ( - git.remote.PushInfo.ERROR - | git.remote.PushInfo.REJECTED - | git.remote.PushInfo.REMOTE_FAILURE - | git.remote.PushInfo.REMOTE_REJECTED - ): - # Show potentially useful messages from git remote - if progress: - for line in progress.other_lines: - if line.startswith('remote:'): - logging.debug(line) - raise FDroidException( - remote.url - + ' push failed: ' - + str(pushinfo.flags) - + ' ' - + pushinfo.summary - ) - else: - logging.debug(remote.url + ': ' + pushinfo.summary) + bar.done() def upload_to_android_observatory(repo_section): import requests - requests # stop unused import warning - if common.get_options().verbose: + if options.verbose: logging.getLogger("requests").setLevel(logging.INFO) logging.getLogger("urllib3").setLevel(logging.INFO) else: @@ -658,17 +474,13 @@ def upload_to_android_observatory(repo_section): def upload_apk_to_android_observatory(path): # depend on requests and lxml only if users enable AO import requests + from . import net from lxml.html import fromstring - from . import net - apkfilename = os.path.basename(path) - r = requests.post( - 'https://androidobservatory.org/', - data={'q': common.sha256sum(path), 'searchby': 'hash'}, - headers=net.HEADERS, - timeout=300, - ) + r = requests.post('https://androidobservatory.org/', + data={'q': update.sha256sum(path), 'searchby': 'hash'}, + headers=net.HEADERS) if r.status_code == 200: # from now on XPath will be used to retrieve the message in the HTML # androidobservatory doesn't have a nice API to talk with @@ -685,30 +497,22 @@ def upload_apk_to_android_observatory(path): page = 'https://androidobservatory.org' if href: - message = _('Found {apkfilename} at {url}').format( - apkfilename=apkfilename, url=(page + href) - ) + message = (_('Found {apkfilename} at {url}') + .format(apkfilename=apkfilename, url=(page + href))) logging.debug(message) return # upload the file with a post request - logging.info( - _('Uploading {apkfilename} to androidobservatory.org').format( - apkfilename=apkfilename - ) - ) - r = requests.post( - 'https://androidobservatory.org/upload', - files={'apk': (apkfilename, open(path, 'rb'))}, - headers=net.HEADERS, - allow_redirects=False, - timeout=300, - ) + logging.info(_('Uploading {apkfilename} to androidobservatory.org') + .format(apkfilename=apkfilename)) + r = requests.post('https://androidobservatory.org/upload', + files={'apk': (apkfilename, open(path, 'rb'))}, + headers=net.HEADERS, + allow_redirects=False) def upload_to_virustotal(repo_section, virustotal_apikey): import requests - requests # stop unused import warning if repo_section == 'repo': @@ -719,47 +523,44 @@ def upload_to_virustotal(repo_section, virustotal_apikey): with open(os.path.join(repo_section, 'index-v1.json')) as fp: data = json.load(fp) else: - local_jar = os.path.join(repo_section, 'index-v1.jar') - data, _ignored, _ignored = index.get_index_from_jar(local_jar) + data, _ignored, _ignored = index.get_index_from_jar(os.path.join(repo_section, 'index-v1.jar')) for packageName, packages in data['packages'].items(): for package in packages: upload_apk_to_virustotal(virustotal_apikey, **package) -def upload_apk_to_virustotal( - virustotal_apikey, packageName, apkName, hash, versionCode, **kwargs -): +def upload_apk_to_virustotal(virustotal_apikey, packageName, apkName, hash, + versionCode, **kwargs): import requests logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("requests").setLevel(logging.WARNING) - outputfilename = os.path.join( - 'virustotal', packageName + '_' + str(versionCode) + '_' + hash + '.json' - ) + outputfilename = os.path.join('virustotal', + packageName + '_' + str(versionCode) + + '_' + hash + '.json') if os.path.exists(outputfilename): logging.debug(apkName + ' results are in ' + outputfilename) return outputfilename repofilename = os.path.join('repo', apkName) logging.info('Checking if ' + repofilename + ' is on virustotal') - headers = {"User-Agent": "F-Droid"} + headers = { + "User-Agent": "F-Droid" + } if 'headers' in kwargs: for k, v in kwargs['headers'].items(): headers[k] = v - apikey = { + data = { 'apikey': virustotal_apikey, 'resource': hash, } needs_file_upload = False while True: - report_url = ( - 'https://www.virustotal.com/vtapi/v2/file/report?' - + urllib.parse.urlencode(apikey) - ) - r = requests.get(report_url, headers=headers, timeout=300) + r = requests.get('https://www.virustotal.com/vtapi/v2/file/report?' + + urllib.parse.urlencode(data), headers=headers) if r.status_code == 200: response = r.json() if response['response_code'] == 0: @@ -774,14 +575,11 @@ def upload_apk_to_virustotal( json.dump(response, fp, indent=2, sort_keys=True) if response.get('positives', 0) > 0: - logging.warning( - _('{path} has been flagged by virustotal {count} times:').format( - path=repofilename, count=response['positives'] - ), - +'\n\t' + response['permalink'], - ) + logging.warning(repofilename + ' has been flagged by virustotal ' + + str(response['positives']) + ' times:' + + '\n\t' + response['permalink']) break - if r.status_code == 204: + elif r.status_code == 204: logging.warning(_('virustotal.com is rate limiting, waiting to retry...')) time.sleep(30) # wait for public API rate limiting @@ -791,48 +589,32 @@ def upload_apk_to_virustotal( size = os.path.getsize(repofilename) if size > 200000000: # VirusTotal API 200MB hard limit - logging.error( - _('{path} more than 200MB, manually upload: {url}').format( - path=repofilename, url=manual_url - ) - ) + logging.error(_('{path} more than 200MB, manually upload: {url}') + .format(path=repofilename, url=manual_url)) elif size > 32000000: # VirusTotal API requires fetching a URL to upload bigger files - query_url = ( - 'https://www.virustotal.com/vtapi/v2/file/scan/upload_url?' - + urllib.parse.urlencode(apikey) - ) - r = requests.get(query_url, headers=headers, timeout=300) + r = requests.get('https://www.virustotal.com/vtapi/v2/file/scan/upload_url?' + + urllib.parse.urlencode(data), headers=headers) if r.status_code == 200: upload_url = r.json().get('upload_url') elif r.status_code == 403: - logging.error( - _( - 'VirusTotal API key cannot upload files larger than 32MB, ' - + 'use {url} to upload {path}.' - ).format(path=repofilename, url=manual_url) - ) + logging.error(_('VirusTotal API key cannot upload files larger than 32MB, ' + + 'use {url} to upload {path}.') + .format(path=repofilename, url=manual_url)) else: r.raise_for_status() else: upload_url = 'https://www.virustotal.com/vtapi/v2/file/scan' if upload_url: - logging.info( - _('Uploading {apkfilename} to virustotal').format(apkfilename=repofilename) - ) - r = requests.post( - upload_url, - data=apikey, - headers=headers, - files={'file': (apkName, open(repofilename, 'rb'))}, - timeout=300, - ) - logging.debug( - _('If this upload fails, try manually uploading to {url}').format( - url=manual_url - ) - ) + logging.info(_('Uploading {apkfilename} to virustotal') + .format(apkfilename=repofilename)) + files = { + 'file': (apkName, open(repofilename, 'rb')) + } + r = requests.post(upload_url, data=data, headers=headers, files=files) + logging.debug(_('If this upload fails, try manually uploading to {url}') + .format(url=manual_url)) r.raise_for_status() response = r.json() logging.info(response['verbose_msg'] + " " + response['permalink']) @@ -841,7 +623,7 @@ def upload_apk_to_virustotal( def push_binary_transparency(git_repo_path, git_remote): - """Push the binary transparency git repo to the specifed remote. + '''push the binary transparency git repo to the specifed remote. If the remote is a local directory, make sure it exists, and is a git repo. This is used to move this git repo from an offline @@ -854,15 +636,18 @@ def push_binary_transparency(git_repo_path, git_remote): case, git_remote is a dir on the local file system, e.g. a thumb drive. - """ - logging.info(_('Pushing binary transparency log to {url}').format(url=git_remote)) + ''' + import git + + logging.info(_('Pushing binary transparency log to {url}') + .format(url=git_remote)) if os.path.isdir(os.path.dirname(git_remote)): # from offline machine to thumbdrive remote_path = os.path.abspath(git_repo_path) if not os.path.isdir(os.path.join(git_remote, '.git')): os.makedirs(git_remote, exist_ok=True) - thumbdriverepo = git.Repo.init(git_remote, initial_branch=GIT_BRANCH) + thumbdriverepo = git.Repo.init(git_remote) local = thumbdriverepo.create_remote('local', remote_path) else: thumbdriverepo = git.Repo(git_remote) @@ -873,7 +658,7 @@ def push_binary_transparency(git_repo_path, git_remote): local.set_url(remote_path) else: local = thumbdriverepo.create_remote('local', remote_path) - local.pull(GIT_BRANCH) + local.pull('master') else: # from online machine to remote on a server on the internet gitrepo = git.Repo(git_repo_path) @@ -884,186 +669,70 @@ def push_binary_transparency(git_repo_path, git_remote): origin.set_url(git_remote) else: origin = gitrepo.create_remote('origin', git_remote) - for _i in range(3): - try: - origin.push(GIT_BRANCH) - except git.GitCommandError as e: - logging.error(e) - continue - break - else: - raise FDroidException(_("Pushing to remote server failed!")) + origin.push('master') -def find_release_infos(index_v2_path, repo_dir, package_names): - """Find files, texts, etc. for uploading to a release page in index-v2.json. +def update_wiki(): + try: + import mwclient + site = mwclient.Site((config['wiki_protocol'], config['wiki_server']), + path=config['wiki_path']) + site.login(config['wiki_user'], config['wiki_password']) - This function parses index-v2.json for file-paths elegible for deployment - to release pages. (e.g. GitHub releases) It also groups these files by - packageName and versionName. e.g. to get a list of files for all specific - release of fdroid client you may call: - - find_binary_release_infos()['org.fdroid.fdroid']['0.19.2'] - - All paths in the returned data-structure are of type pathlib.Path. - """ - release_infos = {} - with open(index_v2_path, 'r') as f: - idx = json.load(f) - for package_name in package_names: - package = idx.get('packages', {}).get(package_name, {}) - for version in package.get('versions', {}).values(): - if package_name not in release_infos: - release_infos[package_name] = {} - version_name = version['manifest']['versionName'] - version_path = repo_dir / version['file']['name'].lstrip("/") - files = [version_path] - asc_path = pathlib.Path(str(version_path) + '.asc') - if asc_path.is_file(): - files.append(asc_path) - sig_path = pathlib.Path(str(version_path) + '.sig') - if sig_path.is_file(): - files.append(sig_path) - release_infos[package_name][version_name] = { - 'files': files, - 'whatsNew': version.get('whatsNew', {}).get("en-US"), - 'hasReleaseChannels': len(version.get('releaseChannels', [])) > 0, - } - return release_infos - - -def upload_to_github_releases(repo_section, gh_config, global_gh_token): - repo_dir = pathlib.Path(repo_section) - index_v2_path = repo_dir / 'index-v2.json' - if not index_v2_path.is_file(): - logging.warning( - _( - "Error deploying 'github_releases', {} not present. (You might " - "need to run `fdroid update` first.)" - ).format(index_v2_path) - ) - return - - package_names = [] - for repo_conf in gh_config: - for package_name in repo_conf.get('packageNames', []): - package_names.append(package_name) - - release_infos = fdroidserver.deploy.find_release_infos( - index_v2_path, repo_dir, package_names - ) - - for repo_conf in gh_config: - upload_to_github_releases_repo(repo_conf, release_infos, global_gh_token) - - -def upload_to_github_releases_repo(repo_conf, release_infos, global_gh_token): - projectUrl = repo_conf.get("projectUrl") - if not projectUrl: - logging.warning( - _( - "One of the 'github_releases' config items is missing the " - "'projectUrl' value. skipping ..." - ) - ) - return - token = repo_conf.get("token") or global_gh_token - if not token: - logging.warning( - _( - "One of the 'github_releases' config items is missing the " - "'token' value. skipping ..." - ) - ) - return - conf_package_names = repo_conf.get("packageNames", []) - if type(conf_package_names) == str: - conf_package_names = [conf_package_names] - if not conf_package_names: - logging.warning( - _( - "One of the 'github_releases' config items is missing the " - "'packageNames' value. skipping ..." - ) - ) - return - - # lookup all versionNames (git tags) for all packages available in the - # local fdroid repo - all_local_versions = set() - for package_name in conf_package_names: - for version in release_infos.get(package_name, {}).keys(): - all_local_versions.add(version) - - gh = fdroidserver.github.GithubApi(token, projectUrl) - unreleased_tags = gh.list_unreleased_tags() - - for version in all_local_versions: - if version in unreleased_tags: - # Making sure we're not uploading this version when releaseChannels - # is set. (releaseChannels usually mean it's e.g. an alpha or beta - # version) - if ( - not release_infos.get(conf_package_names[0], {}) - .get(version, {}) - .get('hasReleaseChannels') - ): - # collect files associated with this github release - files = [] - for package in conf_package_names: - files.extend( - release_infos.get(package, {}).get(version, {}).get('files', []) - ) - # always use the whatsNew text from the first app listed in - # config.yml github_releases.packageNames - text = ( - release_infos.get(conf_package_names[0], {}) - .get(version, {}) - .get('whatsNew') - or '' - ) - if 'release_notes_prepend' in repo_conf: - text = repo_conf['release_notes_prepend'] + "\n\n" + text - # create new release on github and upload all associated files - gh.create_release(version, files, text) + # Write a page with the last build log for this version code + wiki_page_path = 'deploy_' + time.strftime('%s', start_timestamp) + newpage = site.Pages[wiki_page_path] + txt = '' + txt += "* command line: " + ' '.join(sys.argv) + "\n" + txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n' + txt += "* completed at " + common.get_wiki_timestamp() + '\n' + txt += "\n\n" + newpage.save(txt, summary='Run log') + newpage = site.Pages['deploy'] + newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect') + except Exception as e: + logging.error(_('Error while attempting to publish log: %s') % e) def main(): - global config + global config, options parser = ArgumentParser() common.setup_global_opts(parser) - parser.add_argument( - "-i", - "--identity-file", - default=None, - help=_("Specify an identity file to provide to SSH for rsyncing"), - ) - parser.add_argument( - "--local-copy-dir", - default=None, - help=_("Specify a local folder to sync the repo to"), - ) - parser.add_argument( - "--no-checksum", - action="store_true", - default=False, - help=_("Don't use rsync checksums"), - ) - parser.add_argument( - "--no-keep-git-mirror-archive", - action="store_true", - default=False, - help=_("If a git mirror gets to big, allow the archive to be deleted"), - ) - options = common.parse_args(parser) - config = common.read_config() + parser.add_argument("-i", "--identity-file", default=None, + help=_("Specify an identity file to provide to SSH for rsyncing")) + parser.add_argument("--local-copy-dir", default=None, + help=_("Specify a local folder to sync the repo to")) + parser.add_argument("--no-checksum", action="store_true", default=False, + help=_("Don't use rsync checksums")) + parser.add_argument("--no-keep-git-mirror-archive", action="store_true", default=False, + help=_("If a git mirror gets to big, allow the archive to be deleted")) + options = parser.parse_args() + config = common.read_config(options) if config.get('nonstandardwebroot') is True: standardwebroot = False else: standardwebroot = True + for serverwebroot in config.get('serverwebroot', []): + # this supports both an ssh host:path and just a path + s = serverwebroot.rstrip('/').split(':') + if len(s) == 1: + fdroiddir = s[0] + elif len(s) == 2: + host, fdroiddir = s + else: + logging.error(_('Malformed serverwebroot line:') + ' ' + serverwebroot) + sys.exit(1) + repobase = os.path.basename(fdroiddir) + if standardwebroot and repobase != 'fdroid': + logging.error('serverwebroot path does not end with "fdroid", ' + + 'perhaps you meant one of these:\n\t' + + serverwebroot.rstrip('/') + '/fdroid\n\t' + + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid') + sys.exit(1) + if options.local_copy_dir is not None: local_copy_dir = options.local_copy_dir elif config.get('local_copy_dir'): @@ -1076,23 +745,17 @@ def main(): logging.error(_('local_copy_dir must be directory, not a file!')) sys.exit(1) if not os.path.exists(os.path.dirname(fdroiddir)): - logging.error( - _('The root dir for local_copy_dir "{path}" does not exist!').format( - path=os.path.dirname(fdroiddir) - ) - ) + logging.error(_('The root dir for local_copy_dir "{path}" does not exist!') + .format(path=os.path.dirname(fdroiddir))) sys.exit(1) if not os.path.isabs(fdroiddir): logging.error(_('local_copy_dir must be an absolute path!')) sys.exit(1) repobase = os.path.basename(fdroiddir) if standardwebroot and repobase != 'fdroid': - logging.error( - _( - 'local_copy_dir does not end with "fdroid", ' - + 'perhaps you meant: "{path}"' - ).format(path=fdroiddir + '/fdroid') - ) + logging.error(_('local_copy_dir does not end with "fdroid", ' + + 'perhaps you meant: "{path}"') + .format(path=fdroiddir + '/fdroid')) sys.exit(1) if local_copy_dir[-1] != '/': local_copy_dir += '/' @@ -1100,22 +763,16 @@ def main(): if not os.path.exists(fdroiddir): os.mkdir(fdroiddir) - if ( - not config.get('awsbucket') - and not config.get('serverwebroot') - and not config.get('servergitmirrors') - and not config.get('androidobservatory') - and not config.get('binary_transparency_remote') - and not config.get('virustotal_apikey') - and not config.get('github_releases') - and local_copy_dir is None - ): - logging.warning( - _('No option set! Edit your config.yml to set at least one of these:') - + '\nserverwebroot, servergitmirrors, local_copy_dir, awsbucket, ' - + 'virustotal_apikey, androidobservatory, github_releases ' - + 'or binary_transparency_remote' - ) + if not config.get('awsbucket') \ + and not config.get('serverwebroot') \ + and not config.get('servergitmirrors') \ + and not config.get('androidobservatory') \ + and not config.get('binary_transparency_remote') \ + and not config.get('virustotal_apikey') \ + and local_copy_dir is None: + logging.warning(_('No option set! Edit your config.py to set at least one of these:') + + '\nserverwebroot, servergitmirrors, local_copy_dir, awsbucket, ' + + 'virustotal_apikey, androidobservatory, or binary_transparency_remote') sys.exit(1) repo_sections = ['repo'] @@ -1126,48 +783,32 @@ def main(): if config['per_app_repos']: repo_sections += common.get_per_app_repos() - if os.path.isdir('unsigned') or ( - local_copy_dir is not None - and os.path.isdir(os.path.join(local_copy_dir, 'unsigned')) - ): - repo_sections.append('unsigned') - for repo_section in repo_sections: if local_copy_dir is not None: if config['sync_from_local_copy_dir']: sync_from_localcopy(repo_section, local_copy_dir) else: update_localcopy(repo_section, local_copy_dir) - if config.get('serverwebroot'): - update_serverwebroots( - config['serverwebroot'], repo_section, standardwebroot - ) - if config.get('servergitmirrors'): + for serverwebroot in config.get('serverwebroot', []): + update_serverwebroot(serverwebroot, repo_section) + if config.get('servergitmirrors', []): # update_servergitmirrors will take care of multiple mirrors so don't need a foreach - update_servergitmirrors(config['servergitmirrors'], repo_section) + servergitmirrors = config.get('servergitmirrors', []) + update_servergitmirrors(servergitmirrors, repo_section) if config.get('awsbucket'): - awsbucket = config['awsbucket'] - index_only = config.get('awsbucket_index_only') - update_remote_storage_with_rclone( - repo_section, - awsbucket, - index_only, - options.verbose, - options.quiet, - not options.no_checksum, - ) + update_awsbucket(repo_section) if config.get('androidobservatory'): upload_to_android_observatory(repo_section) if config.get('virustotal_apikey'): upload_to_virustotal(repo_section, config.get('virustotal_apikey')) - if config.get('github_releases'): - upload_to_github_releases( - repo_section, config.get('github_releases'), config.get('github_token') - ) - binary_transparency_remote = config.get('binary_transparency_remote') - if binary_transparency_remote: - push_binary_transparency(BINARY_TRANSPARENCY_DIR, binary_transparency_remote) + binary_transparency_remote = config.get('binary_transparency_remote') + if binary_transparency_remote: + push_binary_transparency(BINARY_TRANSPARENCY_DIR, + binary_transparency_remote) + + if config.get('wiki_server') and config.get('wiki_path'): + update_wiki() common.write_status_json(common.setup_status_output(start_timestamp)) sys.exit(0) diff --git a/fdroidserver/exception.py b/fdroidserver/exception.py index 682ccef7..f9f876ce 100644 --- a/fdroidserver/exception.py +++ b/fdroidserver/exception.py @@ -1,6 +1,6 @@ class FDroidException(Exception): + def __init__(self, value=None, detail=None): - super().__init__() self.value = value self.detail = detail @@ -9,22 +9,26 @@ class FDroidException(Exception): return self.detail return '[...]\n' + self.detail[-16000:] + def get_wikitext(self): + ret = repr(self.value) + "\n" + if self.detail: + ret += "=detail=\n" + ret += "
\n" + self.shortened_detail() + "
\n" + return ret + def __str__(self): if self.value is None: ret = __name__ else: ret = str(self.value) if self.detail: - ret += ( - "\n==== detail begin ====\n%s\n==== detail end ====" - % ''.join(self.detail).strip() - ) + ret += "\n==== detail begin ====\n%s\n==== detail end ====" % ''.join(self.detail).strip() return ret class MetaDataException(Exception): + def __init__(self, value): - super().__init__() self.value = value def __str__(self): @@ -35,10 +39,6 @@ class VCSException(FDroidException): pass -class NoVersionCodeException(FDroidException): - pass - - class NoSubmodulesException(VCSException): pass @@ -49,10 +49,3 @@ class BuildException(FDroidException): class VerificationException(FDroidException): pass - - -class ConfigurationException(FDroidException): - def __init__(self, value=None, detail=None): - super().__init__() - self.value = value - self.detail = detail diff --git a/fdroidserver/github.py b/fdroidserver/github.py deleted file mode 100644 index 34a3ee53..00000000 --- a/fdroidserver/github.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env python3 -# -# github.py - part of the FDroid server tools -# Copyright (C) 2024, Michael Pöhn, michael@poehn.at -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import json -import pathlib -import urllib.parse -import urllib.request - - -class GithubApi: - """Wrapper for some select calls to GitHub Json/REST API. - - This class wraps some calls to api.github.com. This is not intended to be a - general API wrapper. Instead it's purpose is to return pre-filtered and - transformed data that's playing well with other fdroidserver functions. - - With the GitHub API, the token is optional, but it has pretty - severe rate limiting. - - """ - - def __init__(self, api_token, repo_path): - self._api_token = api_token - if repo_path.startswith("https://github.com/"): - self._repo_path = repo_path[19:] - else: - self._repo_path = repo_path - - def _req(self, url, data=None): - h = { - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - } - if self._api_token: - h["Authorization"] = f"Bearer {self._api_token}" - return urllib.request.Request( - url, - headers=h, - data=data, - ) - - def list_released_tags(self): - """List of all tags that are associated with a release for this repo on GitHub.""" - names = [] - req = self._req(f"https://api.github.com/repos/{self._repo_path}/releases") - with urllib.request.urlopen(req) as resp: # nosec CWE-22 disable bandit warning - releases = json.load(resp) - for release in releases: - names.append(release['tag_name']) - return names - - def list_unreleased_tags(self): - all_tags = self.list_all_tags() - released_tags = self.list_released_tags() - return [x for x in all_tags if x not in released_tags] - - def get_latest_apk(self): - req = self._req( - f"https://api.github.com/repos/{self._repo_path}/releases/latest" - ) - with urllib.request.urlopen(req) as resp: # nosec CWE-22 disable bandit warning - assets = json.load(resp)['assets'] - for asset in assets: - url = asset.get('browser_download_url') - if url and url.endswith('.apk'): - return url - - def tag_exists(self, tag): - """ - Check if git tag is present on github. - - https://docs.github.com/en/rest/git/refs?apiVersion=2022-11-28#list-matching-references--fine-grained-access-tokens - """ - req = self._req( - f"https://api.github.com/repos/{self._repo_path}/git/matching-refs/tags/{tag}" - ) - with urllib.request.urlopen(req) as resp: # nosec CWE-22 disable bandit warning - rd = json.load(resp) - return len(rd) == 1 and rd[0].get("ref", False) == f"refs/tags/{tag}" - return False - - def list_all_tags(self): - """Get list of all tags for this repo on GitHub.""" - tags = [] - req = self._req( - f"https://api.github.com/repos/{self._repo_path}/git/matching-refs/tags/" - ) - with urllib.request.urlopen(req) as resp: # nosec CWE-22 disable bandit warning - refs = json.load(resp) - for ref in refs: - r = ref.get('ref', '') - if r.startswith('refs/tags/'): - tags.append(r[10:]) - return tags - - def create_release(self, tag, files, body=''): - """ - Create a new release on github. - - also see: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#create-a-release - - :returns: True if release was created, False if release already exists - :raises: urllib exceptions in case of network or api errors, also - raises an exception when the tag doesn't exists. - """ - # Querying github to create a new release for a non-existent tag, will - # also create that tag on github. So we need an additional check to - # prevent this behavior. - if not self.tag_exists(tag): - raise Exception( - f"can't create github release for {self._repo_path} {tag}, tag doesn't exists" - ) - # create the relase on github - req = self._req( - f"https://api.github.com/repos/{self._repo_path}/releases", - data=json.dumps( - { - "tag_name": tag, - "body": body, - } - ).encode("utf-8"), - ) - try: - with urllib.request.urlopen( # nosec CWE-22 disable bandit warning - req - ) as resp: - release_id = json.load(resp)['id'] - except urllib.error.HTTPError as e: - if e.status == 422: - codes = [x['code'] for x in json.load(e).get('errors', [])] - if "already_exists" in codes: - return False - raise e - - # attach / upload all files for the relase - for file in files: - self._create_release_asset(release_id, file) - - return True - - def _create_release_asset(self, release_id, file): - """ - Attach a file to a release on GitHub. - - This uploads a file to github relases, it will be attached to the supplied release - - also see: https://docs.github.com/en/rest/releases/assets?apiVersion=2022-11-28#upload-a-release-asset - """ - file = pathlib.Path(file) - with open(file, 'rb') as f: - req = urllib.request.Request( - f"https://uploads.github.com/repos/{self._repo_path}/releases/{release_id}/assets?name={file.name}", - headers={ - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {self._api_token}", - "X-GitHub-Api-Version": "2022-11-28", - "Content-Type": "application/octet-stream", - }, - data=f.read(), - ) - with urllib.request.urlopen(req): # nosec CWE-22 disable bandit warning - return True - return False diff --git a/fdroidserver/gpgsign.py b/fdroidserver/gpgsign.py index 4341cb36..86258b63 100644 --- a/fdroidserver/gpgsign.py +++ b/fdroidserver/gpgsign.py @@ -16,22 +16,25 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import glob -import logging import os -import time +import glob from argparse import ArgumentParser +import logging +import time -from . import _, common +from . import _ +from . import common from .common import FDroidPopen from .exception import FDroidException config = None +options = None start_timestamp = time.gmtime() def status_update_json(signed): - """Output a JSON file with metadata about this run.""" + """Output a JSON file with metadata about this run""" + logging.debug(_('Outputting JSON')) output = common.setup_status_output(start_timestamp) if signed: @@ -40,14 +43,15 @@ def status_update_json(signed): def main(): - global config + + global config, options # Parse command line... parser = ArgumentParser() common.setup_global_opts(parser) - common.parse_args(parser) + options = parser.parse_args() - config = common.read_config() + config = common.read_config(options) repodirs = ['repo'] if config['archive_older'] != 0: @@ -56,20 +60,22 @@ def main(): signed = [] for output_dir in repodirs: if not os.path.isdir(output_dir): - raise FDroidException( - _("Missing output directory") + " '" + output_dir + "'" - ) + raise FDroidException(_("Missing output directory") + " '" + output_dir + "'") # Process any apks that are waiting to be signed... for f in sorted(glob.glob(os.path.join(output_dir, '*.*'))): - if not common.is_repo_file(f, for_gpg_signing=True): + if common.get_file_extension(f) == 'asc': + continue + if not common.is_repo_file(f): continue filename = os.path.basename(f) sigfilename = filename + ".asc" sigpath = os.path.join(output_dir, sigfilename) if not os.path.exists(sigpath): - gpgargs = ['gpg', '-a', '--output', sigpath, '--detach-sig'] + gpgargs = ['gpg', '-a', + '--output', sigpath, + '--detach-sig'] if 'gpghome' in config: gpgargs.extend(['--homedir', config['gpghome']]) if 'gpgkey' in config: diff --git a/fdroidserver/import.py b/fdroidserver/import.py new file mode 100644 index 00000000..48af3b9c --- /dev/null +++ b/fdroidserver/import.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +# +# import.py - part of the FDroid server tools +# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com +# Copyright (C) 2013-2014 Daniel Martí +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import configparser +import git +import json +import os +import shutil +import sys +import yaml +from argparse import ArgumentParser +import logging + +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + +from . import _ +from . import common +from . import metadata +from .exception import FDroidException + + +config = None +options = None + + +# WARNING! This cannot be imported as a Python module, so reuseable functions need to go into common.py! + +def clone_to_tmp_dir(app): + tmp_dir = 'tmp' + if not os.path.isdir(tmp_dir): + logging.info(_("Creating temporary directory")) + os.makedirs(tmp_dir) + + tmp_dir = os.path.join(tmp_dir, 'importer') + if os.path.exists(tmp_dir): + shutil.rmtree(tmp_dir) + vcs = common.getvcs(app.RepoType, app.Repo, tmp_dir) + vcs.gotorevision(options.rev) + + return tmp_dir + + +def check_for_kivy_buildozer(tmp_importer_dir, app, build): + versionCode = None + buildozer_spec = os.path.join(tmp_importer_dir, 'buildozer.spec') + if os.path.exists(buildozer_spec): + config = configparser.ConfigParser() + config.read(buildozer_spec) + import pprint + pprint.pprint(sorted(config['app'].keys())) + app.id = config['app'].get('package.domain') + print(app.id) + app.AutoName = config['app'].get('package.name', app.AutoName) + app.License = config['app'].get('license', app.License) + app.Description = config['app'].get('description', app.Description) + build.versionName = config['app'].get('version') + build.output = 'bin/%s-$$VERSION$$-release-unsigned.apk' % app.AutoName + build.ndk = 'r17c' + build.srclibs = [ + 'buildozer@586152c', + 'python-for-android@ccb0f8e1', + ] + build.sudo = [ + 'apt-get update', + 'apt-get install -y build-essential libffi-dev libltdl-dev', + ] + build.prebuild = [ + 'sed -iE "/^[# ]*android\\.(ant|ndk|sdk)_path[ =]/d" buildozer.spec', + 'sed -iE "/^[# ]*android.accept_sdk_license[ =]+.*/d" buildozer.spec', + 'sed -iE "/^[# ]*android.skip_update[ =]+.*/d" buildozer.spec', + 'sed -iE "/^[# ]*p4a.source_dir[ =]+.*/d" buildozer.spec', + 'sed -i "s,\\[app\\],[app]\\n\\nandroid.sdk_path = $$SDK$$\\nandroid.ndk_path = $$NDK$$\\np4a.source_dir = $$python-for-android$$\\nandroid.accept_sdk_license = False\\nandroid.skip_update = True\\nandroid.ant_path = /usr/bin/ant\\n," buildozer.spec', + 'pip3 install --user --upgrade $$buildozer$$ Cython==0.28.6', + ] + build.build = [ + 'PATH="$HOME/.local/bin:$PATH" buildozer android release', + ] + return build.get('versionName'), versionCode, app.get('id') + + +def main(): + + global config, options + + # Parse command line... + parser = ArgumentParser() + common.setup_global_opts(parser) + parser.add_argument("-u", "--url", default=None, + help=_("Project URL to import from.")) + parser.add_argument("-s", "--subdir", default=None, + help=_("Path to main Android project subdirectory, if not in root.")) + parser.add_argument("-c", "--categories", default=None, + help=_("Comma separated list of categories.")) + parser.add_argument("-l", "--license", default=None, + help=_("Overall license of the project.")) + parser.add_argument("--omit-disable", action="store_true", default=False, + help=_("Do not add 'disable:' to the generated build entries")) + parser.add_argument("--rev", default=None, + help=_("Allows a different revision (or git branch) to be specified for the initial import")) + metadata.add_metadata_arguments(parser) + options = parser.parse_args() + metadata.warnings_action = options.W + + config = common.read_config(options) + + apps = metadata.read_metadata() + app = None + + tmp_importer_dir = None + + local_metadata_files = common.get_local_metadata_files() + if local_metadata_files != []: + raise FDroidException(_("This repo already has local metadata: %s") % local_metadata_files[0]) + + build = metadata.Build() + if options.url is None and os.path.isdir('.git'): + app = metadata.App() + app.AutoName = os.path.basename(os.getcwd()) + app.RepoType = 'git' + + if os.path.exists('build.gradle') or os.path.exists('build.gradle.kts'): + build.gradle = ['yes'] + + git_repo = git.repo.Repo(os.getcwd()) + for remote in git.Remote.iter_items(git_repo): + if remote.name == 'origin': + url = git_repo.remotes.origin.url + if url.startswith('https://git'): # github, gitlab + app.SourceCode = url.rstrip('.git') + app.Repo = url + break + write_local_file = True + elif options.url: + app = common.get_app_from_url(options.url) + tmp_importer_dir = clone_to_tmp_dir(app) + git_repo = git.repo.Repo(tmp_importer_dir) + if not options.omit_disable: + build.disable = 'Generated by import.py - check/set version fields and commit id' + write_local_file = False + else: + raise FDroidException("Specify project url.") + + app.UpdateCheckMode = 'Tags' + build.commit = common.get_head_commit_id(git_repo) + + versionName, versionCode, appid = check_for_kivy_buildozer(tmp_importer_dir, app, build) + + # Extract some information... + paths = common.get_all_gradle_and_manifests(tmp_importer_dir) + subdir = common.get_gradle_subdir(tmp_importer_dir, paths) + if paths: + versionName, versionCode, appid = common.parse_androidmanifests(paths, app) + if not appid: + raise FDroidException(_("Couldn't find Application ID")) + if not versionName: + logging.warning(_('Could not find latest version name')) + if not versionCode: + logging.warning(_('Could not find latest version code')) + elif not appid: + raise FDroidException(_("No gradle project could be found. Specify --subdir?")) + + # Make sure it's actually new... + if appid in apps: + raise FDroidException(_('Package "{appid}" already exists').format(appid=appid)) + + # Create a build line... + build.versionName = versionName or 'Unknown' + build.versionCode = versionCode or '0' # TODO heinous but this is still a str + if options.subdir: + build.subdir = options.subdir + build.gradle = ['yes'] + elif subdir: + build.subdir = subdir + build.gradle = ['yes'] + + if options.license: + app.License = options.license + if options.categories: + app.Categories = options.categories.split(',') + if os.path.exists(os.path.join(subdir, 'jni')): + build.buildjni = ['yes'] + if os.path.exists(os.path.join(subdir, 'build.gradle')) \ + or os.path.exists(os.path.join(subdir, 'build.gradle')): + build.gradle = ['yes'] + + package_json = os.path.join(tmp_importer_dir, 'package.json') # react-native + pubspec_yaml = os.path.join(tmp_importer_dir, 'pubspec.yaml') # flutter + if os.path.exists(package_json): + build.sudo = ['apt-get update || apt-get update', 'apt-get install -t stretch-backports npm', 'npm install -g react-native-cli'] + build.init = ['npm install'] + with open(package_json) as fp: + data = json.load(fp) + app.AutoName = data.get('name', app.AutoName) + app.License = data.get('license', app.License) + app.Description = data.get('description', app.Description) + app.WebSite = data.get('homepage', app.WebSite) + app_json = os.path.join(tmp_importer_dir, 'app.json') + if os.path.exists(app_json): + with open(app_json) as fp: + data = json.load(fp) + app.AutoName = data.get('name', app.AutoName) + if os.path.exists(pubspec_yaml): + with open(pubspec_yaml) as fp: + data = yaml.load(fp, Loader=SafeLoader) + app.AutoName = data.get('name', app.AutoName) + app.License = data.get('license', app.License) + app.Description = data.get('description', app.Description) + build.srclibs = ['flutter@stable'] + build.output = 'build/app/outputs/apk/release/app-release.apk' + build.build = [ + '$$flutter$$/bin/flutter config --no-analytics', + '$$flutter$$/bin/flutter packages pub get', + '$$flutter$$/bin/flutter build apk', + ] + + git_modules = os.path.join(tmp_importer_dir, '.gitmodules') + if os.path.exists(git_modules): + build.submodules = True + + metadata.post_metadata_parse(app) + + app.builds.append(build) + + if write_local_file: + metadata.write_metadata('.fdroid.yml', app) + else: + # Keep the repo directory to save bandwidth... + if not os.path.exists('build'): + os.mkdir('build') + build_dir = os.path.join('build', appid) + if os.path.exists(build_dir): + logging.warning(_('{path} already exists, ignoring import results!') + .format(path=build_dir)) + sys.exit(1) + elif tmp_importer_dir is not None: + shutil.move(tmp_importer_dir, build_dir) + with open('build/.fdroidvcs-' + appid, 'w') as f: + f.write(app.RepoType + ' ' + app.Repo) + + metadatapath = os.path.join('metadata', appid + '.yml') + metadata.write_metadata(metadatapath, app) + logging.info("Wrote " + metadatapath) + + +if __name__ == "__main__": + main() diff --git a/fdroidserver/import_subcommand.py b/fdroidserver/import_subcommand.py deleted file mode 100644 index 017ebe54..00000000 --- a/fdroidserver/import_subcommand.py +++ /dev/null @@ -1,470 +0,0 @@ -#!/usr/bin/env python3 -"""Extract application metadata from a source repository.""" -# -# import_subcommand.py - part of the FDroid server tools -# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com -# Copyright (C) 2013-2014 Daniel Martí -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import json -import logging -import os -import re -import shutil -import stat -import sys -import urllib -from argparse import ArgumentParser -from pathlib import Path -from typing import Optional - -import git -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - -from . import _, common, metadata -from .exception import FDroidException - -config = None - -SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?') -GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''') -APPLICATION_ID_REGEX = re.compile(r'''\s*applicationId\s=?\s?['"].*['"]''') - - -def get_all_gradle_and_manifests(build_dir): - paths = [] - for root, dirs, files in os.walk(build_dir): - for f in sorted(files): - if f == 'AndroidManifest.xml' or f.endswith(('.gradle', '.gradle.kts')): - full = Path(root) / f - paths.append(full) - return paths - - -def get_gradle_subdir(build_dir, paths): - """Get the subdir where the gradle build is based.""" - first_gradle_dir = None - for path in paths: - if not first_gradle_dir: - first_gradle_dir = path.parent.relative_to(build_dir) - if path.exists() and SETTINGS_GRADLE_REGEX.match(path.name): - for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text(encoding='utf-8')): - for f in (path.parent / m.group(1)).glob('build.gradle*'): - with f.open(encoding='utf-8') as fp: - for line in fp: - if common.ANDROID_PLUGIN_REGEX.match( - line - ) or APPLICATION_ID_REGEX.match(line): - return f.parent.relative_to(build_dir) - if first_gradle_dir and first_gradle_dir != Path('.'): - return first_gradle_dir - - -def handle_retree_error_on_windows(function, path, excinfo): - """Python can't remove a readonly file on Windows so chmod first.""" - if function in (os.unlink, os.rmdir, os.remove) and excinfo[0] == PermissionError: - os.chmod(path, stat.S_IWRITE) - function(path) - - -def clone_to_tmp_dir(app: metadata.App, rev=None) -> Path: - """Clone the source repository of an app to a temporary directory for further processing. - - Parameters - ---------- - app - The App instance to clone the source of. - - Returns - ------- - tmp_dir - The (temporary) directory the apps source has been cloned into. - - """ - tmp_dir = Path('tmp') - tmp_dir.mkdir(exist_ok=True) - - tmp_dir = tmp_dir / 'importer' - - if tmp_dir.exists(): - shutil.rmtree(str(tmp_dir), onerror=handle_retree_error_on_windows) - vcs = common.getvcs(app.RepoType, app.Repo, tmp_dir) - vcs.gotorevision(rev) - - return tmp_dir - - -def getrepofrompage(url: str) -> tuple[Optional[str], str]: - """Get the repo type and address from the given web page. - - The page is scanned in a rather naive manner for 'git clone xxxx', - 'hg clone xxxx', etc, and when one of these is found it's assumed - that's the information we want. Returns repotype, address, or - None, reason - - Parameters - ---------- - url - The url to look for repository information at. - - Returns - ------- - repotype_or_none - The found repository type or None if an error occured. - address_or_reason - The address to the found repository or the reason if an error occured. - - """ - if not url.startswith('http'): - return (None, _('{url} does not start with "http"!'.format(url=url))) - req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out - if req.getcode() != 200: - return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode())) - page = req.read().decode(req.headers.get_content_charset()) - - # Works for BitBucket - m = re.search('data-fetch-url="(.*)"', page) - if m is not None: - repo = m.group(1) - - if repo.endswith('.git'): - return ('git', repo) - - return ('hg', repo) - - # Works for BitBucket (obsolete) - index = page.find('hg clone') - if index != -1: - repotype = 'hg' - repo = page[index + 9 :] - index = repo.find('<') - if index == -1: - return (None, _("Error while getting repo address")) - repo = repo[:index] - repo = repo.split('"')[0] - return (repotype, repo) - - # Works for BitBucket (obsolete) - index = page.find('git clone') - if index != -1: - repotype = 'git' - repo = page[index + 10 :] - index = repo.find('<') - if index == -1: - return (None, _("Error while getting repo address")) - repo = repo[:index] - repo = repo.split('"')[0] - return (repotype, repo) - - return (None, _("No information found.") + page) - - -def get_app_from_url(url: str) -> metadata.App: - """Guess basic app metadata from the URL. - - The URL must include a network hostname, unless it is an lp:, - file:, or git/ssh URL. This throws ValueError on bad URLs to - match urlparse(). - - Parameters - ---------- - url - The URL to look to look for app metadata at. - - Returns - ------- - app - App instance with the found metadata. - - Raises - ------ - :exc:`~fdroidserver.exception.FDroidException` - If the VCS type could not be determined. - :exc:`ValueError` - If the URL is invalid. - - """ - parsed = urllib.parse.urlparse(url) - invalid_url = False - if not parsed.scheme or not parsed.path: - invalid_url = True - - app = metadata.App() - app.Repo = url - if url.startswith('git://') or url.startswith('git@'): - app.RepoType = 'git' - elif parsed.netloc == 'github.com': - app.RepoType = 'git' - app.SourceCode = url - app.IssueTracker = url + '/issues' - elif parsed.netloc in ('gitlab.com', 'framagit.org'): - # git can be fussy with gitlab URLs unless they end in .git - if url.endswith('.git'): - url = url[:-4] - app.Repo = url + '.git' - app.RepoType = 'git' - app.SourceCode = url - app.IssueTracker = url + '/issues' - elif parsed.netloc == 'notabug.org': - if url.endswith('.git'): - url = url[:-4] - app.Repo = url + '.git' - app.RepoType = 'git' - app.SourceCode = url - app.IssueTracker = url + '/issues' - elif parsed.netloc == 'bitbucket.org': - if url.endswith('/'): - url = url[:-1] - app.SourceCode = url + '/src' - app.IssueTracker = url + '/issues' - # Figure out the repo type and adddress... - app.RepoType, app.Repo = getrepofrompage(url) - elif parsed.netloc == 'codeberg.org': - app.RepoType = 'git' - app.SourceCode = url - app.IssueTracker = url + '/issues' - elif url.startswith('https://') and url.endswith('.git'): - app.RepoType = 'git' - - if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'): - invalid_url = True - - if invalid_url: - raise ValueError(_('"{url}" is not a valid URL!'.format(url=url))) - - if not app.RepoType: - raise FDroidException("Unable to determine vcs type. " + app.Repo) - - return app - - -def main(): - """Extract app metadata and write it to a file. - - The behaviour of this function is influenced by the configuration file as - well as command line parameters. - - Raises - ------ - :exc:`~fdroidserver.exception.FDroidException` - If the repository already has local metadata, no URL is specified and - the current directory is not a Git repository, no application ID could - be found, no Gradle project could be found or there is already metadata - for the found application ID. - - """ - global config - - # Parse command line... - parser = ArgumentParser() - common.setup_global_opts(parser) - parser.add_argument("-u", "--url", help=_("Project URL to import from.")) - parser.add_argument( - "-s", - "--subdir", - help=_("Path to main Android project subdirectory, if not in root."), - ) - parser.add_argument( - "-c", - "--categories", - help=_("Comma separated list of categories."), - ) - parser.add_argument("-l", "--license", help=_("Overall license of the project.")) - parser.add_argument( - "--omit-disable", - action="store_true", - help=_("Do not add 'disable:' to the generated build entries"), - ) - parser.add_argument( - "--rev", - help=_( - "Allows a different revision (or git branch) to be specified for the initial import" - ), - ) - metadata.add_metadata_arguments(parser) - options = common.parse_args(parser) - metadata.warnings_action = options.W - - config = common.read_config() - - apps = metadata.read_metadata() - app = None - - tmp_importer_dir = None - - local_metadata_files = common.get_local_metadata_files() - if local_metadata_files: - raise FDroidException( - _("This repo already has local metadata: %s") % local_metadata_files[0] - ) - - build = metadata.Build() - app = metadata.App() - if options.url is None and Path('.git').is_dir(): - app.RepoType = 'git' - tmp_importer_dir = Path.cwd() - git_repo = git.Repo(tmp_importer_dir) - for remote in git.Remote.iter_items(git_repo): - if remote.name == 'origin': - url = git_repo.remotes.origin.url - app = get_app_from_url(url) - break - write_local_file = True - elif options.url: - app = get_app_from_url(options.url) - tmp_importer_dir = clone_to_tmp_dir(app, options.rev) - git_repo = git.Repo(tmp_importer_dir) - - if not options.omit_disable: - build.disable = ( - 'Generated by `fdroid import` - check version fields and commitid' - ) - write_local_file = False - else: - raise FDroidException("Specify project url.") - - app.AutoUpdateMode = 'Version' - app.UpdateCheckMode = 'Tags' - build.commit = common.get_head_commit_id(tmp_importer_dir) - - # Extract some information... - paths = get_all_gradle_and_manifests(tmp_importer_dir) - gradle_subdir = get_gradle_subdir(tmp_importer_dir, paths) - if paths: - versionName, versionCode, appid = common.parse_androidmanifests(paths, app) - if not appid: - raise FDroidException(_("Couldn't find Application ID")) - if not versionName: - logging.warning(_('Could not find latest versionName')) - if not versionCode: - logging.warning(_('Could not find latest versionCode')) - else: - raise FDroidException(_("No gradle project could be found. Specify --subdir?")) - - # Make sure it's actually new... - if appid in apps: - raise FDroidException(_('Package "{appid}" already exists').format(appid=appid)) - - # Create a build line... - build.versionName = versionName or 'Unknown' - app.CurrentVersion = build.versionName - build.versionCode = versionCode or 0 - app.CurrentVersionCode = build.versionCode - if options.subdir: - build.subdir = options.subdir - elif gradle_subdir: - build.subdir = gradle_subdir.as_posix() - # subdir might be None - subdir = Path(tmp_importer_dir / build.subdir) if build.subdir else tmp_importer_dir - - if options.license: - app.License = options.license - if options.categories: - app.Categories = options.categories.split(',') - if (subdir / 'jni').exists(): - build.buildjni = ['yes'] - if (subdir / 'build.gradle').exists() or (subdir / 'build.gradle.kts').exists(): - build.gradle = ['yes'] - - app.AutoName = common.fetch_real_name(subdir, build.gradle) - - package_json = tmp_importer_dir / 'package.json' # react-native - pubspec_yaml = tmp_importer_dir / 'pubspec.yaml' # flutter - if package_json.exists(): - build.sudo = [ - 'sysctl fs.inotify.max_user_watches=524288 || true', - 'apt-get update', - 'apt-get install -y npm', - ] - build.init = ['npm install --build-from-source'] - with package_json.open() as fp: - data = json.load(fp) - app.AutoName = app.AutoName or data.get('name') - app.License = data.get('license', app.License) - app.Description = data.get('description', app.Description) - app.WebSite = data.get('homepage', app.WebSite) - app_json = tmp_importer_dir / 'app.json' - build.scanignore = ['android/build.gradle'] - build.scandelete = ['node_modules'] - if app_json.exists(): - with app_json.open() as fp: - data = json.load(fp) - app.AutoName = app.AutoName or data.get('name') - if pubspec_yaml.exists(): - with pubspec_yaml.open() as fp: - data = yaml.load(fp, Loader=SafeLoader) - app.AutoName = app.AutoName or data.get('name') - app.License = data.get('license', app.License) - app.Description = data.get('description', app.Description) - app.UpdateCheckData = 'pubspec.yaml|version:\\s.+\\+(\\d+)|.|version:\\s(.+)\\+' - build.srclibs = ['flutter@stable'] - build.output = 'build/app/outputs/flutter-apk/app-release.apk' - build.subdir = None - build.gradle = None - build.prebuild = [ - 'export PUB_CACHE=$(pwd)/.pub-cache', - '$$flutter$$/bin/flutter config --no-analytics', - '$$flutter$$/bin/flutter packages pub get', - ] - build.scandelete = [ - '.pub-cache', - ] - build.build = [ - 'export PUB_CACHE=$(pwd)/.pub-cache', - '$$flutter$$/bin/flutter build apk', - ] - - git_modules = tmp_importer_dir / '.gitmodules' - if git_modules.exists(): - build.submodules = True - - metadata.post_parse_yaml_metadata(app) - - app['Builds'].append(build) - - if write_local_file: - metadata.write_metadata(Path('.fdroid.yml'), app) - else: - # Keep the repo directory to save bandwidth... - Path('build').mkdir(exist_ok=True) - build_dir = Path('build') / appid - if build_dir.exists(): - logging.warning( - _('{path} already exists, ignoring import results!').format( - path=build_dir - ) - ) - sys.exit(1) - elif tmp_importer_dir: - # For Windows: Close the repo or a git.exe instance holds handles to repo - try: - git_repo.close() - except AttributeError: # Debian/stretch's version does not have close() - pass - shutil.move(tmp_importer_dir, build_dir) - Path('build/.fdroidvcs-' + appid).write_text(app.RepoType + ' ' + app.Repo) - - metadatapath = Path('metadata') / (appid + '.yml') - metadata.write_metadata(metadatapath, app) - logging.info("Wrote " + str(metadatapath)) - - -if __name__ == "__main__": - main() diff --git a/fdroidserver/index.py b/fdroidserver/index.py index b63729e4..e707824d 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -20,81 +20,54 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Process the index files. - -This module is loaded by all fdroid subcommands since it is loaded in -fdroidserver/__init__.py. Any narrowly used dependencies should be -imported where they are used to limit dependencies for subcommands -like publish/signindex/gpgsign. This eliminates the need to have -these installed on the signing server. - -""" - -import calendar import collections -import hashlib import json import logging import os import re import shutil -import sys import tempfile import urllib.parse import zipfile +import calendar from binascii import hexlify, unhexlify from datetime import datetime, timezone -from pathlib import Path from xml.dom.minidom import Document -from fdroidserver._yaml import yaml -from fdroidserver.common import ( - ANTIFEATURES_CONFIG_NAME, - CATEGORIES_CONFIG_NAME, - CONFIG_CONFIG_NAME, - DEFAULT_LOCALE, - MIRRORS_CONFIG_NAME, - RELEASECHANNELS_CONFIG_NAME, - FDroidPopen, - FDroidPopenBytes, - load_publish_signer_fingerprints, -) +from . import _ +from . import common +from . import metadata +from . import net +from . import signindex +from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints from fdroidserver.exception import FDroidException, VerificationException -from . import _, common, metadata, signindex - def make(apps, apks, repodir, archive): """Generate the repo index files. This requires properly initialized options and config objects. - Parameters - ---------- - apps - OrderedDict of apps to go into the index, each app should have - at least one associated apk - apks - list of apks to go into the index - repodir - the repo directory - archive - True if this is the archive repo, False if it's the - main one. + :param apps: OrderedDict of apps to go into the index, each app should have + at least one associated apk + :param apks: list of apks to go into the index + :param repodir: the repo directory + :param archive: True if this is the archive repo, False if it's the + main one. """ from fdroidserver.update import METADATA_VERSION - if not hasattr(common.options, 'nosign') or not common.options.nosign: + if not common.options.nosign: common.assert_config_keystore(common.config) # Historically the index has been sorted by App Name, so we enforce this ordering here - sortedids = sorted(apps, key=lambda appid: common.get_app_display_name(apps[appid]).upper()) + sortedids = sorted(apps, key=lambda appid: apps[appid]['Name'].upper()) sortedapps = collections.OrderedDict() for appid in sortedids: sortedapps[appid] = apps[appid] repodict = collections.OrderedDict() - repodict['timestamp'] = datetime.now(timezone.utc) + repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc) repodict['version'] = METADATA_VERSION if common.config['repo_maxage'] != 0: @@ -102,23 +75,36 @@ def make(apps, apks, repodir, archive): if archive: repodict['name'] = common.config['archive_name'] - repodict['icon'] = common.config.get('archive_icon', common.default_config['repo_icon']) + repodict['icon'] = os.path.basename(common.config['archive_icon']) + repodict['address'] = common.config['archive_url'] repodict['description'] = common.config['archive_description'] - archive_url = common.config.get('archive_url', common.config['repo_url'][:-4] + 'archive') - repodict['address'] = archive_url - if 'archive_web_base_url' in common.config: - repodict["webBaseUrl"] = common.config['archive_web_base_url'] - repo_section = os.path.basename(urllib.parse.urlparse(archive_url).path) + urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path) else: repodict['name'] = common.config['repo_name'] - repodict['icon'] = common.config.get('repo_icon', common.default_config['repo_icon']) + repodict['icon'] = os.path.basename(common.config['repo_icon']) repodict['address'] = common.config['repo_url'] - if 'repo_web_base_url' in common.config: - repodict["webBaseUrl"] = common.config['repo_web_base_url'] repodict['description'] = common.config['repo_description'] - repo_section = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path) + urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path) - add_mirrors_to_repodict(repo_section, repodict) + mirrorcheckfailed = False + mirrors = [] + for mirror in common.config.get('mirrors', []): + base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/')) + if common.config.get('nonstandardwebroot') is not True and base != 'fdroid': + logging.error(_("mirror '%s' does not end with 'fdroid'!") % mirror) + mirrorcheckfailed = True + # must end with / or urljoin strips a whole path segment + if mirror.endswith('/'): + mirrors.append(urllib.parse.urljoin(mirror, urlbasepath)) + else: + mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath)) + for mirror in common.config.get('servergitmirrors', []): + for url in get_mirror_service_urls(mirror): + mirrors.append(url + '/' + repodir) + if mirrorcheckfailed: + raise FDroidException(_("Malformed repository mirrors.")) + if mirrors: + repodict['mirrors'] = mirrors requestsdict = collections.OrderedDict() for command in ('install', 'uninstall'): @@ -133,599 +119,15 @@ def make(apps, apks, repodir, archive): raise TypeError(_('only accepts strings, lists, and tuples')) requestsdict[command] = packageNames - signer_fingerprints = load_publish_signer_fingerprints() + fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints() - make_v0(sortedapps, apks, repodir, repodict, requestsdict, signer_fingerprints) - make_v1(sortedapps, apks, repodir, repodict, requestsdict, signer_fingerprints) - make_v2( - sortedapps, apks, repodir, repodict, requestsdict, signer_fingerprints, archive - ) - make_website(sortedapps, repodir, repodict) - make_altstore( - sortedapps, - apks, - common.config, - repodir, - pretty=common.options.pretty, - ) + make_v0(sortedapps, apks, repodir, repodict, requestsdict, + fdroid_signing_key_fingerprints) + make_v1(sortedapps, apks, repodir, repodict, requestsdict, + fdroid_signing_key_fingerprints) -def _should_file_be_generated(path, magic_string): - if os.path.exists(path): - with open(path) as f: - # if the magic_string is not in the first line the file should be overwritten - if magic_string not in f.readline(): - return False - return True - - -def make_website(apps, repodir, repodict): - # do not change this string, as it will break updates for files with older versions of this string - autogenerate_comment = "auto-generated - fdroid index updates will overwrite this file" - - if not os.path.exists(repodir): - os.makedirs(repodir) - - html_name = 'index.html' - html_file = os.path.join(repodir, html_name) - - if _should_file_be_generated(html_file, autogenerate_comment): - import qrcode - - _ignored, repo_pubkey_fingerprint = extract_pubkey() - repo_pubkey_fingerprint_stripped = repo_pubkey_fingerprint.replace(" ", "") - link = repodict["address"] - link_fingerprinted = '{link}?fingerprint={fingerprint}'.format( - link=link, fingerprint=repo_pubkey_fingerprint_stripped - ) - qrcode.make(link_fingerprinted).save(os.path.join(repodir, "index.png")) - with open(html_file, 'w') as f: - name = repodict["name"] - description = repodict["description"] - icon = repodict["icon"] - f.write(""" - - - - - - - {name} - - - - - - - - - - - - -

- {name} -

-
-

- - - QR: test - - - {description} -
-
- Currently it serves - - {number_of_apps} - - apps. To add it to your F-Droid app, scan the QR code (click it to enlarge) or use this link: -

-

- - - {link} - - -

-

- If you would like to manually verify the fingerprint (SHA-256) of the repository signing key, here it is: -
- - {fingerprint} - -

-
- - -""".format(autogenerate_comment=autogenerate_comment, - description=description, - fingerprint=repo_pubkey_fingerprint, - icon=icon, - link=link, - link_fingerprinted=link_fingerprinted, - name=name, - number_of_apps=str(len(apps)))) - - css_file = os.path.join(repodir, "index.css") - if _should_file_be_generated(css_file, autogenerate_comment): - with open(css_file, "w") as f: - # this auto generated comment was not included via .format(), as python seems to have problems with css files in combination with .format() - f.write("""/* auto-generated - fdroid index updates will overwrite this file */ -BODY { - font-family : Arial, Helvetica, Sans-Serif; - color : #0000ee; - background-color : #ffffff; -} -p { - text-align : justify; -} -p.center { - text-align : center; -} -TD { - font-family : Arial, Helvetica, Sans-Serif; - color : #0000ee; -} -body,td { - font-size : 14px; -} -TH { - font-family : Arial, Helvetica, Sans-Serif; - color : #0000ee; - background-color : #F5EAD4; -} -a:link { - color : #bb0000; -} -a:visited { - color : #ff0000; -} -.zitat { - margin-left : 1cm; - margin-right : 1cm; - font-style : italic; -} -#intro { - border-spacing : 1em; - border : 1px solid gray; - border-radius : 0.5em; - box-shadow : 10px 10px 5px #888; - margin : 1.5em; - font-size : .9em; - width : 600px; - max-width : 90%; - display : table; - margin-left : auto; - margin-right : auto; - font-size : .8em; - color : #555555; -} -#intro > p { - margin-top : 0; -} -#intro p:last-child { - margin-bottom : 0; -} -.last { - border-bottom : 1px solid black; - padding-bottom : .5em; - text-align : center; -} -table { - border-collapse : collapse; -} -h2 { - text-align : center; -} -.perms { - font-family : monospace; - font-size : .8em; -} -.repoapplist { - display : table; - border-collapse : collapse; - margin-left : auto; - margin-right : auto; - width : 600px; - max-width : 90%; -} -.approw, appdetailrow { - display : table-row; -} -.appdetailrow { - display : flex; - padding : .5em; -} -.appiconbig, .appdetailblock, .appdetailcell { - display : table-cell -} -.appiconbig { - vertical-align : middle; - text-align : center; -} -.appdetailinner { - width : 100%; -} -.applinkcell { - text-align : center; - float : right; - width : 100%; - margin-bottom : .1em; -} -.paddedlink { - margin : 1em; -} -.approw { - border-spacing : 1em; - border : 1px solid gray; - border-radius : 0.5em; - padding : 0.5em; - margin : 1.5em; -} -.appdetailinner .appdetailrow:first-child { - background-color : #d5d5d5; -} -.appdetailinner .appdetailrow:first-child .appdetailcell { - min-width : 33%; - flex : 1 33%; - text-align : center; -} -.appdetailinner .appdetailrow:first-child .appdetailcell:first-child { - text-align : left; -} -.appdetailinner .appdetailrow:first-child .appdetailcell:last-child { - float : none; - text-align : right; -} -.minor-details { - font-size : .8em; - color : #555555; -} -.boldname { - font-weight : bold; -} -#appcount { - text-align : center; - margin-bottom : .5em; -} -kbd { - padding : 0.1em 0.6em; - border : 1px solid #CCC; - background-color : #F7F7F7; - color : #333; - box-shadow : 0px 1px 0px rgba(0, 0, 0, 0.2), 0px 0px 0px 2px #FFF inset; - border-radius : 3px; - display : inline-block; - margin : 0px 0.1em; - text-shadow : 0px 1px 0px #FFF; - white-space : nowrap; -} -div.filterline, div.repoline { - display : table; - margin-left : auto; - margin-right : auto; - margin-bottom : 1em; - vertical-align : middle; - display : table; - font-size : .8em; -} -.filterline form { - display : table-row; -} -.filterline .filtercell { - display : table-cell; - vertical-align : middle; -} -fieldset { - float : left; -} -fieldset select, fieldset input, #reposelect select, #reposelect input { - font-size : .9em; -} -.pager { - display : table; - margin-left : auto; - margin-right : auto; - width : 600px; - max-width : 90%; - padding-top : .6em; -} -/* should correspond to .repoapplist */ -.pagerrow { - display : table-row; -} -.pagercell { - display : table-cell; -} -.pagercell.left { - text-align : left; - padding-right : 1em; -} -.pagercell.middle { - text-align : center; - font-size : .9em; - color : #555; -} -.pagercell.right { - text-align : right; - padding-left : 1em; -} -.anti { - color : peru; -} -.antibold { - color : crimson; -} -#footer { - text-align : center; - margin-top : 1em; - font-size : 11px; - color : #555; -} -#footer img { - vertical-align : middle; -} -@media (max-width: 600px) { - .repoapplist { - display : block; - } - .appdetailinner, .appdetailrow { - display : block; - } - .appdetailcell { - display : block; - float : left; - line-height : 1.5em; - } -}""") - - -def dict_diff(source, target): - if not isinstance(target, dict) or not isinstance(source, dict): - return target - - result = {key: None for key in source if key not in target} - - for key, value in target.items(): - if key not in source: - result[key] = value - elif value != source[key]: - result[key] = dict_diff(source[key], value) - - return result - - -def convert_datetime(obj): - if isinstance(obj, datetime): - # Java prefers milliseconds - # we also need to account for time zone/daylight saving time - return int(calendar.timegm(obj.timetuple()) * 1000) - return obj - - -def package_metadata(app, repodir): - meta = {} - for element in ( - "added", - # "binaries", - "Categories", - "Changelog", - "IssueTracker", - "lastUpdated", - "License", - "SourceCode", - "Translation", - "WebSite", - "featureGraphic", - "promoGraphic", - "tvBanner", - "screenshots", - "AuthorEmail", - "AuthorName", - "AuthorPhone", - "AuthorWebSite", - "Bitcoin", - "Liberapay", - "Litecoin", - "OpenCollective", - ): - if element in app and app[element]: - element_new = element[:1].lower() + element[1:] - meta[element_new] = convert_datetime(app[element]) - - for element in ( - "Name", - "Summary", - "Description", - "video", - ): - element_new = element[:1].lower() + element[1:] - if element in app and app[element]: - meta[element_new] = {DEFAULT_LOCALE: convert_datetime(app[element])} - elif "localized" in app: - localized = {k: v[element_new] for k, v in app["localized"].items() if element_new in v} - if localized: - meta[element_new] = localized - - if "name" not in meta and app["AutoName"]: - meta["name"] = {DEFAULT_LOCALE: app["AutoName"]} - - # fdroidserver/metadata.py App default - if meta["license"] == "Unknown": - del meta["license"] - - if app["Donate"]: - meta["donate"] = [app["Donate"]] - - # TODO handle different resolutions - if app.get("icon"): - icon_path = os.path.join(repodir, "icons", app["icon"]) - meta["icon"] = {DEFAULT_LOCALE: common.file_entry(icon_path)} - - if "iconv2" in app: - meta["icon"] = app["iconv2"] - - return meta - - -def convert_version(version, app, repodir): - """Convert the internal representation of Builds: into index-v2 versions. - - The diff algorithm of index-v2 uses null/None to mean a field to - be removed, so this function handles any Nones that are in the - metadata file. - - """ - ver = {} - if "added" in version: - ver["added"] = convert_datetime(version["added"]) - else: - ver["added"] = 0 - - ver["file"] = { - "name": "/{}".format(version["apkName"]), - version["hashType"]: version["hash"], - "size": version["size"], - } - - ipfsCIDv1 = version.get("ipfsCIDv1") - if ipfsCIDv1: - ver["file"]["ipfsCIDv1"] = ipfsCIDv1 - - if "srcname" in version: - ver["src"] = common.file_entry( - os.path.join(repodir, version["srcname"]), - version["srcnameSha256"], - ) - - if "obbMainFile" in version: - ver["obbMainFile"] = common.file_entry( - os.path.join(repodir, version["obbMainFile"]), - version["obbMainFileSha256"], - ) - - if "obbPatchFile" in version: - ver["obbPatchFile"] = common.file_entry( - os.path.join(repodir, version["obbPatchFile"]), - version["obbPatchFileSha256"], - ) - - ver["manifest"] = manifest = {} - - for element in ( - "nativecode", - "versionName", - "maxSdkVersion", - ): - if element in version: - manifest[element] = version[element] - - if "versionCode" in version: - manifest["versionCode"] = version["versionCode"] - - if "features" in version and version["features"]: - manifest["features"] = features = [] - for feature in version["features"]: - # TODO get version from manifest, default (0) is omitted - # features.append({"name": feature, "version": 1}) - features.append({"name": feature}) - - if "minSdkVersion" in version: - manifest["usesSdk"] = {} - manifest["usesSdk"]["minSdkVersion"] = version["minSdkVersion"] - if "targetSdkVersion" in version: - manifest["usesSdk"]["targetSdkVersion"] = version["targetSdkVersion"] - else: - # https://developer.android.com/guide/topics/manifest/uses-sdk-element.html#target - manifest["usesSdk"]["targetSdkVersion"] = manifest["usesSdk"]["minSdkVersion"] - - if "signer" in version: - manifest["signer"] = {"sha256": [version["signer"]]} - - for element in ("uses-permission", "uses-permission-sdk-23"): - en = element.replace("uses-permission", "usesPermission").replace("-sdk-23", "Sdk23") - if element in version and version[element]: - manifest[en] = [] - for perm in version[element]: - if perm[1]: - manifest[en].append({"name": perm[0], "maxSdkVersion": perm[1]}) - else: - manifest[en].append({"name": perm[0]}) - - # index-v2 has only per-version antifeatures, not per package. - antiFeatures = app.get('AntiFeatures', {}).copy() - for name, descdict in version.get('antiFeatures', dict()).items(): - antiFeatures[name] = descdict - if antiFeatures: - ver['antiFeatures'] = { - k: dict(sorted(antiFeatures[k].items())) for k in sorted(antiFeatures) - } - - if "versionCode" in version: - if version["versionCode"] > app["CurrentVersionCode"]: - ver[RELEASECHANNELS_CONFIG_NAME] = ["Beta"] - - builds = app.get("Builds", []) - - if len(builds) > 0 and version["versionCode"] == builds[-1]["versionCode"]: - if "localized" in app: - localized = {k: v["whatsNew"] for k, v in app["localized"].items() if "whatsNew" in v} - if localized: - ver["whatsNew"] = localized - - for build in builds: - if build['versionCode'] == version['versionCode'] and "whatsNew" in build: - ver["whatsNew"] = build["whatsNew"] - break - - return ver - - -def v2_repo(repodict, repodir, archive): - repo = {} - - repo["name"] = {DEFAULT_LOCALE: repodict["name"]} - repo["description"] = {DEFAULT_LOCALE: repodict["description"]} - repo["icon"] = { - DEFAULT_LOCALE: common.file_entry("%s/icons/%s" % (repodir, repodict["icon"])) - } - - config = common.load_localized_config(CONFIG_CONFIG_NAME, repodir) - if config: - localized_config = config["archive" if archive else "repo"] - if "name" in localized_config: - repo["name"] = localized_config["name"] - if "description" in localized_config: - repo["description"] = localized_config["description"] - if "icon" in localized_config: - repo["icon"] = localized_config["icon"] - - repo["address"] = repodict["address"] - if "mirrors" in repodict: - repo["mirrors"] = repodict["mirrors"] - if "webBaseUrl" in repodict: - repo["webBaseUrl"] = repodict["webBaseUrl"] - - repo["timestamp"] = repodict["timestamp"] - - antiFeatures = common.load_localized_config(ANTIFEATURES_CONFIG_NAME, repodir) - if antiFeatures: - repo[ANTIFEATURES_CONFIG_NAME] = antiFeatures - - categories = common.load_localized_config(CATEGORIES_CONFIG_NAME, repodir) - if categories: - repo[CATEGORIES_CONFIG_NAME] = categories - - releaseChannels = common.load_localized_config(RELEASECHANNELS_CONFIG_NAME, repodir) - if releaseChannels: - repo[RELEASECHANNELS_CONFIG_NAME] = releaseChannels - - return repo - - -def make_v2(apps, packages, repodir, repodict, requestsdict, signer_fingerprints, archive): +def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints): def _index_encoder_default(obj): if isinstance(obj, set): @@ -742,176 +144,25 @@ def make_v2(apps, packages, repodir, repodict, requestsdict, signer_fingerprints raise TypeError(repr(obj) + " is not JSON serializable") output = collections.OrderedDict() - output["repo"] = v2_repo(repodict, repodir, archive) - if requestsdict and (requestsdict["install"] or requestsdict["uninstall"]): - output["repo"]["requests"] = requestsdict - - # establish sort order of the index - sort_package_versions(packages, signer_fingerprints) - - output_packages = collections.OrderedDict() - output['packages'] = output_packages - categories_used_by_apps = set() - for package in packages: - packageName = package['packageName'] - if packageName not in apps: - logging.info(_('Ignoring package without metadata: ') + package['apkName']) - continue - if not package.get('versionName'): - app = apps[packageName] - for build in app.get('Builds', []): - if build['versionCode'] == package['versionCode']: - versionName = build.get('versionName') - logging.info(_('Overriding blank versionName in {apkfilename} from metadata: {version}') - .format(apkfilename=package['apkName'], version=versionName)) - package['versionName'] = versionName - break - if packageName in output_packages: - packagelist = output_packages[packageName] - else: - packagelist = {} - output_packages[packageName] = packagelist - app = apps[packageName] - categories_used_by_apps.update(app.get('Categories', [])) - packagelist["metadata"] = package_metadata(app, repodir) - if "signer" in package: - packagelist["metadata"]["preferredSigner"] = package["signer"] - - packagelist["versions"] = {} - - packagelist["versions"][package["hash"]] = convert_version(package, apps[packageName], repodir) - - if categories_used_by_apps and not output['repo'].get(CATEGORIES_CONFIG_NAME): - output['repo'][CATEGORIES_CONFIG_NAME] = dict() - # include definitions for "auto-defined" categories, e.g. just used in app metadata - for category in sorted(categories_used_by_apps): - if category not in output['repo'][CATEGORIES_CONFIG_NAME]: - output['repo'][CATEGORIES_CONFIG_NAME][category] = dict() - if 'name' not in output['repo'][CATEGORIES_CONFIG_NAME][category]: - output['repo'][CATEGORIES_CONFIG_NAME][category]['name'] = {DEFAULT_LOCALE: category} - # do not include defined categories if no apps use them - for category in list(output['repo'].get(CATEGORIES_CONFIG_NAME, list())): - if category not in categories_used_by_apps: - del output['repo'][CATEGORIES_CONFIG_NAME][category] - msg = _('Category "{category}" defined but not used for any apps!') - logging.warning(msg.format(category=category)) - - entry = {} - entry["timestamp"] = repodict["timestamp"] - - entry["version"] = repodict["version"] - if "maxage" in repodict: - entry["maxAge"] = repodict["maxage"] - - json_name = 'index-v2.json' - index_file = os.path.join(repodir, json_name) - with open(index_file, "w", encoding="utf-8") as fp: - if common.options.pretty: - json.dump(output, fp, default=_index_encoder_default, indent=2, ensure_ascii=False) - else: - json.dump(output, fp, default=_index_encoder_default, ensure_ascii=False) - - json_name = "tmp/{}_{}.json".format(repodir, convert_datetime(repodict["timestamp"])) - with open(json_name, "w", encoding="utf-8") as fp: - if common.options.pretty: - json.dump(output, fp, default=_index_encoder_default, indent=2, ensure_ascii=False) - else: - json.dump(output, fp, default=_index_encoder_default, ensure_ascii=False) - - entry["index"] = common.file_entry(index_file) - entry["index"]["numPackages"] = len(output.get("packages", [])) - - indexes = sorted(Path().glob("tmp/{}*.json".format(repodir)), key=lambda x: x.name) - indexes.pop() # remove current index - # remove older indexes - while len(indexes) > 10: - indexes.pop(0).unlink() - - indexes = [json.loads(Path(fn).read_text(encoding="utf-8")) for fn in indexes] - - for diff in Path().glob("{}/diff/*.json".format(repodir)): - diff.unlink() - - entry["diffs"] = {} - for old in indexes: - diff_name = str(old["repo"]["timestamp"]) + ".json" - diff_file = os.path.join(repodir, "diff", diff_name) - diff = dict_diff(old, output) - if not os.path.exists(os.path.join(repodir, "diff")): - os.makedirs(os.path.join(repodir, "diff")) - with open(diff_file, "w", encoding="utf-8") as fp: - if common.options.pretty: - json.dump(diff, fp, default=_index_encoder_default, indent=2, ensure_ascii=False) - else: - json.dump(diff, fp, default=_index_encoder_default, ensure_ascii=False) - - entry["diffs"][old["repo"]["timestamp"]] = common.file_entry(diff_file) - entry["diffs"][old["repo"]["timestamp"]]["numPackages"] = len(diff.get("packages", [])) - - json_name = "entry.json" - index_file = os.path.join(repodir, json_name) - with open(index_file, "w", encoding="utf-8") as fp: - if common.options.pretty: - json.dump(entry, fp, default=_index_encoder_default, indent=2, ensure_ascii=False) - else: - json.dump(entry, fp, default=_index_encoder_default, ensure_ascii=False) - - if common.options.nosign: - _copy_to_local_copy_dir(repodir, index_file) - logging.debug(_('index-v2 must have a signature, use `fdroid signindex` to create it!')) - else: - signindex.config = common.config - signindex.sign_index(repodir, json_name) - - -def make_v1(apps, packages, repodir, repodict, requestsdict, signer_fingerprints): - - def _index_encoder_default(obj): - if isinstance(obj, set): - return sorted(list(obj)) - if isinstance(obj, datetime): - # Java prefers milliseconds - # we also need to account for time zone/daylight saving time - return int(calendar.timegm(obj.timetuple()) * 1000) - if isinstance(obj, dict): - d = collections.OrderedDict() - for key in sorted(obj.keys()): - d[key] = obj[key] - return d - raise TypeError(repr(obj) + " is not JSON serializable") - - output = collections.OrderedDict() - output['repo'] = repodict.copy() + output['repo'] = repodict output['requests'] = requestsdict - # index-v1 only supports a list of URL strings for additional mirrors - mirrors = [] - for mirror in repodict.get('mirrors', []): - url = mirror['url'] - if url != repodict['address']: - mirrors.append(mirror['url']) - if mirrors: - output['repo']['mirrors'] = mirrors - # establish sort order of the index - sort_package_versions(packages, signer_fingerprints) + v1_sort_packages(packages, fdroid_signing_key_fingerprints) appslist = [] output['apps'] = appslist - for packageName, app_dict in apps.items(): + for packageName, appdict in apps.items(): d = collections.OrderedDict() appslist.append(d) - for k, v in sorted(app_dict.items()): + for k, v in sorted(appdict.items()): if not v: continue - if k in ('Builds', 'metadatapath', - 'ArchivePolicy', 'AutoName', 'AutoUpdateMode', 'MaintainerNotes', + if k in ('builds', 'comments', 'metadatapath', + 'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes', 'Provides', 'Repo', 'RepoType', 'RequiresRoot', 'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode', - 'UpdateCheckName', 'NoSourceSince', 'VercodeOperation', - 'summary', 'description', 'promoGraphic', 'screenshots', 'whatsNew', - 'featureGraphic', 'iconv2', 'tvBanner', - ): + 'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'): continue # name things after the App class fields in fdroidclient @@ -919,27 +170,26 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, signer_fingerprints k = 'packageName' elif k == 'CurrentVersionCode': # TODO make SuggestedVersionCode the canonical name k = 'suggestedVersionCode' - v = str(v) elif k == 'CurrentVersion': # TODO make SuggestedVersionName the canonical name k = 'suggestedVersionName' + elif k == 'AutoName': + if 'Name' not in apps[packageName]: + d['name'] = v + continue else: k = k[:1].lower() + k[1:] d[k] = v - # establish sort order in lists, sets, and localized dicts - for app_dict in output['apps']: - localized = app_dict.get('localized') + # establish sort order in localized dicts + for app in output['apps']: + localized = app.get('localized') if localized: lordered = collections.OrderedDict() for lkey, lvalue in sorted(localized.items()): lordered[lkey] = collections.OrderedDict() for ikey, iname in sorted(lvalue.items()): lordered[lkey][ikey] = iname - app_dict['localized'] = lordered - # v1 uses a list of keys for Anti-Features - antiFeatures = app_dict.get('antiFeatures', dict()).keys() - if antiFeatures: - app_dict['antiFeatures'] = sorted(set(antiFeatures)) + app['localized'] = lordered output_packages = collections.OrderedDict() output['packages'] = output_packages @@ -950,8 +200,9 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, signer_fingerprints continue if not package.get('versionName'): app = apps[packageName] - for build in app.get('Builds', []): - if build['versionCode'] == package['versionCode']: + versionCodeStr = str(package['versionCode']) # TODO build.versionCode should be int! + for build in app['builds']: + if build['versionCode'] == versionCodeStr: versionName = build.get('versionName') logging.info(_('Overriding blank versionName in {apkfilename} from metadata: {version}') .format(apkfilename=package['apkName'], version=versionName)) @@ -967,10 +218,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, signer_fingerprints for k, v in sorted(package.items()): if not v: continue - if k in ('icon', 'icons', 'icons_src', 'ipfsCIDv1', 'name', 'srcnameSha256'): - continue - if k == 'antiFeatures': - d[k] = sorted(v.keys()) + if k in ('icon', 'icons', 'icons_src', 'name', ): continue d[k] = v @@ -983,37 +231,21 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, signer_fingerprints json.dump(output, fp, default=_index_encoder_default) if common.options.nosign: - _copy_to_local_copy_dir(repodir, index_file) logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!')) else: signindex.config = common.config - signindex.sign_index(repodir, json_name) + signindex.sign_index_v1(repodir, json_name) -def _copy_to_local_copy_dir(repodir, f): - local_copy_dir = common.config.get('local_copy_dir', '') - if os.path.exists(local_copy_dir): - destdir = os.path.join(local_copy_dir, repodir) - if not os.path.exists(destdir): - os.mkdir(destdir) - shutil.copy2(f, destdir, follow_symlinks=False) - elif local_copy_dir: - raise FDroidException(_('"local_copy_dir" {path} does not exist!') - .format(path=local_copy_dir)) - - -def sort_package_versions(packages, signer_fingerprints): - """Sort to ensure a deterministic order for package versions in the index file. - - This sort-order also expresses +def v1_sort_packages(packages, fdroid_signing_key_fingerprints): + """Sorts the supplied list to ensure a deterministic sort order for + package entries in the index file. This sort-order also expresses installation preference to the clients. (First in this list = first to install) - Parameters - ---------- - packages - list of packages which need to be sorted before but into index file. + :param packages: list of packages which need to be sorted before but into index file. """ + GROUP_DEV_SIGNED = 1 GROUP_FDROID_SIGNED = 2 GROUP_OTHER_SIGNED = 3 @@ -1021,28 +253,31 @@ def sort_package_versions(packages, signer_fingerprints): def v1_sort_keys(package): packageName = package.get('packageName', None) - signer = package.get('signer', None) + sig = package.get('signer', None) - dev_signer = common.metadata_find_developer_signature(packageName) + dev_sig = common.metadata_find_developer_signature(packageName) group = GROUP_OTHER_SIGNED - if dev_signer and dev_signer == signer: + if dev_sig and dev_sig == sig: group = GROUP_DEV_SIGNED else: - fdroid_signer = signer_fingerprints.get(packageName, {}).get('signer') - if fdroid_signer and fdroid_signer == signer: + fdroidsig = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer') + if fdroidsig and fdroidsig == sig: group = GROUP_FDROID_SIGNED versionCode = None if package.get('versionCode', None): - versionCode = -package['versionCode'] + versionCode = -int(package['versionCode']) - return packageName, group, signer, versionCode + return(packageName, group, sig, versionCode) packages.sort(key=v1_sort_keys) -def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): - """Aka index.jar aka index.xml.""" +def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints): + """ + aka index.jar aka index.xml + """ + doc = Document() def addElement(name, value, doc, parent): @@ -1062,7 +297,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): addElement(name, value, doc, parent) def addElementCheckLocalized(name, app, key, doc, parent, default=''): - """Fill in field from metadata or localized block. + """Fill in field from metadata or localized block For name/summary/description, they can come only from the app source, or from a dir in fdroiddata. They can be entirely missing from the @@ -1073,12 +308,13 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): alpha- sort order. """ + el = doc.createElement(name) value = app.get(key) lkey = key[:1].lower() + key[1:] localized = app.get('localized') if not value and localized: - for lang in [DEFAULT_LOCALE] + [x for x in localized.keys()]: + for lang in ['en-US'] + [x for x in localized.keys()]: if not lang.startswith('en'): continue if lang in localized: @@ -1090,8 +326,6 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): value = localized[lang].get(lkey) if not value: value = default - if not value and name == 'name' and app.get('AutoName'): - value = app['AutoName'] el.appendChild(doc.createTextNode(value)) parent.appendChild(el) @@ -1099,7 +333,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): doc.appendChild(root) repoel = doc.createElement("repo") - repoel.setAttribute("icon", repodict['icon']) + repoel.setAttribute("icon", os.path.basename(repodict['icon'])) if 'maxage' in repodict: repoel.setAttribute("maxage", str(repodict['maxage'])) repoel.setAttribute("name", repodict['name']) @@ -1110,11 +344,8 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): repoel.setAttribute("version", str(repodict['version'])) addElement('description', repodict['description'], doc, repoel) - # index v0 only supports a list of URL strings for additional mirrors for mirror in repodict.get('mirrors', []): - url = mirror['url'] - if url != repodict['address']: - addElement('mirror', url, doc, repoel) + addElement('mirror', mirror, doc, repoel) root.appendChild(repoel) @@ -1124,27 +355,24 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): root.appendChild(element) element.setAttribute('packageName', packageName) - for appid, app_dict in apps.items(): - app = metadata.App(app_dict) + for appid, appdict in apps.items(): + app = metadata.App(appdict) - if app.get('Disabled') is not None: + if app.Disabled is not None: continue # Get a list of the apks for this app... apklist = [] - name_from_apk = None apksbyversion = collections.defaultdict(lambda: []) for apk in apks: if apk.get('versionCode') and apk.get('packageName') == appid: apksbyversion[apk['versionCode']].append(apk) - if name_from_apk is None: - name_from_apk = apk.get('name') for versionCode, apksforver in apksbyversion.items(): - fdroid_signer = signer_fingerprints.get(appid, {}).get('signer') + fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer') fdroid_signed_apk = None name_match_apk = None for x in apksforver: - if fdroid_signer and x.get('signer', None) == fdroid_signer: + if fdroidsig and x.get('signer', None) == fdroidsig: fdroid_signed_apk = x if common.apk_release_filename.match(x.get('apkName', '')): name_match_apk = x @@ -1170,7 +398,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): if app.lastUpdated: addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel) - addElementCheckLocalized('name', app, 'Name', doc, apel, name_from_apk) + addElementCheckLocalized('name', app, 'Name', doc, apel) addElementCheckLocalized('summary', app, 'Summary', doc, apel) if app.icon: @@ -1195,13 +423,15 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): addElementNonEmpty('donate', app.Donate, doc, apel) addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel) addElementNonEmpty('litecoin', app.Litecoin, doc, apel) + addElementNonEmpty('flattr', app.FlattrID, doc, apel) + addElementNonEmpty('liberapay', app.LiberapayID, doc, apel) addElementNonEmpty('openCollective', app.OpenCollective, doc, apel) # These elements actually refer to the current version (i.e. which # one is recommended. They are historically mis-named, and need # changing, but stay like this for now to support existing clients. addElement('marketversion', app.CurrentVersion, doc, apel) - addElement('marketvercode', str(app.CurrentVersionCode), doc, apel) + addElement('marketvercode', app.CurrentVersionCode, doc, apel) if app.Provides: pv = app.Provides.split(',') @@ -1209,16 +439,14 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): if app.RequiresRoot: addElement('requirements', 'root', doc, apel) - # Sort the APK list into version order, just so the web site + # Sort the apk list into version order, just so the web site # doesn't have to do any work by default... apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True) - antiFeatures = list(app.AntiFeatures) if 'antiFeatures' in apklist[0]: - antiFeatures.extend(apklist[0]['antiFeatures']) - if antiFeatures: - afout = sorted(set(antiFeatures)) - addElementNonEmpty('antifeatures', ','.join(afout), doc, apel) + app.AntiFeatures.extend(apklist[0]['antiFeatures']) + if app.AntiFeatures: + addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel) # Check for duplicates - they will make the client unhappy... for i in range(len(apklist) - 1): @@ -1238,21 +466,19 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): for apk in apklist: file_extension = common.get_file_extension(apk['apkName']) # find the APK for the "Current Version" - if current_version_code < app.CurrentVersionCode: - current_version_file = apk['apkName'] if current_version_code < apk['versionCode']: current_version_code = apk['versionCode'] + if current_version_code < int(app.CurrentVersionCode): + current_version_file = apk['apkName'] apkel = doc.createElement("package") apel.appendChild(apkel) versionName = apk.get('versionName') if not versionName: - for build in app.get('Builds', []): - if ( - build['versionCode'] == apk['versionCode'] - and 'versionName' in build - ): + versionCodeStr = str(apk['versionCode']) # TODO build.versionCode should be int! + for build in app.builds: + if build['versionCode'] == versionCodeStr and 'versionName' in build: versionName = build['versionName'] break if versionName: @@ -1317,12 +543,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): and common.config['make_current_version_link'] \ and repodir == 'repo': # only create these namefield = common.config['current_version_name_source'] - name = app.get(namefield) - if not name and namefield == 'Name': - name = app.get('localized', {}).get(DEFAULT_LOCALE, {}).get('name') - if not name: - name = app.id - sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', str(name).encode('utf-8')) + sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8')) apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8') current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape') if os.path.islink(apklinkname): @@ -1337,29 +558,6 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): os.remove(siglinkname) os.symlink(sigfile_path, siglinkname) - if sys.version_info.minor >= 13: - # Python 3.13 changed minidom so it no longer converts " to an XML entity. - # https://github.com/python/cpython/commit/154477be722ae5c4e18d22d0860e284006b09c4f - # This just puts back the previous implementation, with black code format. - import inspect - import xml.dom.minidom - - def _write_data(writer, text, attr): # pylint: disable=unused-argument - if text: - text = ( - text.replace('&', '&') - .replace('<', '<') - .replace('"', '"') - .replace('>', '>') - ) - writer.write(text) - - argnames = tuple(inspect.signature(xml.dom.minidom._write_data).parameters) - if argnames == ('writer', 'text', 'attr'): - xml.dom.minidom._write_data = _write_data - else: - logging.warning('Failed to monkey patch minidom for index.xml support!') - if common.options.pretty: output = doc.toprettyxml(encoding='utf-8') else: @@ -1386,13 +584,12 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): # Sign the index... signed = os.path.join(repodir, 'index.jar') if common.options.nosign: - _copy_to_local_copy_dir(repodir, os.path.join(repodir, jar_output)) # Remove old signed index if not signing if os.path.exists(signed): os.remove(signed) else: signindex.config = common.config - signindex.sign_jar(signed, use_old_algs=True) + signindex.sign_jar(signed) # Copy the repo icon into the repo directory... icon_dir = os.path.join(repodir, 'icons') @@ -1401,19 +598,12 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): if os.path.exists(repo_icon): shutil.copyfile(common.config['repo_icon'], iconfilename) else: - logging.warning(_('repo_icon "repo/icons/%s" does not exist, generating placeholder.') + logging.warning(_('repo_icon %s does not exist, generating placeholder.') % repo_icon) os.makedirs(os.path.dirname(iconfilename), exist_ok=True) try: import qrcode - qrcode.make(common.config['repo_url']).save(iconfilename) - except ModuleNotFoundError as e: - raise ModuleNotFoundError( - _( - 'The "qrcode" Python package is not installed (e.g. apt-get install python3-qrcode)!' - ) - ) from e except Exception: exampleicon = os.path.join(common.get_examples_dir(), common.default_config['repo_icon']) @@ -1421,16 +611,13 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, signer_fingerprints): def extract_pubkey(): - """Extract and return the repository's public key from the keystore. - - Returns - ------- - public key in hex - repository fingerprint + """ + Extracts and returns the repository's public key from the keystore. + :return: public key in hex, repository fingerprint """ if 'repo_pubkey' in common.config: pubkey = unhexlify(common.config['repo_pubkey']) - elif 'keystorepass' in common.config: + else: env_vars = {'LC_ALL': 'C.UTF-8', 'FDROID_KEY_STORE_PASS': common.config['keystorepass']} p = FDroidPopenBytes([common.config['keytool'], '-exportcert', @@ -1445,142 +632,23 @@ def extract_pubkey(): msg += ' Is your crypto smartcard plugged in?' raise FDroidException(msg) pubkey = p.output - else: - raise FDroidException(_('Neither "repo_pubkey" nor "keystorepass" set in config.yml')) - repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey) return hexlify(pubkey), repo_pubkey_fingerprint -def add_mirrors_to_repodict(repo_section, repodict): - """Convert config into final dict of mirror metadata for the repo. - - Internally and in index-v2, mirrors is a list of dicts, but it can - be specified in the config as a string or list of strings. Also, - index v0 and v1 use a list of URL strings as the data structure. - - The first entry is traditionally the primary mirror and canonical - URL. 'mirrors' should not be present in the index if there is - only the canonical URL, and no other mirrors. - - The metadata items for each mirror entry are sorted by key to - ensure minimum diffs in the index files. - - """ - mirrors_config = common.config.get('mirrors', []) - if type(mirrors_config) not in (list, tuple): - mirrors_config = [mirrors_config] - - mirrors_yml = Path(f'config/{MIRRORS_CONFIG_NAME}.yml') - if mirrors_yml.exists(): - if mirrors_config: - raise FDroidException( - _('mirrors set twice, in config.yml and {path}!').format( - path=mirrors_yml - ) - ) - with mirrors_yml.open() as fp: - mirrors_config = yaml.load(fp) - if not isinstance(mirrors_config, list): - msg = _('{path} is not list, but a {datatype}!') - raise TypeError( - msg.format(path=mirrors_yml, datatype=type(mirrors_config).__name__) - ) - - if type(mirrors_config) not in (list, tuple, set): - msg = 'In config.yml, mirrors: is not list, but a {datatype}!' - raise TypeError(msg.format(datatype=type(mirrors_config).__name__)) - - mirrorcheckfailed = False - mirrors = [] - urls = set() - for mirror in mirrors_config: - if isinstance(mirror, str): - mirror = {'url': mirror} - elif not isinstance(mirror, dict): - logging.error( - _('Bad entry type "{mirrortype}" in mirrors config: {mirror}').format( - mirrortype=type(mirror), mirror=mirror - ) - ) - mirrorcheckfailed = True - continue - config_url = mirror['url'] - base = os.path.basename(urllib.parse.urlparse(config_url).path.rstrip('/')) - if common.config.get('nonstandardwebroot') is not True and base != 'fdroid': - logging.error(_("mirror '%s' does not end with 'fdroid'!") % config_url) - mirrorcheckfailed = True - # must end with / or urljoin strips a whole path segment - if config_url.endswith('/'): - mirror['url'] = urllib.parse.urljoin(config_url, repo_section) - else: - mirror['url'] = urllib.parse.urljoin(config_url + '/', repo_section) - mirrors.append(mirror) - if mirror['url'] in urls: - mirrorcheckfailed = True - logging.error( - _('Duplicate entry "%s" in mirrors config!') % mirror['url'] - ) - urls.add(mirror['url']) - for mirror in common.config.get('servergitmirrors', []): - for url in get_mirror_service_urls(mirror): - mirrors.append({'url': url + '/' + repo_section}) - if mirrorcheckfailed: - raise FDroidException(_("Malformed repository mirrors.")) - - if not mirrors: - return - - repodict['mirrors'] = [] - canonical_url = repodict['address'] - found_primary = False - errors = 0 - for mirror in mirrors: - if canonical_url == mirror['url']: - found_primary = True - mirror['isPrimary'] = True - sortedmirror = dict() - for k in sorted(mirror.keys()): - sortedmirror[k] = mirror[k] - repodict['mirrors'].insert(0, sortedmirror) - elif mirror.get('isPrimary'): - errors += 1 - logging.error( - _('Mirror config for {url} contains "isPrimary" key!').format( - url=mirror['url'] - ) - ) - else: - repodict['mirrors'].append(mirror) - - if errors: - raise FDroidException(_('"isPrimary" key should not be added to mirrors!')) - - if repodict['mirrors'] and not found_primary: - repodict['mirrors'].insert(0, {'isPrimary': True, 'url': repodict['address']}) - - -def get_mirror_service_urls(mirror): - """Get direct URLs from git service for use by fdroidclient. +def get_mirror_service_urls(url): + '''Get direct URLs from git service for use by fdroidclient Via 'servergitmirrors', fdroidserver can create and push a mirror - to certain well known git services like GitLab or GitHub. This + to certain well known git services like gitlab or github. This will always use the 'master' branch since that is the default branch in git. The files are then accessible via alternate URLs, where they are served in their raw format via a CDN rather than from git. + ''' - Both of the GitLab URLs will work with F-Droid, but only the - GitLab Pages will work in the browser This is because the "raw" - URLs are not served with the correct mime types, so any index.html - which is put in the repo will not be rendered. Putting an - index.html file in the repo root is a common way for to make - information about the repo available to end user. - - """ - url = mirror['url'] if url.startswith('git@'): - url = re.sub(r'^git@([^:]+):(.+)', r'https://\1/\2', url) + url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url) segments = url.split("/") @@ -1600,78 +668,36 @@ def get_mirror_service_urls(mirror): segments.extend([branch, folder]) urls.append('/'.join(segments)) elif hostname == "gitlab.com": - git_mirror_path = os.path.join('git-mirror', folder) - if ( - mirror.get('index_only') - or common.get_dir_size(git_mirror_path) <= common.GITLAB_COM_PAGES_MAX_SIZE - ): - # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder" - gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder] - urls.append('/'.join(gitlab_pages)) - else: - logging.warning( - _( - 'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!' - ) - % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) - ) - # GitLab Raw "https://gitlab.com/user/repo/-/raw/branch/folder" - gitlab_raw = segments + ['-', 'raw', branch, folder] + # Both these Gitlab URLs will work with F-Droid, but only the first will work in the browser + # This is because the `raw` URLs are not served with the correct mime types, so any + # index.html which is put in the repo will not be rendered. Putting an index.html file in + # the repo root is a common way for to make information about the repo available to end user. + + # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder" + gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder] + urls.append('/'.join(gitlab_pages)) + # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder" + gitlab_raw = segments + ['raw', branch, folder] urls.append('/'.join(gitlab_raw)) - # GitLab Artifacts "https://user.gitlab.io/-/repo/-/jobs/job_id/artifacts/public/folder" - job_id = os.getenv('CI_JOB_ID') - try: - int(job_id) - gitlab_artifacts = [ - "https:", - "", - user + ".gitlab.io", - '-', - repo, - '-', - 'jobs', - job_id, - 'artifacts', - 'public', - folder, - ] - urls.append('/'.join(gitlab_artifacts)) - except (TypeError, ValueError): - pass # no Job ID to use, ignore + return urls return urls def download_repo_index(url_str, etag=None, verify_fingerprint=True, timeout=600): - """Download and verifies index v1 file, then returns its data. - - Use the versioned functions to be sure you are getting the - expected data format. - - """ - return download_repo_index_v1(url_str, etag, verify_fingerprint, timeout) - - -def download_repo_index_v1(url_str, etag=None, verify_fingerprint=True, timeout=600): - """Download and verifies index v1 file, then returns its data. + """Downloads and verifies index file, then returns its data. Downloads the repository index from the given :param url_str and verifies the repository's fingerprint if :param verify_fingerprint is not False. - Raises - ------ - VerificationException() if the repository could not be verified + :raises: VerificationException() if the repository could not be verified - Returns - ------- - A tuple consisting of: - - The index in JSON v1 format or None if the index did not change + :return: A tuple consisting of: + - The index in JSON format or None if the index did not change - The new eTag as returned by the HTTP request """ - from . import net - url = urllib.parse.urlsplit(url_str) fingerprint = None @@ -1681,12 +707,7 @@ def download_repo_index_v1(url_str, etag=None, verify_fingerprint=True, timeout= raise VerificationException(_("No fingerprint in URL.")) fingerprint = query['fingerprint'][0] - if url.path.endswith('/index-v1.jar'): - path = url.path[:-13].rstrip('/') - else: - path = url.path.rstrip('/') - - url = urllib.parse.SplitResult(url.scheme, url.netloc, path + '/index-v1.jar', '', '') + url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '') download, new_etag = net.http_get(url.geturl(), etag, timeout) if download is None: @@ -1695,127 +716,38 @@ def download_repo_index_v1(url_str, etag=None, verify_fingerprint=True, timeout= with tempfile.NamedTemporaryFile() as fp: fp.write(download) fp.flush() - index, public_key, public_key_fingerprint = get_index_from_jar( - fp.name, fingerprint, allow_deprecated=True - ) + index, public_key, public_key_fingerprint = get_index_from_jar(fp.name, fingerprint) index["repo"]["pubkey"] = hexlify(public_key).decode() index["repo"]["fingerprint"] = public_key_fingerprint index["apps"] = [metadata.App(app) for app in index["apps"]] return index, new_etag -def download_repo_index_v2(url_str, etag=None, verify_fingerprint=True, timeout=None): - """Download and verifies index v2 file, then returns its data. - - Downloads the repository index from the given :param url_str and - verifies the repository's fingerprint if :param verify_fingerprint - is not False. In order to verify the data, the fingerprint must - be provided as part of the URL. - - Raises - ------ - VerificationException() if the repository could not be verified - - Returns - ------- - A tuple consisting of: - - The index in JSON v2 format or None if the index did not change - - The new eTag as returned by the HTTP request +def get_index_from_jar(jarfile, fingerprint=None): + """Returns the data, public key, and fingerprint from index-v1.jar + :raises: VerificationException() if the repository could not be verified """ - from . import net - etag # etag is unused but needs to be there to keep the same API as the earlier functions. - - url = urllib.parse.urlsplit(url_str) - - if timeout is not None: - logging.warning('"timeout" argument of download_repo_index_v2() is deprecated!') - - fingerprint = None - if verify_fingerprint: - query = urllib.parse.parse_qs(url.query) - if 'fingerprint' not in query: - raise VerificationException(_("No fingerprint in URL.")) - fingerprint = query['fingerprint'][0] - - if url.path.endswith('/entry.jar') or url.path.endswith('/index-v2.json'): - path = url.path.rsplit('/', 1)[0] - else: - path = url.path.rstrip('/') - url = urllib.parse.SplitResult(url.scheme, url.netloc, path, '', '') - - mirrors = common.get_mirrors(url, 'entry.jar') - f = net.download_using_mirrors(mirrors) - entry, public_key, fingerprint = get_index_from_jar(f, fingerprint) - - sha256 = entry['index']['sha256'] - mirrors = common.get_mirrors(url, entry['index']['name'][1:]) - f = net.download_using_mirrors(mirrors) - with open(f, 'rb') as fp: - index = fp.read() - if sha256 != hashlib.sha256(index).hexdigest(): - raise VerificationException( - _("SHA-256 of {url} does not match entry!").format(url=url) - ) - return json.loads(index), None - - -def get_index_from_jar(jarfile, fingerprint=None, allow_deprecated=False): - """Return the data, public key and fingerprint from an index JAR with one JSON file. - - The F-Droid index files always contain a single data file and a - JAR Signature. Since index-v1, the data file is always JSON. - That single data file is named the same as the JAR file. - - Parameters - ---------- - fingerprint is the SHA-256 fingerprint of signing key. Only - hex digits count, all other chars will can be discarded. - - Raises - ------ - VerificationException() if the repository could not be verified - - """ logging.debug(_('Verifying index signature:')) - - if allow_deprecated: - common.verify_deprecated_jar_signature(jarfile) - else: - common.verify_jar_signature(jarfile) - + common.verify_jar_signature(jarfile) with zipfile.ZipFile(jarfile) as jar: public_key, public_key_fingerprint = get_public_key_from_jar(jar) if fingerprint is not None: - fingerprint = re.sub(r'[^0-9A-F]', r'', fingerprint.upper()) - if fingerprint != public_key_fingerprint: - raise VerificationException( - _("The repository's fingerprint does not match.") - ) - for f in jar.namelist(): - if not f.startswith('META-INF/'): - jsonfile = f - break - data = json.loads(jar.read(jsonfile)) + if fingerprint.upper() != public_key_fingerprint: + raise VerificationException(_("The repository's fingerprint does not match.")) + data = json.loads(jar.read('index-v1.json').decode()) return data, public_key, public_key_fingerprint def get_public_key_from_jar(jar): - """Get the public key and its fingerprint from a JAR file. + """ + Get the public key and its fingerprint from a JAR file. - Raises - ------ - VerificationException() if the JAR was not signed exactly once + :raises: VerificationException() if the JAR was not signed exactly once - Parameters - ---------- - jar - a zipfile.ZipFile object - - Returns - ------- - the public key from the jar and its fingerprint + :param jar: a zipfile.ZipFile object + :return: the public key from the jar and its fingerprint """ # extract certificate from jar certs = [n for n in jar.namelist() if common.SIGNATURE_BLOCK_FILE_REGEX.match(n)] @@ -1829,100 +761,3 @@ def get_public_key_from_jar(jar): public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '') return public_key, public_key_fingerprint - - -def make_altstore(apps, apks, config, repodir, pretty=False): - """Assemble altstore-index.json for iOS (.ipa) apps. - - builds index files based on: - https://faq.altstore.io/distribute-your-apps/make-a-source - https://faq.altstore.io/distribute-your-apps/updating-apps - """ - if not any(Path(repodir).glob('*.ipa')): - # no IPA files present in repo, nothing to do here, exiting early - return - - indent = 2 if pretty else None - # for now alt-store support is english only - for lang in ['en']: - - # prepare minimal altstore index - idx = { - 'name': config['repo_name'], - "apps": [], - "news": [], - } - - # add optional values if available - # idx["subtitle"] F-Droid doesn't have a corresponding value - if config.get("repo_description"): - idx['description'] = config['repo_description'] - if (Path(repodir) / 'icons' / config['repo_icon']).exists(): - idx['iconURL'] = f"{config['repo_url']}/icons/{config['repo_icon']}" - # idx["headerURL"] F-Droid doesn't have a corresponding value - # idx["website"] F-Droid doesn't have a corresponding value - # idx["patreonURL"] F-Droid doesn't have a corresponding value - # idx["tintColor"] F-Droid doesn't have a corresponding value - # idx["featuredApps"] = [] maybe mappable to F-Droids what's new? - - # assemble "apps" - for packageName, app in apps.items(): - app_name = app.get("Name") or app.get("AutoName") - icon_url = "{}{}".format( - config['repo_url'], - app.get('iconv2', {}).get(DEFAULT_LOCALE, {}).get('name', ''), - ) - screenshot_urls = [ - "{}{}".format(config["repo_url"], s["name"]) - for s in app.get("screenshots", {}) - .get("phone", {}) - .get(DEFAULT_LOCALE, {}) - ] - - a = { - "name": app_name, - 'bundleIdentifier': packageName, - 'developerName': app.get("AuthorName") or f"{app_name} team", - 'iconURL': icon_url, - "localizedDescription": "", - 'appPermissions': { - "entitlements": set(), - "privacy": {}, - }, - 'versions': [], - } - - if app.get('summary'): - a['subtitle'] = app['summary'] - # a["tintColor"] F-Droid doesn't have a corresponding value - # a["category"] F-Droid doesn't have a corresponding value - # a['patreon'] F-Droid doesn't have a corresponding value - a["screenshots"] = screenshot_urls - - # populate 'versions' - for apk in apks: - last4 = apk.get('apkName', '').lower()[-4:] - if apk['packageName'] == packageName and last4 == '.ipa': - v = { - "version": apk["versionName"], - "date": apk["added"].isoformat(), - "downloadURL": f"{config['repo_url']}/{apk['apkName']}", - "size": apk['size'], - } - - # v['localizedDescription'] maybe what's new text? - v["minOSVersion"] = apk["ipa_MinimumOSVersion"] - v["maxOSVersion"] = apk["ipa_DTPlatformVersion"] - - # writing this spot here has the effect that always the - # permissions of the latest processed permissions list used - a['appPermissions']['privacy'] = apk['ipa_permissions'] - a['appPermissions']['entitlements'] = list(apk['ipa_entitlements']) - - a['versions'].append(v) - - if len(a['versions']) > 0: - idx['apps'].append(a) - - with open(Path(repodir) / 'altstore-index.json', "w", encoding="utf-8") as f: - json.dump(idx, f, indent=indent) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 39b18c1a..53eac796 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -19,70 +19,54 @@ # along with this program. If not, see . import glob -import logging import os import re import shutil import socket import sys from argparse import ArgumentParser +import logging -from . import _, common +from . import _ +from . import common from .exception import FDroidException config = {} +options = None def disable_in_config(key, value): - """Write a key/value to the local config.yml, then comment it out.""" + '''write a key/value to the local config.yml, then comment it out''' import yaml - - with open(common.CONFIG_FILE) as fp: - data = fp.read() + with open('config.yml') as f: + data = f.read() pattern = r'\n[\s#]*' + key + r':.*' repl = '\n#' + yaml.dump({key: value}, default_flow_style=False) data = re.sub(pattern, repl, data) - with open(common.CONFIG_FILE, 'w') as fp: - fp.writelines(data) + with open('config.yml', 'w') as f: + f.writelines(data) def main(): - global config + + global options, config # Parse command line... parser = ArgumentParser() common.setup_global_opts(parser) - parser.add_argument( - "-d", - "--distinguished-name", - default=None, - help=_("X.509 'Distinguished Name' used when generating keys"), - ) - parser.add_argument( - "--keystore", - default=None, - help=_("Path to the keystore for the repo signing key"), - ) - parser.add_argument( - "--repo-keyalias", - default=None, - help=_("Alias of the repo signing key in the keystore"), - ) - parser.add_argument( - "--android-home", - default=None, - help=_("Path to the Android SDK (sometimes set in ANDROID_HOME)"), - ) - parser.add_argument( - "--no-prompt", - action="store_true", - default=False, - help=_("Do not prompt for Android SDK path, just fail"), - ) - options = common.parse_args(parser) - - common.set_console_logging(options.verbose, options.color) + parser.add_argument("-d", "--distinguished-name", default=None, + help=_("X.509 'Distinguished Name' used when generating keys")) + parser.add_argument("--keystore", default=None, + help=_("Path to the keystore for the repo signing key")) + parser.add_argument("--repo-keyalias", default=None, + help=_("Alias of the repo signing key in the keystore")) + parser.add_argument("--android-home", default=None, + help=_("Path to the Android SDK (sometimes set in ANDROID_HOME)")) + parser.add_argument("--no-prompt", action="store_true", default=False, + help=_("Do not prompt for Android SDK path, just fail")) + options = parser.parse_args() + aapt = None fdroiddir = os.getcwd() test_config = dict() examplesdir = common.get_examples_dir() @@ -97,10 +81,9 @@ def main(): # exist, prompt the user using platform-specific default # and if the user leaves it blank, ignore and move on. default_sdk_path = '' - if sys.platform in ('win32', 'cygwin'): - p = os.path.join( - os.getenv('USERPROFILE'), 'AppData', 'Local', 'Android', 'android-sdk' - ) + if sys.platform == 'win32' or sys.platform == 'cygwin': + p = os.path.join(os.getenv('USERPROFILE'), + 'AppData', 'Local', 'Android', 'android-sdk') elif sys.platform == 'darwin': # on OSX, Homebrew is common and has an easy path to detect p = '/usr/local/opt/android-sdk' @@ -114,13 +97,10 @@ def main(): test_config['sdk_path'] = default_sdk_path if not common.test_sdk_exists(test_config): - del test_config['sdk_path'] + del(test_config['sdk_path']) while not options.no_prompt: try: - s = input( - _('Enter the path to the Android SDK (%s) here:\n> ') - % default_sdk_path - ) + s = input(_('Enter the path to the Android SDK (%s) here:\n> ') % default_sdk_path) except KeyboardInterrupt: print('') sys.exit(1) @@ -133,28 +113,16 @@ def main(): default_sdk_path = '' if test_config.get('sdk_path') and not common.test_sdk_exists(test_config): - raise FDroidException( - _("Android SDK not found at {path}!").format(path=test_config['sdk_path']) - ) + raise FDroidException(_("Android SDK not found at {path}!") + .format(path=test_config['sdk_path'])) - if not os.path.exists(common.CONFIG_FILE): + if not os.path.exists('config.yml') and not os.path.exists('config.py'): # 'metadata' and 'tmp' are created in fdroid if not os.path.exists('repo'): os.mkdir('repo') - example_config_yml = os.path.join(examplesdir, common.CONFIG_FILE) - if os.path.exists(example_config_yml): - shutil.copyfile(example_config_yml, common.CONFIG_FILE) - else: - from pkg_resources import get_distribution - - versionstr = get_distribution('fdroidserver').version - if not versionstr: - versionstr = 'master' - with open(common.CONFIG_FILE, 'w') as fp: - fp.write('# see https://gitlab.com/fdroid/fdroidserver/blob/') - fp.write(versionstr) - fp.write(f'/examples/{common.CONFIG_FILE}\n') - os.chmod(common.CONFIG_FILE, 0o0600) + shutil.copy(os.path.join(examplesdir, 'fdroid-icon.png'), fdroiddir) + shutil.copyfile(os.path.join(examplesdir, 'config.yml'), 'config.yml') + os.chmod('config.yml', 0o0600) # If android_home is None, test_config['sdk_path'] will be used and # "$ANDROID_HOME" may be used if the env var is set up correctly. # If android_home is not None, the path given from the command line @@ -162,14 +130,16 @@ def main(): if 'sdk_path' in test_config: common.write_to_config(test_config, 'sdk_path', options.android_home) else: - logging.warning( - 'Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...' - ) + logging.warning('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...') logging.info('Try running `fdroid init` in an empty directory.') raise FDroidException('Repository already exists.') # now that we have a local config.yml, read configuration... - config = common.read_config() + config = common.read_config(options) + + # enable apksigner by default so v2/v3 APK signatures validate + if common.find_apksigner() is not None: + test_config['apksigner'] = common.find_apksigner() # the NDK is optional and there may be multiple versions of it, so it's # left for the user to configure @@ -188,9 +158,8 @@ def main(): else: keystore = os.path.abspath(options.keystore) if not os.path.exists(keystore): - logging.info( - '"' + keystore + '" does not exist, creating a new keystore there.' - ) + logging.info('"' + keystore + + '" does not exist, creating a new keystore there.') common.write_to_config(test_config, 'keystore', keystore) repo_keyalias = None keydname = None @@ -201,19 +170,12 @@ def main(): keydname = options.distinguished_name common.write_to_config(test_config, 'keydname', keydname) if keystore == 'NONE': # we're using a smartcard - common.write_to_config( - test_config, 'repo_keyalias', '1' - ) # seems to be the default + common.write_to_config(test_config, 'repo_keyalias', '1') # seems to be the default disable_in_config('keypass', 'never used with smartcard') - common.write_to_config( - test_config, - 'smartcardoptions', - ( - '-storetype PKCS11 ' - + '-providerClass sun.security.pkcs11.SunPKCS11 ' - + '-providerArg opensc-fdroid.cfg' - ), - ) + common.write_to_config(test_config, 'smartcardoptions', + ('-storetype PKCS11 ' + + '-providerClass sun.security.pkcs11.SunPKCS11 ' + + '-providerArg opensc-fdroid.cfg')) # find opensc-pkcs11.so if not os.path.exists('opensc-fdroid.cfg'): if os.path.exists('/usr/lib/opensc-pkcs11.so'): @@ -221,49 +183,35 @@ def main(): elif os.path.exists('/usr/lib64/opensc-pkcs11.so'): opensc_so = '/usr/lib64/opensc-pkcs11.so' else: - files = glob.glob( - '/usr/lib/' + os.uname()[4] + '-*-gnu/opensc-pkcs11.so' - ) + files = glob.glob('/usr/lib/' + os.uname()[4] + '-*-gnu/opensc-pkcs11.so') if len(files) > 0: opensc_so = files[0] else: opensc_so = '/usr/lib/opensc-pkcs11.so' - logging.warning( - 'No OpenSC PKCS#11 module found, ' - + 'install OpenSC then edit "opensc-fdroid.cfg"!' - ) + logging.warning('No OpenSC PKCS#11 module found, ' + + 'install OpenSC then edit "opensc-fdroid.cfg"!') with open('opensc-fdroid.cfg', 'w') as f: f.write('name = OpenSC\nlibrary = ') f.write(opensc_so) f.write('\n') - logging.info( - "Repo setup using a smartcard HSM. Please edit keystorepass and repo_keyalias in config.yml." - ) - logging.info( - "If you want to generate a new repo signing key in the HSM you can do that with 'fdroid update " - "--create-key'." - ) + logging.info("Repo setup using a smartcard HSM. Please edit keystorepass and repo_keyalias in config.yml.") + logging.info("If you want to generate a new repo signing key in the HSM you can do that with 'fdroid update " + "--create-key'.") elif os.path.exists(keystore): to_set = ['keystorepass', 'keypass', 'repo_keyalias', 'keydname'] if repo_keyalias: to_set.remove('repo_keyalias') if keydname: to_set.remove('keydname') - logging.warning( - '\n' - + _('Using existing keystore "{path}"').format(path=keystore) - + '\n' - + _('Now set these in config.yml:') - + ' ' - + ', '.join(to_set) - + '\n' - ) + logging.warning('\n' + _('Using existing keystore "{path}"').format(path=keystore) + + '\n' + _('Now set these in config.yml:') + ' ' + + ', '.join(to_set) + '\n') else: password = common.genpassword() c = dict(test_config) c['keystorepass'] = password c['keypass'] = password - c['repo_keyalias'] = repo_keyalias or socket.getfqdn() + c['repo_keyalias'] = socket.getfqdn() c['keydname'] = 'CN=' + c['repo_keyalias'] + ', OU=F-Droid' common.write_to_config(test_config, 'keystorepass', password) common.write_to_config(test_config, 'keypass', password) @@ -274,25 +222,17 @@ def main(): msg = '\n' msg += _('Built repo based in "%s" with this config:') % fdroiddir msg += '\n\n Android SDK:\t\t\t' + config['sdk_path'] + if aapt: + msg += '\n Android SDK Build Tools:\t' + os.path.dirname(aapt) + msg += '\n Android NDK r12b (optional):\t$ANDROID_NDK' msg += '\n ' + _('Keystore for signing key:\t') + keystore if repo_keyalias is not None: msg += '\n Alias for key in store:\t' + repo_keyalias - msg += '\n\n' - msg += ( - _( - """To complete the setup, add your APKs to "%s" + msg += '\n\n' + '''To complete the setup, add your APKs to "%s" then run "fdroid update -c; fdroid update". You might also want to edit "config.yml" to set the URL, repo name, and more. You should also set up a signing key (a temporary one might have been automatically generated). For more info: https://f-droid.org/docs/Setup_an_F-Droid_App_Repo -and https://f-droid.org/docs/Signing_Process""" - ) - % os.path.join(fdroiddir, 'repo') - ) - if not options.quiet: - # normally, INFO is only shown with --verbose, but show this unless --quiet - logger = logging.getLogger() - logger.setLevel(logging.INFO) - logger.info(msg) - logging.shutdown() +and https://f-droid.org/docs/Signing_Process''' % os.path.join(fdroiddir, 'repo') + logging.info(msg) diff --git a/fdroidserver/install.py b/fdroidserver/install.py index 8c1dc948..0e50c27b 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -17,372 +17,62 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import glob -import locale -import logging -import os import sys -import termios -import tty -from argparse import ArgumentParser, BooleanOptionalAction -from pathlib import Path -from urllib.parse import urlencode, urlparse, urlunparse +import os +import glob +from argparse import ArgumentParser +import logging -import defusedxml.ElementTree as XMLElementTree - -from . import _, common, github, index, net +from . import _ +from . import common +from .common import SdkToolsPopen from .exception import FDroidException -DEFAULT_IPFS_GATEWAYS = ("https://gateway.ipfs.io/ipfs/",) -MAVEN_CENTRAL_MIRRORS = [ - { - "url": "https://repo1.maven.org/maven2/", - "dnsA": ["199.232.16.209"], - "worksWithoutSNI": True, - }, - { - "url": "https://repo.maven.apache.org/maven2/", - "dnsA": ["199.232.16.215"], - "worksWithoutSNI": True, - }, - { - "url": "https://maven-central-asia.storage-download.googleapis.com/maven2/", - }, - { - "url": "https://maven-central-eu.storage-download.googleapis.com/maven2/", - }, - { - "url": "https://maven-central.storage-download.googleapis.com/maven2/", - }, -] - - -# pylint: disable=unused-argument -def download_apk(appid='org.fdroid.fdroid', privacy_mode=False): - """Download an APK from F-Droid via the first mirror that works.""" - url = urlunparse( - urlparse(common.FDROIDORG_MIRRORS[0]['url'])._replace( - query=urlencode({'fingerprint': common.FDROIDORG_FINGERPRINT}) - ) - ) - - data, _ignored = index.download_repo_index_v2(url) - app = data.get('packages', dict()).get(appid) - preferred_version = None - for version in app['versions'].values(): - if not preferred_version: - # if all else fails, use the first one - preferred_version = version - if not version.get('releaseChannels'): - # prefer APK in default release channel - preferred_version = version - break - - mirrors = common.append_filename_to_mirrors( - preferred_version['file']['name'][1:], common.FDROIDORG_MIRRORS - ) - ipfsCIDv1 = preferred_version['file'].get('ipfsCIDv1') - if ipfsCIDv1: - for gateway in DEFAULT_IPFS_GATEWAYS: - mirrors.append({'url': os.path.join(gateway, ipfsCIDv1)}) - f = net.download_using_mirrors(mirrors) - if f and os.path.exists(f): - versionCode = preferred_version['manifest']['versionCode'] - f = Path(f) - return str(f.rename(f.with_stem(f'{appid}_{versionCode}')).resolve()) - - -def download_fdroid_apk(privacy_mode=False): # pylint: disable=unused-argument - """Directly download the current F-Droid APK and verify it. - - This downloads the "download button" link, which is the version - that is best tested for new installs. - - """ - mirror = common.FDROIDORG_MIRRORS[0] - mirror['url'] = urlunparse(urlparse(mirror['url'])._replace(path='F-Droid.apk')) - return net.download_using_mirrors([mirror]) - - -def download_fdroid_apk_from_github(privacy_mode=False): - """Download F-Droid.apk from F-Droid's GitHub Releases.""" - if common.config and not privacy_mode: - token = common.config.get('github_token') - else: - token = None - gh = github.GithubApi(token, 'https://github.com/f-droid/fdroidclient') - latest_apk = gh.get_latest_apk() - filename = os.path.basename(latest_apk) - return net.download_file(latest_apk, os.path.join(common.get_cachedir(), filename)) - - -def download_fdroid_apk_from_ipns(privacy_mode=False): - """Download the F-Droid APK from an IPNS repo.""" - cid = 'k51qzi5uqu5dl4hbcksbdmplanu9n4hivnqsupqe6vzve1pdbeh418ssptldd3' - mirrors = [ - {"url": f"https://ipfs.io/ipns/{cid}/F-Droid.apk"}, - ] - if not privacy_mode: - mirrors.append({"url": f"https://{cid}.ipns.dweb.link/F-Droid.apk"}) - return net.download_using_mirrors(mirrors) - - -def download_fdroid_apk_from_maven(privacy_mode=False): - """Download F-Droid.apk from Maven Central and official mirrors.""" - path = 'org/fdroid/fdroid/F-Droid' - if privacy_mode: - mirrors = MAVEN_CENTRAL_MIRRORS[:2] # skip the Google servers - else: - mirrors = MAVEN_CENTRAL_MIRRORS - metadata = net.download_using_mirrors( - common.append_filename_to_mirrors( - os.path.join(path, 'maven-metadata.xml'), mirrors - ) - ) - version = XMLElementTree.parse(metadata).getroot().findall('*.//latest')[0].text - mirrors = common.append_filename_to_mirrors( - os.path.join(path, version, f'F-Droid-{version}.apk'), mirrors - ) - return net.download_using_mirrors(mirrors) - - -def install_fdroid_apk(privacy_mode=False): - """Download and install F-Droid.apk using all tricks we can muster. - - By default, this first tries to fetch the official install APK - which is offered when someone clicks the "download" button on - https://f-droid.org/. Then it will try all the mirrors and - methods until it gets something successful, or runs out of - options. - - There is privacy_mode which tries to download from mirrors first, - so that this downloads from a mirror that has many different kinds - of files available, thereby breaking the clear link to F-Droid. - - Returns - ------- - None for success or the error message. - - """ - country_code = locale.getlocale()[0].split('_')[-1] - if privacy_mode is None and country_code in ('CN', 'HK', 'IR', 'TM'): - logging.warning( - _('Privacy mode was enabled based on your locale ({country_code}).').format( - country_code=country_code - ) - ) - privacy_mode = True - - if privacy_mode or not (common.config and common.config.get('jarsigner')): - download_methods = [ - download_fdroid_apk_from_maven, - download_fdroid_apk_from_ipns, - download_fdroid_apk_from_github, - ] - else: - download_methods = [ - download_apk, - download_fdroid_apk_from_maven, - download_fdroid_apk_from_github, - download_fdroid_apk_from_ipns, - download_fdroid_apk, - ] - for method in download_methods: - try: - f = method(privacy_mode=privacy_mode) - break - except Exception as e: - logging.info(e) - else: - return _('F-Droid.apk could not be downloaded from any known source!') - - fingerprint = common.apk_signer_fingerprint(f) - if fingerprint.upper() != common.FDROIDORG_FINGERPRINT: - return _('{path} has the wrong fingerprint ({fingerprint})!').format( - path=f, fingerprint=fingerprint - ) - install_apk(f) - - -def install_apk(f): - if common.config and common.config.get('apksigner'): - # TODO this should always verify, but that requires APK sig verification in Python #94 - logging.info(_('Verifying package {path} with apksigner.').format(path=f)) - common.verify_apk_signature(f) - if common.config and common.config.get('adb'): - if devices(): - install_apks_to_devices([f]) - os.remove(f) - else: - os.remove(f) - return _('No devices found for `adb install`! Please plug one in.') +options = None +config = None def devices(): - """Get the list of device serials for use with adb commands.""" - p = common.SdkToolsPopen(['adb', "devices"]) + p = SdkToolsPopen(['adb', "devices"]) if p.returncode != 0: raise FDroidException("An error occured when finding devices: %s" % p.output) - serials = list() - for line in p.output.splitlines(): - columns = line.strip().split("\t", maxsplit=1) - if len(columns) == 2: - serial, status = columns - if status == 'device': - serials.append(serial) - else: - d = {'serial': serial, 'status': status} - logging.warning(_('adb reports {serial} is "{status}"!'.format(**d))) - return serials - - -def install_apks_to_devices(apks): - """Install the list of APKs to all Android devices reported by `adb devices`.""" - for apk in apks: - # Get device list each time to avoid device not found errors - devs = devices() - if not devs: - raise FDroidException(_("No attached devices found")) - logging.info(_("Installing %s...") % apk) - for dev in devs: - logging.info( - _("Installing '{apkfilename}' on {dev}...").format( - apkfilename=apk, dev=dev - ) - ) - p = common.SdkToolsPopen(['adb', "-s", dev, "install", apk]) - fail = "" - for line in p.output.splitlines(): - if line.startswith("Failure"): - fail = line[9:-1] - if not fail: - continue - - if fail == "INSTALL_FAILED_ALREADY_EXISTS": - logging.warning( - _('"{apkfilename}" is already installed on {dev}.').format( - apkfilename=apk, dev=dev - ) - ) - else: - raise FDroidException( - _("Failed to install '{apkfilename}' on {dev}: {error}").format( - apkfilename=apk, dev=dev, error=fail - ) - ) - - -def read_char(): - """Read input from the terminal prompt one char at a time.""" - fd = sys.stdin.fileno() - old_settings = termios.tcgetattr(fd) - try: - tty.setraw(fd) - ch = sys.stdin.read(1) - finally: - termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) - return ch - - -def strtobool(val): - """Convert a localized string representation of truth to True or False.""" - return val.lower() in ('', 'y', 'yes', _('yes'), _('true')) # '' is pressing Enter - - -def prompt_user(yes, msg): - """Prompt user for yes/no, supporting Enter and Esc as accepted answers.""" - run_install = yes - if yes is None and sys.stdout.isatty(): - print(msg, end=' ', flush=True) - answer = '' - while True: - in_char = read_char() - if in_char == '\r': # Enter key - break - if not in_char.isprintable(): - sys.exit(1) - print(in_char, end='', flush=True) - answer += in_char - run_install = strtobool(answer) - print() - return run_install + lines = [line for line in p.output.splitlines() if not line.startswith('* ')] + if len(lines) < 3: + return [] + lines = lines[1:-1] + return [line.split()[0] for line in lines] def main(): - parser = ArgumentParser( - usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]" - ) + + global options, config + + # Parse command line... + parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) - parser.add_argument( - "appid", - nargs='*', - help=_("application ID with optional versionCode in the form APPID[:VERCODE]"), - ) - parser.add_argument( - "-a", - "--all", - action="store_true", - default=False, - help=_("Install all signed applications available"), - ) - parser.add_argument( - "-p", - "--privacy-mode", - action=BooleanOptionalAction, - default=None, - help=_("Download F-Droid.apk using mirrors that leak less to the network"), - ) - parser.add_argument( - "-y", - "--yes", - action="store_true", - default=None, - help=_("Automatic yes to all prompts."), - ) - parser.add_argument( - "-n", - "--no", - action="store_false", - dest='yes', - help=_("Automatic no to all prompts."), - ) - options = common.parse_args(parser) - - common.set_console_logging(options.verbose, options.color) - logging.captureWarnings(True) # for SNIMissingWarning - - common.get_config() + parser.add_argument("appid", nargs='*', help=_("application ID with optional versionCode in the form APPID[:VERCODE]")) + parser.add_argument("-a", "--all", action="store_true", default=False, + help=_("Install all signed applications available")) + options = parser.parse_args() if not options.appid and not options.all: - run_install = prompt_user( - options.yes, - _('Would you like to download and install F-Droid.apk via adb? (YES/no)'), - ) - if run_install: - sys.exit(install_fdroid_apk(options.privacy_mode)) - sys.exit(1) + parser.error(_("option %s: If you really want to install all the signed apps, use --all") % "all") + + config = common.read_config(options) output_dir = 'repo' - if (options.appid or options.all) and not os.path.isdir(output_dir): - logging.error(_("No signed output directory - nothing to do")) - run_install = prompt_user( - options.yes, - _('Would you like to download the app(s) from f-droid.org? (YES/no)'), - ) - if run_install: - for appid in options.appid: - f = download_apk(appid) - install_apk(f) - sys.exit(install_fdroid_apk(options.privacy_mode)) - sys.exit(1) + if not os.path.isdir(output_dir): + logging.info(_("No signed output directory - nothing to do")) + sys.exit(0) if options.appid: + vercodes = common.read_pkg_args(options.appid, True) - common.get_metadata_files(vercodes) # only check appids apks = {appid: None for appid in vercodes} - # Get the signed APK with the highest vercode + # Get the signed apk with the highest vercode for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))): + try: appid, vercode = common.publishednameinfo(apkfile) except FDroidException: @@ -395,15 +85,35 @@ def main(): for appid, apk in apks.items(): if not apk: - raise FDroidException(_("No signed APK available for %s") % appid) - install_apks_to_devices(apks.values()) + raise FDroidException(_("No signed apk available for %s") % appid) - elif options.all: - apks = { - common.publishednameinfo(apkfile)[0]: apkfile - for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))) - } - install_apks_to_devices(apks.values()) + else: + + apks = {common.publishednameinfo(apkfile)[0]: apkfile for apkfile in + sorted(glob.glob(os.path.join(output_dir, '*.apk')))} + + for appid, apk in apks.items(): + # Get device list each time to avoid device not found errors + devs = devices() + if not devs: + raise FDroidException(_("No attached devices found")) + logging.info(_("Installing %s...") % apk) + for dev in devs: + logging.info(_("Installing '{apkfilename}' on {dev}...").format(apkfilename=apk, dev=dev)) + p = SdkToolsPopen(['adb', "-s", dev, "install", apk]) + fail = "" + for line in p.output.splitlines(): + if line.startswith("Failure"): + fail = line[9:-1] + if not fail: + continue + + if fail == "INSTALL_FAILED_ALREADY_EXISTS": + logging.warning(_('"{apkfilename}" is already installed on {dev}.') + .format(apkfilename=apk, dev=dev)) + else: + raise FDroidException(_("Failed to install '{apkfilename}' on {dev}: {error}") + .format(apkfilename=apk, dev=dev, error=fail)) logging.info('\n' + _('Finished')) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 99b1a392..b5acc9ad 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -16,28 +16,25 @@ # You should have received a copy of the GNU Affero General Public Licen # along with this program. If not, see . -import difflib -import platform +from argparse import ArgumentParser +import glob +import os import re import sys import urllib.parse -from argparse import ArgumentParser -from pathlib import Path -from fdroidserver._yaml import yaml - -from . import _, common, metadata, rewritemeta +from . import _ +from . import common +from . import metadata +from . import rewritemeta config = None +options = None def enforce_https(domain): - return ( - re.compile( - r'^http://([^/]*\.)?' + re.escape(domain) + r'(/.*)?', re.IGNORECASE - ), - domain + " URLs should always use https://", - ) + return (re.compile(r'^http://([^/]*\.)?' + re.escape(domain) + r'(/.*)?', re.IGNORECASE), + domain + " URLs should always use https://") https_enforcings = [ @@ -62,10 +59,8 @@ https_enforcings = [ def forbid_shortener(domain): - return ( - re.compile(r'https?://[^/]*' + re.escape(domain) + r'/.*'), - _("URL shorteners should not be used"), - ) + return (re.compile(r'https?://[^/]*' + re.escape(domain) + r'/.*'), + _("URL shorteners should not be used")) http_url_shorteners = [ @@ -124,200 +119,70 @@ http_url_shorteners = [ forbid_shortener('➡.ws'), ] -http_checks = ( - https_enforcings - + http_url_shorteners - + [ - ( - re.compile(r'^(?!https?://)[^/]+'), - _("URL must start with https:// or http://"), - ), - ( - re.compile(r'^https://(github|gitlab)\.com(/[^/]+){2,3}\.git'), - _("Appending .git is not necessary"), - ), - ( - re.compile( - r'^https://[^/]*(github|gitlab|bitbucket|rawgit|githubusercontent)\.[a-zA-Z]+/([^/]+/){2,3}(master|main)/' - ), - _( - "Use /HEAD instead of /master or /main to point at a file in the default branch" - ), - ), - ] -) +http_checks = https_enforcings + http_url_shorteners + [ + (re.compile(r'^(?!https?://)[^/]+'), + _("URL must start with https:// or http://")), + (re.compile(r'^https://(github|gitlab)\.com(/[^/]+){2,3}\.git'), + _("Appending .git is not necessary")), + (re.compile(r'^https://[^/]*(github|gitlab|bitbucket|rawgit|githubusercontent)\.[a-zA-Z]+/([^/]+/){2,3}master/'), + _("Use /HEAD instead of /master to point at a file in the default branch")), +] regex_checks = { 'WebSite': http_checks, 'SourceCode': http_checks, 'Repo': https_enforcings, 'UpdateCheckMode': https_enforcings, - 'IssueTracker': http_checks - + [ - (re.compile(r'.*github\.com/[^/]+/[^/]+/*$'), _("/issues is missing")), - (re.compile(r'.*gitlab\.com/[^/]+/[^/]+/*$'), _("/issues is missing")), + 'IssueTracker': http_checks + [ + (re.compile(r'.*github\.com/[^/]+/[^/]+/*$'), + _("/issues is missing")), + (re.compile(r'.*gitlab\.com/[^/]+/[^/]+/*$'), + _("/issues is missing")), ], - 'Donate': http_checks - + [ - ( - re.compile(r'.*liberapay\.com'), - _("Liberapay donation methods belong in the Liberapay: field"), - ), - ( - re.compile(r'.*opencollective\.com'), - _("OpenCollective donation methods belong in the OpenCollective: field"), - ), + 'Donate': http_checks + [ + (re.compile(r'.*flattr\.com'), + _("Flattr donation methods belong in the FlattrID: field")), + (re.compile(r'.*liberapay\.com'), + _("Liberapay donation methods belong in the Liberapay: field")), + (re.compile(r'.*opencollective\.com'), + _("OpenCollective donation methods belong in the OpenCollective: field")), ], 'Changelog': http_checks, 'Author Name': [ - (re.compile(r'^\s'), _("Unnecessary leading space")), - (re.compile(r'.*\s$'), _("Unnecessary trailing space")), + (re.compile(r'^\s'), + _("Unnecessary leading space")), + (re.compile(r'.*\s$'), + _("Unnecessary trailing space")), ], 'Summary': [ - ( - re.compile(r'.*\b(free software|open source)\b.*', re.IGNORECASE), - _("No need to specify that the app is Free Software"), - ), - ( - re.compile( - r'.*((your|for).*android|android.*(app|device|client|port|version))', - re.IGNORECASE, - ), - _("No need to specify that the app is for Android"), - ), - (re.compile(r'.*[a-z0-9][.!?]( |$)'), _("Punctuation should be avoided")), - (re.compile(r'^\s'), _("Unnecessary leading space")), - (re.compile(r'.*\s$'), _("Unnecessary trailing space")), + (re.compile(r'.*\b(free software|open source)\b.*', re.IGNORECASE), + _("No need to specify that the app is Free Software")), + (re.compile(r'.*((your|for).*android|android.*(app|device|client|port|version))', re.IGNORECASE), + _("No need to specify that the app is for Android")), + (re.compile(r'.*[a-z0-9][.!?]( |$)'), + _("Punctuation should be avoided")), + (re.compile(r'^\s'), + _("Unnecessary leading space")), + (re.compile(r'.*\s$'), + _("Unnecessary trailing space")), ], - 'Description': https_enforcings - + http_url_shorteners - + [ - (re.compile(r'\s*[*#][^ .]'), _("Invalid bulleted list")), - ( - re.compile(r'https://f-droid.org/[a-z][a-z](_[A-Za-z]{2,4})?/'), - _("Locale included in f-droid.org URL"), - ), - (re.compile(r'^\s'), _("Unnecessary leading space")), - (re.compile(r'.*\s$'), _("Unnecessary trailing space")), - ( - re.compile( - r'.*<(applet|base|body|button|embed|form|head|html|iframe|img|input|link|object|picture|script|source|style|svg|video).*', - re.IGNORECASE, - ), - _("Forbidden HTML tags"), - ), - ( - re.compile(r""".*\s+src=["']javascript:.*"""), - _("Javascript in HTML src attributes"), - ), + 'Description': https_enforcings + http_url_shorteners + [ + (re.compile(r'\s*[*#][^ .]'), + _("Invalid bulleted list")), + (re.compile(r'https://f-droid.org/[a-z][a-z](_[A-Za-z]{2,4})?/'), + _("Locale included in f-droid.org URL")), + (re.compile(r'^\s'), + _("Unnecessary leading space")), + (re.compile(r'.*\s$'), + _("Unnecessary trailing space")), + (re.compile(r'.*<(applet|base|body|button|embed|form|head|html|iframe|img|input|link|object|picture|script|source|style|svg|video).*', re.IGNORECASE), + _("Forbidden HTML tags")), + (re.compile(r'''.*\s+src=["']javascript:.*'''), + _("Javascript in HTML src attributes")), ], } -# config keys that are currently ignored by lint, but could be supported. -ignore_config_keys = ( - 'github_releases', - 'java_paths', -) - -bool_keys = ( - 'allow_disabled_algorithms', - 'androidobservatory', - 'build_server_always', - 'deploy_process_logs', - 'keep_when_not_allowed', - 'make_current_version_link', - 'nonstandardwebroot', - 'per_app_repos', - 'refresh_scanner', - 'scan_binary', - 'sync_from_local_copy_dir', -) - -check_config_keys = ( - 'ant', - 'apk_signing_key_block_list', - 'archive', - 'archive_description', - 'archive_icon', - 'archive_name', - 'archive_older', - 'archive_url', - 'archive_web_base_url', - 'awsbucket', - 'awsbucket_index_only', - 'binary_transparency_remote', - 'cachedir', - 'char_limits', - 'current_version_name_source', - 'git_mirror_size_limit', - 'github_token', - 'gpghome', - 'gpgkey', - 'gradle', - 'identity_file', - 'install_list', - 'java_paths', - 'keyaliases', - 'keydname', - 'keypass', - 'keystore', - 'keystorepass', - 'lint_licenses', - 'local_copy_dir', - 'mirrors', - 'mvn3', - 'ndk_paths', - 'path_to_custom_rclone_config', - 'rclone_config', - 'repo', - 'repo_description', - 'repo_icon', - 'repo_key_sha256', - 'repo_keyalias', - 'repo_maxage', - 'repo_name', - 'repo_pubkey', - 'repo_url', - 'repo_web_base_url', - 'scanner_signature_sources', - 'sdk_path', - 'servergitmirrors', - 'serverwebroot', - 'smartcardoptions', - 'sync_from_local_copy_dir', - 'uninstall_list', - 'virustotal_apikey', -) - -locale_pattern = re.compile(r"[a-z]{2,3}(-([A-Z][a-zA-Z]+|\d+|[a-z]+))*") - -versioncode_check_pattern = re.compile(r"(\\d|\[(0-9|\\d)_?(a-fA-F)?])[+]") - -ANTIFEATURES_KEYS = None -ANTIFEATURES_PATTERN = None -CATEGORIES_KEYS = list() - - -def load_antiFeatures_config(): - """Lazy loading, since it might read a lot of files.""" - global ANTIFEATURES_KEYS, ANTIFEATURES_PATTERN - k = common.ANTIFEATURES_CONFIG_NAME - if not ANTIFEATURES_KEYS or k not in common.config: - common.config[k] = common.load_localized_config(k, 'repo') - ANTIFEATURES_KEYS = sorted(common.config[k].keys()) - ANTIFEATURES_PATTERN = ','.join(ANTIFEATURES_KEYS) - - -def load_categories_config(): - """Lazy loading, since it might read a lot of files.""" - global CATEGORIES_KEYS - k = common.CATEGORIES_CONFIG_NAME - if not CATEGORIES_KEYS: - if config and k in config: - CATEGORIES_KEYS = config[k] - else: - config[k] = common.load_localized_config(k, 'repo') - CATEGORIES_KEYS = list(config[k].keys()) +locale_pattern = re.compile(r'^[a-z]{2,3}(-[A-Z][A-Z])?$') def check_regexes(app): @@ -341,17 +206,18 @@ def get_lastbuild(builds): lastbuild = None for build in builds: if not build.disable: - vercode = build.versionCode + vercode = int(build.versionCode) if lowest_vercode == -1 or vercode < lowest_vercode: lowest_vercode = vercode - if not lastbuild or build.versionCode > lastbuild.versionCode: + if not lastbuild or int(build.versionCode) > int(lastbuild.versionCode): lastbuild = build return lastbuild -def check_update_check_data_url(app): # noqa: D403 - """UpdateCheckData must have a valid HTTPS URL to protect checkupdates runs.""" - if app.UpdateCheckData and app.UpdateCheckMode == 'HTTP': +def check_update_check_data_url(app): + """UpdateCheckData must have a valid HTTPS URL to protect checkupdates runs + """ + if app.UpdateCheckData: urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|') for url in (urlcode, urlver): if url != '.': @@ -362,58 +228,34 @@ def check_update_check_data_url(app): # noqa: D403 yield _('UpdateCheckData must use HTTPS URL: {url}').format(url=url) -def check_update_check_data_int(app): # noqa: D403 - """UpdateCheckData regex must match integers.""" - if app.UpdateCheckData: - urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|') - # codeex can be empty as well - if codeex and not versioncode_check_pattern.search(codeex): - yield _( - f'UpdateCheckData must match the versionCode as integer (\\d or [0-9]): {codeex}' - ) - - def check_vercode_operation(app): - if not app.VercodeOperation: - return - invalid_ops = [] - for op in app.VercodeOperation: - if not common.VERCODE_OPERATION_RE.match(op): - invalid_ops += op - if invalid_ops: - yield _('Invalid VercodeOperation: {invalid_ops}').format( - invalid_ops=invalid_ops - ) + if app.VercodeOperation and not common.VERCODE_OPERATION_RE.match(app.VercodeOperation): + yield _('Invalid VercodeOperation: {field}').format(field=app.VercodeOperation) def check_ucm_tags(app): - lastbuild = get_lastbuild(app.get('Builds', [])) - if ( - lastbuild is not None - and lastbuild.commit - and app.UpdateCheckMode == 'RepoManifest' - and not lastbuild.commit.startswith('unknown') - and lastbuild.versionCode == app.CurrentVersionCode - and not lastbuild.forcevercode - and any(s in lastbuild.commit for s in '.,_-/') - ): - yield _( - "Last used commit '{commit}' looks like a tag, but UpdateCheckMode is '{ucm}'" - ).format(commit=lastbuild.commit, ucm=app.UpdateCheckMode) + lastbuild = get_lastbuild(app.builds) + if (lastbuild is not None + and lastbuild.commit + and app.UpdateCheckMode == 'RepoManifest' + and not lastbuild.commit.startswith('unknown') + and lastbuild.versionCode == app.CurrentVersionCode + and not lastbuild.forcevercode + and any(s in lastbuild.commit for s in '.,_-/')): + yield _("Last used commit '{commit}' looks like a tag, but UpdateCheckMode is '{ucm}'")\ + .format(commit=lastbuild.commit, ucm=app.UpdateCheckMode) def check_char_limits(app): limits = config['char_limits'] if len(app.Summary) > limits['summary']: - yield _("Summary of length {length} is over the {limit} char limit").format( - length=len(app.Summary), limit=limits['summary'] - ) + yield _("Summary of length {length} is over the {limit} char limit")\ + .format(length=len(app.Summary), limit=limits['summary']) if len(app.Description) > limits['description']: - yield _("Description of length {length} is over the {limit} char limit").format( - length=len(app.Description), limit=limits['description'] - ) + yield _("Description of length {length} is over the {limit} char limit")\ + .format(length=len(app.Description), limit=limits['description']) def check_old_links(app): @@ -430,9 +272,8 @@ def check_old_links(app): for f in ['WebSite', 'SourceCode', 'IssueTracker', 'Changelog']: v = app.get(f) if any(s in v for s in old_sites): - yield _("App is in '{repo}' but has a link to {url}").format( - repo=app.Repo, url=v - ) + yield _("App is in '{repo}' but has a link to {url}")\ + .format(repo=app.Repo, url=v) def check_useless_fields(app): @@ -445,10 +286,8 @@ filling_ucms = re.compile(r'^(Tags.*|RepoManifest.*)') def check_checkupdates_ran(app): if filling_ucms.match(app.UpdateCheckMode): - if not app.AutoName and not app.CurrentVersion and app.CurrentVersionCode == 0: - yield _( - "UpdateCheckMode is set but it looks like checkupdates hasn't been run yet." - ) + if not app.AutoName and not app.CurrentVersion and app.CurrentVersionCode == '0': + yield _("UpdateCheckMode is set but it looks like checkupdates hasn't been run yet") def check_empty_fields(app): @@ -456,14 +295,37 @@ def check_empty_fields(app): yield _("Categories are not set") +all_categories = set([ + "Connectivity", + "Development", + "Games", + "Graphics", + "Internet", + "Money", + "Multimedia", + "Navigation", + "Phone & SMS", + "Reading", + "Science & Education", + "Security", + "Sports & Health", + "System", + "Theming", + "Time", + "Writing", +]) + + def check_categories(app): - """App uses 'Categories' key and parsed config uses 'categories' key.""" for categ in app.Categories: - if categ not in CATEGORIES_KEYS: + if categ not in all_categories: yield _("Categories '%s' is not valid" % categ) def check_duplicates(app): + if app.Name and app.Name == app.AutoName: + yield _("Name '%s' is just the auto name - remove it") % app.Name + links_seen = set() for f in ['Source Code', 'Web Site', 'Issue Tracker', 'Changelog']: v = app.get(f) @@ -475,7 +337,7 @@ def check_duplicates(app): else: links_seen.add(v) - name = common.get_app_display_name(app) + name = app.Name or app.AutoName if app.Summary and name: if app.Summary.lower() == name.lower(): yield _("Summary '%s' is just the app's name") % app.Summary @@ -505,281 +367,189 @@ def check_mediawiki_links(app): yield _("URL {url} in Description: {error}").format(url=url, error=r) +def check_bulleted_lists(app): + validchars = ['*', '#'] + lchar = '' + lcount = 0 + for line in app.Description.splitlines(): + if len(line) < 1: + lcount = 0 + continue + + if line[0] == lchar and line[1] == ' ': + lcount += 1 + if lcount > 2 and lchar not in validchars: + yield _("Description has a list (%s) but it isn't bulleted (*) nor numbered (#)") % lchar + break + else: + lchar = line[0] + lcount = 1 + + def check_builds(app): supported_flags = set(metadata.build_flags) # needed for YAML and JSON - for build in app.get('Builds', []): + for build in app.builds: if build.disable: if build.disable.startswith('Generated by import.py'): - yield _( - "Build generated by `fdroid import` - remove disable line once ready" - ) + yield _("Build generated by `fdroid import` - remove disable line once ready") continue - for s in ['master', 'main', 'origin', 'HEAD', 'default', 'trunk']: + for s in ['master', 'origin', 'HEAD', 'default', 'trunk']: if build.commit and build.commit.startswith(s): - yield _( - "Branch '{branch}' used as commit in build '{versionName}'" - ).format(branch=s, versionName=build.versionName) + yield _("Branch '{branch}' used as commit in build '{versionName}'")\ + .format(branch=s, versionName=build.versionName) for srclib in build.srclibs: if '@' in srclib: ref = srclib.split('@')[1].split('/')[0] if ref.startswith(s): - yield _( - "Branch '{branch}' used as commit in srclib '{srclib}'" - ).format(branch=s, srclib=srclib) + yield _("Branch '{branch}' used as commit in srclib '{srclib}'")\ + .format(branch=s, srclib=srclib) else: - yield ( - _('srclibs missing name and/or @') - + ' (srclibs: ' - + srclib - + ')' - ) + yield _('srclibs missing name and/or @') + ' (srclibs: ' + srclib + ')' for key in build.keys(): if key not in supported_flags: yield _('%s is not an accepted build field') % key def check_files_dir(app): - dir_path = Path('metadata') / app.id - if not dir_path.is_dir(): + dir_path = os.path.join('metadata', app.id) + if not os.path.isdir(dir_path): return files = set() - for path in dir_path.iterdir(): - name = path.name - if not ( - path.is_file() or name == 'signatures' or locale_pattern.fullmatch(name) - ): + for name in os.listdir(dir_path): + path = os.path.join(dir_path, name) + if not (os.path.isfile(path) or name == 'signatures' or locale_pattern.match(name)): yield _("Found non-file at %s") % path continue files.add(name) - used = { - 'signatures', - } - for build in app.get('Builds', []): + used = {'signatures', } + for build in app.builds: for fname in build.patch: if fname not in files: - yield _("Unknown file '{filename}' in build '{versionName}'").format( - filename=fname, versionName=build.versionName - ) + yield _("Unknown file '{filename}' in build '{versionName}'")\ + .format(filename=fname, versionName=build.versionName) else: used.add(fname) for name in files.difference(used): - if locale_pattern.fullmatch(name): + if locale_pattern.match(name): continue - yield _("Unused file at %s") % (dir_path / name) + yield _("Unused file at %s") % os.path.join(dir_path, name) def check_format(app): - if common.options.format and not rewritemeta.proper_format(app): + if options.format and not rewritemeta.proper_format(app): yield _("Run rewritemeta to fix formatting") def check_license_tag(app): - """Ensure all license tags contain only valid/approved values. - - It is possible to disable license checking by setting a null or empty value, - e.g. `lint_licenses: ` or `lint_licenses: []` - - """ - if 'lint_licenses' in config: - lint_licenses = config['lint_licenses'] - if lint_licenses is None: - return - else: - lint_licenses = APPROVED_LICENSES - if app.License not in lint_licenses: - if lint_licenses == APPROVED_LICENSES: - yield _( - 'Unexpected license tag "{}"! Only use FSF or OSI ' - 'approved tags from https://spdx.org/license-list' - ).format(app.License) + '''Ensure all license tags contain only valid/approved values''' + if config['lint_licenses'] is None: + return + if app.License not in config['lint_licenses']: + if config['lint_licenses'] == APPROVED_LICENSES: + yield _('Unexpected license tag "{}"! Only use FSF or OSI ' + 'approved tags from https://spdx.org/license-list') \ + .format(app.License) else: - yield _( - 'Unexpected license tag "{}"! Only use license tags ' - 'configured in your config file' - ).format(app.License) + yield _('Unexpected license tag "{}"! Only use license tags ' + 'configured in your config file').format(app.License) def check_extlib_dir(apps): - dir_path = Path('build/extlib') - extlib_files = set() - for path in dir_path.glob('**/*'): - if path.is_file(): - extlib_files.add(path.relative_to(dir_path)) + dir_path = os.path.join('build', 'extlib') + unused_extlib_files = set() + for root, dirs, files in os.walk(dir_path): + for name in files: + unused_extlib_files.add(os.path.join(root, name)[len(dir_path) + 1:]) used = set() for app in apps: - if app.Disabled: - continue - archive_policy = common.calculate_archive_policy( - app, common.config['archive_older'] - ) - builds = [build for build in app.Builds if not build.disable] - - for i in range(len(builds)): - build = builds[i] + for build in app.builds: for path in build.extlibs: - path = Path(path) - if path not in extlib_files: - # Don't show error on archived versions - if i >= len(builds) - archive_policy: - yield _( - "{appid}: Unknown extlib {path} in build '{versionName}'" - ).format(appid=app.id, path=path, versionName=build.versionName) + if path not in unused_extlib_files: + yield _("{appid}: Unknown extlib {path} in build '{versionName}'")\ + .format(appid=app.id, path=path, versionName=build.versionName) else: used.add(path) - for path in extlib_files.difference(used): - if path.name not in [ - '.gitignore', - 'source.txt', - 'origin.txt', - 'md5.txt', - 'LICENSE', - 'LICENSE.txt', - 'COPYING', - 'COPYING.txt', - 'NOTICE', - 'NOTICE.txt', - ]: - yield _("Unused extlib at %s") % (dir_path / path) + for path in unused_extlib_files.difference(used): + if any(path.endswith(s) for s in [ + '.gitignore', + 'source.txt', 'origin.txt', 'md5.txt', + 'LICENSE', 'LICENSE.txt', + 'COPYING', 'COPYING.txt', + 'NOTICE', 'NOTICE.txt', + ]): + continue + yield _("Unused extlib at %s") % os.path.join(dir_path, path) def check_app_field_types(app): - """Check the fields have valid data types.""" + """Check the fields have valid data types""" + for field in app.keys(): v = app.get(field) t = metadata.fieldtype(field) if v is None: continue - elif field == 'Builds': + elif field == 'builds': if not isinstance(v, list): - yield ( - _( - "{appid}: {field} must be a '{type}', but it is a '{fieldtype}'!" - ).format( - appid=app.id, - field=field, - type='list', - fieldtype=v.__class__.__name__, - ) - ) + yield(_("{appid}: {field} must be a '{type}', but it is a '{fieldtype}'!") + .format(appid=app.id, field=field, + type='list', fieldtype=v.__class__.__name__)) elif t == metadata.TYPE_LIST and not isinstance(v, list): - yield ( - _( - "{appid}: {field} must be a '{type}', but it is a '{fieldtype}!'" - ).format( - appid=app.id, - field=field, - type='list', - fieldtype=v.__class__.__name__, - ) - ) - elif t == metadata.TYPE_STRING and type(v) not in (str, bool, dict): - yield ( - _( - "{appid}: {field} must be a '{type}', but it is a '{fieldtype}'!" - ).format( - appid=app.id, - field=field, - type='str', - fieldtype=v.__class__.__name__, - ) - ) - elif t == metadata.TYPE_STRINGMAP and not isinstance(v, dict): - yield ( - _( - "{appid}: {field} must be a '{type}', but it is a '{fieldtype}'!" - ).format( - appid=app.id, - field=field, - type='dict', - fieldtype=v.__class__.__name__, - ) - ) - elif t == metadata.TYPE_INT and not isinstance(v, int): - yield ( - _( - "{appid}: {field} must be a '{type}', but it is a '{fieldtype}'!" - ).format( - appid=app.id, - field=field, - type='int', - fieldtype=v.__class__.__name__, - ) - ) - - -def check_antiFeatures(app): - """Check the Anti-Features keys match those declared in the config.""" - pattern = ANTIFEATURES_PATTERN - msg = _("'{value}' is not a valid {field} in {appid}. Regex pattern: {pattern}") - - field = 'AntiFeatures' # App entries use capitalized CamelCase - for value in app.get(field, []): - if value not in ANTIFEATURES_KEYS: - yield msg.format(value=value, field=field, appid=app.id, pattern=pattern) - - field = 'antifeatures' # Build entries use all lowercase - for build in app.get('Builds', []): - build_antiFeatures = build.get(field, []) - for value in build_antiFeatures: - if value not in ANTIFEATURES_KEYS: - yield msg.format( - value=value, field=field, appid=app.id, pattern=pattern - ) + yield(_("{appid}: {field} must be a '{type}', but it is a '{fieldtype}!'") + .format(appid=app.id, field=field, + type='list', fieldtype=v.__class__.__name__)) + elif t == metadata.TYPE_STRING and not type(v) in (str, bool, dict): + yield(_("{appid}: {field} must be a '{type}', but it is a '{fieldtype}'!") + .format(appid=app.id, field=field, + type='str', fieldtype=v.__class__.__name__)) def check_for_unsupported_metadata_files(basedir=""): - """Check whether any non-metadata files are in metadata/.""" - basedir = Path(basedir) + """Checks whether any non-metadata files are in metadata/""" + global config - if not (basedir / 'metadata').exists(): - return False return_value = False - for f in (basedir / 'metadata').iterdir(): - if f.is_dir(): - if not Path(str(f) + '.yml').exists(): + for f in glob.glob(basedir + 'metadata/*') + glob.glob(basedir + 'metadata/.*'): + if os.path.isdir(f): + if not os.path.exists(f + '.yml'): print(_('"%s/" has no matching metadata file!') % f) return_value = True - elif f.suffix == '.yml': - packageName = f.stem + elif f.endswith('.yml'): + packageName = os.path.splitext(os.path.basename(f))[0] if not common.is_valid_package_name(packageName): - print( - '"' - + packageName - + '" is an invalid package name!\n' - + 'https://developer.android.com/studio/build/application-id' - ) + print('"' + packageName + '" is an invalid package name!\n' + + 'https://developer.android.com/studio/build/application-id') return_value = True else: - print( - _( - '"{path}" is not a supported file format (use: metadata/*.yml)' - ).format(path=f.relative_to(basedir)) - ) + print(_('"{path}" is not a supported file format (use: metadata/*.yml)') + .format(path=f.replace(basedir, ''))) return_value = True return return_value def check_current_version_code(app): - """Check that the CurrentVersionCode is currently available.""" - if app.get('ArchivePolicy') == 0: + """Check that the CurrentVersionCode is currently available""" + + archive_policy = app.get('ArchivePolicy') + if archive_policy and archive_policy.split()[0] == "0": return cv = app.get('CurrentVersionCode') - if cv is not None and cv == 0: + if cv is not None and int(cv) == 0: return - builds = app.get('Builds') + builds = app.get('builds') active_builds = 0 min_versionCode = None if builds: for build in builds: - vc = build['versionCode'] + vc = int(build['versionCode']) if min_versionCode is None or min_versionCode > vc: min_versionCode = vc if not build.get('disable'): @@ -788,232 +558,38 @@ def check_current_version_code(app): break if active_builds == 0: return # all builds are disabled - if cv is not None and cv < min_versionCode: - yield ( - _( - 'CurrentVersionCode {cv} is less than oldest build entry {versionCode}' - ).format(cv=cv, versionCode=min_versionCode) - ) - - -def check_updates_expected(app): - """Check if update checking makes sense.""" - if (app.get('NoSourceSince') or app.get('ArchivePolicy') == 0) and not all( - app.get(key, 'None') == 'None' for key in ('AutoUpdateMode', 'UpdateCheckMode') - ): - yield _( - 'App has NoSourceSince or ArchivePolicy "0 versions" or 0 but AutoUpdateMode or UpdateCheckMode are not None' - ) - - -def check_updates_ucm_http_aum_pattern(app): # noqa: D403 - """AutoUpdateMode with UpdateCheckMode: HTTP must have a pattern.""" - if app.UpdateCheckMode == "HTTP" and app.AutoUpdateMode == "Version": - yield _("AutoUpdateMode with UpdateCheckMode: HTTP must have a pattern.") - - -def check_certificate_pinned_binaries(app): - keys = app.get('AllowedAPKSigningKeys') - known_keys = common.config.get('apk_signing_key_block_list', []) - if keys: - if known_keys: - for key in keys: - if key in known_keys: - yield _('Known debug key is used in AllowedAPKSigningKeys: ') + key - return - if app.get('Binaries') is not None: - yield _( - 'App has Binaries but does not have corresponding AllowedAPKSigningKeys to pin certificate.' - ) - return - builds = app.get('Builds') - if builds is None: - return - for build in builds: - if build.get('binary') is not None: - yield _( - 'App version has binary but does not have corresponding AllowedAPKSigningKeys to pin certificate.' - ) - return - - -def lint_config(arg): - path = Path(arg) - passed = True - - mirrors_name = f'{common.MIRRORS_CONFIG_NAME}.yml' - config_name = f'{common.CONFIG_CONFIG_NAME}.yml' - categories_name = f'{common.CATEGORIES_CONFIG_NAME}.yml' - antifeatures_name = f'{common.ANTIFEATURES_CONFIG_NAME}.yml' - - yamllintresult = common.run_yamllint(path) - if yamllintresult: - print(yamllintresult) - passed = False - - with path.open() as fp: - data = yaml.load(fp) - common.config_type_check(arg, data) - - if path.name == mirrors_name: - import pycountry - - valid_country_codes = [c.alpha_2 for c in pycountry.countries] - for mirror in data: - code = mirror.get('countryCode') - if code and code not in valid_country_codes: - passed = False - msg = _( - '{path}: "{code}" is not a valid ISO_3166-1 alpha-2 country code!' - ).format(path=str(path), code=code) - if code.upper() in valid_country_codes: - m = [code.upper()] - else: - m = difflib.get_close_matches( - code.upper(), valid_country_codes, 2, 0.5 - ) - if m: - msg += ' ' - msg += _('Did you mean {code}?').format(code=', '.join(sorted(m))) - print(msg) - elif path.name == config_name and path.parent.name != 'config': - valid_keys = set(tuple(common.default_config) + bool_keys + check_config_keys) - for key in ignore_config_keys: - if key in valid_keys: - valid_keys.remove(key) - for key in data: - if key not in valid_keys: - passed = False - msg = _("ERROR: {key} not a valid key!").format(key=key) - m = difflib.get_close_matches(key.lower(), valid_keys, 2, 0.5) - if m: - msg += ' ' - msg += _('Did you mean {code}?').format(code=', '.join(sorted(m))) - print(msg) - continue - - if key in bool_keys: - t = bool - else: - t = type(common.default_config.get(key, "")) - - show_error = False - if t is str: - if type(data[key]) not in (str, list, dict): - passed = False - show_error = True - elif type(data[key]) != t: - passed = False - show_error = True - if show_error: - print( - _("ERROR: {key}'s value should be of type {t}!").format( - key=key, t=t.__name__ - ) - ) - elif path.name in (config_name, categories_name, antifeatures_name): - for key in data: - if path.name == config_name and key not in ('archive', 'repo'): - passed = False - print( - _('ERROR: {key} in {path} is not "archive" or "repo"!').format( - key=key, path=path - ) - ) - allowed_keys = ['name'] - if path.name in [config_name, antifeatures_name]: - allowed_keys.append('description') - # only for source strings currently - if path.parent.name == 'config': - allowed_keys.append('icon') - for subkey in data[key]: - if subkey not in allowed_keys: - passed = False - print( - _( - 'ERROR: {key}:{subkey} in {path} is not in allowed keys: {allowed_keys}!' - ).format( - key=key, - subkey=subkey, - path=path, - allowed_keys=', '.join(allowed_keys), - ) - ) - - return passed + if cv is not None and int(cv) < min_versionCode: + yield(_('CurrentVersionCode {cv} is less than oldest build entry {versionCode}') + .format(cv=cv, versionCode=min_versionCode)) def main(): - global config + + global config, options # Parse command line... parser = ArgumentParser() common.setup_global_opts(parser) - parser.add_argument( - "-f", - "--format", - action="store_true", - default=False, - help=_("Also warn about formatting issues, like rewritemeta -l"), - ) - parser.add_argument( - '--force-yamllint', - action="store_true", - default=False, - help=_( - "When linting the entire repository yamllint is disabled by default. " - "This option forces yamllint regardless." - ), - ) - parser.add_argument( - "appid", nargs='*', help=_("application ID of file to operate on") - ) + parser.add_argument("-f", "--format", action="store_true", default=False, + help=_("Also warn about formatting issues, like rewritemeta -l")) + parser.add_argument('--force-yamllint', action="store_true", default=False, + help=_("When linting the entire repository yamllint is disabled by default. " + "This option forces yamllint regardless.")) + parser.add_argument("appid", nargs='*', help=_("application ID of file to operate on")) metadata.add_metadata_arguments(parser) - options = common.parse_args(parser) + options = parser.parse_args() metadata.warnings_action = options.W - config = common.read_config() - load_antiFeatures_config() - load_categories_config() + config = common.read_config(options) - if options.force_yamllint: - import yamllint # throw error if it is not installed - - yamllint # make pyflakes ignore this - - paths = list() - for arg in options.appid: - if ( - arg == common.CONFIG_FILE - or Path(arg).parent.name == 'config' - or Path(arg).parent.parent.name == 'config' # localized - ): - paths.append(arg) - - failed = 0 - if paths: - for path in paths: - options.appid.remove(path) - if not lint_config(path): - failed += 1 - # an empty list of appids means check all apps, avoid that if files were given - if not options.appid: - sys.exit(failed) - - if not lint_metadata(options): - failed += 1 - - if failed: - sys.exit(failed) - - -def lint_metadata(options): - apps = common.read_app_args(options.appid) + # Get all apps... + allapps = metadata.read_metadata(options.appid) + apps = common.read_app_args(options.appid, allapps, False) anywarns = check_for_unsupported_metadata_files() apps_check_funcs = [] - if not options.appid: + if len(options.appid) == 0: # otherwise it finds tons of unused extlibs apps_check_funcs.append(check_extlib_dir) for check_func in apps_check_funcs: @@ -1025,41 +601,37 @@ def lint_metadata(options): if app.Disabled: continue + if options.force_yamllint: + import yamllint # throw error if it is not installed + yamllint # make pyflakes ignore this + # only run yamllint when linting individual apps. - if options.appid or options.force_yamllint: + if len(options.appid) > 0 or options.force_yamllint: + # run yamllint on app metadata - ymlpath = Path('metadata') / (appid + '.yml') - if ymlpath.is_file(): + ymlpath = os.path.join('metadata', appid + '.yml') + if os.path.isfile(ymlpath): yamllintresult = common.run_yamllint(ymlpath) - if yamllintresult: + if yamllintresult != '': print(yamllintresult) # run yamllint on srclib metadata srclibs = set() - for build in app.get('Builds', []): + for build in app.builds: for srclib in build.srclibs: - name, _ref, _number, _subdir = common.parse_srclib_spec(srclib) - srclibs.add(name + '.yml') + srclibs.add(srclib) for srclib in srclibs: - srclibpath = Path('srclibs') / srclib - if srclibpath.is_file(): - if platform.system() == 'Windows': - # Handle symlink on Windows - symlink = srclibpath.read_text() - if symlink in srclibs: - continue - elif (srclibpath.parent / symlink).is_file(): - srclibpath = srclibpath.parent / symlink + name, ref, number, subdir = common.parse_srclib_spec(srclib) + srclibpath = os.path.join('srclibs', name + '.yml') + if os.path.isfile(srclibpath): yamllintresult = common.run_yamllint(srclibpath) - if yamllintresult: + if yamllintresult != '': print(yamllintresult) app_check_funcs = [ check_app_field_types, - check_antiFeatures, check_regexes, check_update_check_data_url, - check_update_check_data_int, check_vercode_operation, check_ucm_tags, check_char_limits, @@ -1070,14 +642,12 @@ def lint_metadata(options): check_categories, check_duplicates, check_mediawiki_links, + check_bulleted_lists, check_builds, check_files_dir, check_format, check_license_tag, check_current_version_code, - check_updates_expected, - check_updates_ucm_http_aum_pattern, - check_certificate_pinned_binaries, ] for check_func in app_check_funcs: @@ -1085,7 +655,8 @@ def lint_metadata(options): anywarns = True print("%s: %s" % (appid, warn)) - return not anywarns + if anywarns: + sys.exit(1) # A compiled, public domain list of official SPDX license tags. generated @@ -1114,8 +685,9 @@ APPROVED_LICENSES = [ 'Artistic-1.0-Perl', 'Artistic-1.0-cl8', 'Artistic-2.0', - 'BSD-1-Clause', + 'Beerware', 'BSD-2-Clause', + 'BSD-2-Clause-FreeBSD', 'BSD-2-Clause-Patent', 'BSD-3-Clause', 'BSD-3-Clause-Clear', @@ -1123,8 +695,6 @@ APPROVED_LICENSES = [ 'BSD-4-Clause', 'BSL-1.0', 'BitTorrent-1.1', - 'CAL-1.0', - 'CAL-1.0-Combined-Work-Exception', 'CATOSL-1.1', 'CC-BY-4.0', 'CC-BY-SA-4.0', @@ -1187,6 +757,7 @@ APPROVED_LICENSES = [ 'LiLiQ-Rplus-1.1', 'MIT', 'MIT-0', + 'MIT-CMU', 'MPL-1.0', 'MPL-1.1', 'MPL-2.0', @@ -1195,7 +766,6 @@ APPROVED_LICENSES = [ 'MS-RL', 'MirOS', 'Motosoto', - 'MulanPSL-2.0', 'Multics', 'NASA-1.3', 'NCSA', @@ -1211,12 +781,9 @@ APPROVED_LICENSES = [ 'ODbL-1.0', 'OFL-1.0', 'OFL-1.1', - 'OFL-1.1-RFN', - 'OFL-1.1-no-RFN', 'OGTSL', 'OLDAP-2.3', 'OLDAP-2.7', - 'OLDAP-2.8', 'OSET-PL-2.1', 'OSL-1.0', 'OSL-1.1', @@ -1240,9 +807,7 @@ APPROVED_LICENSES = [ 'SPL-1.0', 'SimPL-2.0', 'Sleepycat', - 'UCL-1.0', 'UPL-1.0', - 'Unicode-DFS-2016', 'Unlicense', 'VSL-1.0', 'Vim', @@ -1252,6 +817,7 @@ APPROVED_LICENSES = [ 'X11', 'XFree86-1.1', 'Xnet', + 'XSkat', 'YPL-1.1', 'ZPL-2.0', 'ZPL-2.1', diff --git a/fdroidserver/looseversion.py b/fdroidserver/looseversion.py deleted file mode 100644 index c2a32213..00000000 --- a/fdroidserver/looseversion.py +++ /dev/null @@ -1,300 +0,0 @@ -# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 -# -------------------------------------------- -# -# 1. This LICENSE AGREEMENT is between the Python Software Foundation -# ("PSF"), and the Individual or Organization ("Licensee") accessing and -# otherwise using this software ("Python") in source or binary form and -# its associated documentation. -# -# 2. Subject to the terms and conditions of this License Agreement, PSF hereby -# grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, -# analyze, test, perform and/or display publicly, prepare derivative works, -# distribute, and otherwise use Python alone or in any derivative version, -# provided, however, that PSF's License Agreement and PSF's notice of copyright, -# i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -# 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 Python Software Foundation; -# All Rights Reserved" are retained in Python alone or in any derivative version -# prepared by Licensee. -# -# 3. In the event Licensee prepares a derivative work that is based on -# or incorporates Python or any part thereof, and wants to make -# the derivative work available to others as provided herein, then -# Licensee hereby agrees to include in any such work a brief summary of -# the changes made to Python. -# -# 4. PSF is making Python available to Licensee on an "AS IS" -# basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -# IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -# INFRINGE ANY THIRD PARTY RIGHTS. -# -# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -# FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. -# -# 6. This License Agreement will automatically terminate upon a material -# breach of its terms and conditions. -# -# 7. Nothing in this License Agreement shall be deemed to create any -# relationship of agency, partnership, or joint venture between PSF and -# Licensee. This License Agreement does not grant permission to use PSF -# trademarks or trade name in a trademark sense to endorse or promote -# products or services of Licensee, or any third party. -# -# 8. By copying, installing or otherwise using Python, Licensee -# agrees to be bound by the terms and conditions of this License -# Agreement. -# -# SPDX-License-Identifier: Python-2.0 -# -# downloaded from: -# https://github.com/effigies/looseversion/blob/e1a5a176a92dc6825deda4205c1be6d05e9ed352/src/looseversion/__init__.py - -"""Provides classes to represent module version numbers (one class for -each style of version numbering). There are currently two such classes -implemented: StrictVersion and LooseVersion. - -Every version number class implements the following interface: - * the 'parse' method takes a string and parses it to some internal - representation; if the string is an invalid version number, - 'parse' raises a ValueError exception - * the class constructor takes an optional string argument which, - if supplied, is passed to 'parse' - * __str__ reconstructs the string that was passed to 'parse' (or - an equivalent string -- ie. one that will generate an equivalent - version number instance) - * __repr__ generates Python code to recreate the version number instance - * _cmp compares the current instance with either another instance - of the same class or a string (which will be parsed to an instance - of the same class, thus must follow the same rules) -""" -import re -import sys - -__license__ = "Python License 2.0" - -# The rules according to Greg Stein: -# 1) a version number has 1 or more numbers separated by a period or by -# sequences of letters. If only periods, then these are compared -# left-to-right to determine an ordering. -# 2) sequences of letters are part of the tuple for comparison and are -# compared lexicographically -# 3) recognize the numeric components may have leading zeroes -# -# The LooseVersion class below implements these rules: a version number -# string is split up into a tuple of integer and string components, and -# comparison is a simple tuple comparison. This means that version -# numbers behave in a predictable and obvious way, but a way that might -# not necessarily be how people *want* version numbers to behave. There -# wouldn't be a problem if people could stick to purely numeric version -# numbers: just split on period and compare the numbers as tuples. -# However, people insist on putting letters into their version numbers; -# the most common purpose seems to be: -# - indicating a "pre-release" version -# ('alpha', 'beta', 'a', 'b', 'pre', 'p') -# - indicating a post-release patch ('p', 'pl', 'patch') -# but of course this can't cover all version number schemes, and there's -# no way to know what a programmer means without asking him. -# -# The problem is what to do with letters (and other non-numeric -# characters) in a version number. The current implementation does the -# obvious and predictable thing: keep them as strings and compare -# lexically within a tuple comparison. This has the desired effect if -# an appended letter sequence implies something "post-release": -# eg. "0.99" < "0.99pl14" < "1.0", and "5.001" < "5.001m" < "5.002". -# -# However, if letters in a version number imply a pre-release version, -# the "obvious" thing isn't correct. Eg. you would expect that -# "1.5.1" < "1.5.2a2" < "1.5.2", but under the tuple/lexical comparison -# implemented here, this just isn't so. -# -# Two possible solutions come to mind. The first is to tie the -# comparison algorithm to a particular set of semantic rules, as has -# been done in the StrictVersion class above. This works great as long -# as everyone can go along with bondage and discipline. Hopefully a -# (large) subset of Python module programmers will agree that the -# particular flavor of bondage and discipline provided by StrictVersion -# provides enough benefit to be worth using, and will submit their -# version numbering scheme to its domination. The free-thinking -# anarchists in the lot will never give in, though, and something needs -# to be done to accommodate them. -# -# Perhaps a "moderately strict" version class could be implemented that -# lets almost anything slide (syntactically), and makes some heuristic -# assumptions about non-digits in version number strings. This could -# sink into special-case-hell, though; if I was as talented and -# idiosyncratic as Larry Wall, I'd go ahead and implement a class that -# somehow knows that "1.2.1" < "1.2.2a2" < "1.2.2" < "1.2.2pl3", and is -# just as happy dealing with things like "2g6" and "1.13++". I don't -# think I'm smart enough to do it right though. -# -# In any case, I've coded the test suite for this module (see -# ../test/test_version.py) specifically to fail on things like comparing -# "1.2a2" and "1.2". That's not because the *code* is doing anything -# wrong, it's because the simple, obvious design doesn't match my -# complicated, hairy expectations for real-world version numbers. It -# would be a snap to fix the test suite to say, "Yep, LooseVersion does -# the Right Thing" (ie. the code matches the conception). But I'd rather -# have a conception that matches common notions about version numbers. - - -if sys.version_info >= (3,): - - class _Py2Int(int): - """Integer object that compares < any string""" - - def __gt__(self, other): - if isinstance(other, str): - return False - return super().__gt__(other) - - def __lt__(self, other): - if isinstance(other, str): - return True - return super().__lt__(other) - -else: - _Py2Int = int - - -class LooseVersion(object): - """Version numbering for anarchists and software realists. - Implements the standard interface for version number classes as - described above. A version number consists of a series of numbers, - separated by either periods or strings of letters. When comparing - version numbers, the numeric components will be compared - numerically, and the alphabetic components lexically. The following - are all valid version numbers, in no particular order: - - 1.5.1 - 1.5.2b2 - 161 - 3.10a - 8.02 - 3.4j - 1996.07.12 - 3.2.pl0 - 3.1.1.6 - 2g6 - 11g - 0.960923 - 2.2beta29 - 1.13++ - 5.5.kw - 2.0b1pl0 - - In fact, there is no such thing as an invalid version number under - this scheme; the rules for comparison are simple and predictable, - but may not always give the results you want (for some definition - of "want"). - """ - - component_re = re.compile(r"(\d+ | [a-z]+ | \.)", re.VERBOSE) - - def __init__(self, vstring=None): - if vstring: - self.parse(vstring) - - def __eq__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return NotImplemented - return c == 0 - - def __lt__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return NotImplemented - return c < 0 - - def __le__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return NotImplemented - return c <= 0 - - def __gt__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return NotImplemented - return c > 0 - - def __ge__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return NotImplemented - return c >= 0 - - def parse(self, vstring): - # I've given up on thinking I can reconstruct the version string - # from the parsed tuple -- so I just store the string here for - # use by __str__ - self.vstring = vstring - components = [x for x in self.component_re.split(vstring) if x and x != "."] - for i, obj in enumerate(components): - try: - components[i] = int(obj) - except ValueError: - pass - - self.version = components - - def __str__(self): - return self.vstring - - def __repr__(self): - return "LooseVersion ('%s')" % str(self) - - def _cmp(self, other): - other = self._coerce(other) - if other is NotImplemented: - return NotImplemented - - if self.version == other.version: - return 0 - if self.version < other.version: - return -1 - if self.version > other.version: - return 1 - return NotImplemented - - @classmethod - def _coerce(cls, other): - if isinstance(other, cls): - return other - elif isinstance(other, str): - return cls(other) - elif "distutils" in sys.modules: - # Using this check to avoid importing distutils and suppressing the warning - try: - from distutils.version import LooseVersion as deprecated - except ImportError: - return NotImplemented - if isinstance(other, deprecated): - return cls(str(other)) - return NotImplemented - - -class LooseVersion2(LooseVersion): - """LooseVersion variant that restores Python 2 semantics - - In Python 2, comparing LooseVersions where paired components could be string - and int always resulted in the string being "greater". In Python 3, this produced - a TypeError. - """ - - def parse(self, vstring): - # I've given up on thinking I can reconstruct the version string - # from the parsed tuple -- so I just store the string here for - # use by __str__ - self.vstring = vstring - components = [x for x in self.component_re.split(vstring) if x and x != "."] - for i, obj in enumerate(components): - try: - components[i] = _Py2Int(obj) - except ValueError: - pass - - self.version = components diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 0d9195be..b4b0ab26 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -18,19 +18,21 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import logging -import math import os -import platform import re +import glob +import logging +import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader +import importlib from collections import OrderedDict -from pathlib import Path -import ruamel.yaml - -from . import _, common -from ._yaml import yaml -from .exception import MetaDataException +import fdroidserver.common +from fdroidserver import _ +from fdroidserver.exception import MetaDataException, FDroidException srclibs = None warnings_action = None @@ -41,7 +43,7 @@ VALID_USERNAME_REGEX = re.compile(r'^[a-z\d](?:[a-z\d/._-]){0,38}$', re.IGNORECA def _warn_or_exception(value, cause=None): - """Output warning or Exception depending on -W.""" + '''output warning or Exception depending on -W''' if warnings_action == 'ignore': pass elif warnings_action == 'error': @@ -67,7 +69,9 @@ yaml_app_field_order = [ 'Translation', 'Changelog', 'Donate', + 'FlattrID', 'Liberapay', + 'LiberapayID', 'OpenCollective', 'Bitcoin', 'Litecoin', @@ -85,8 +89,6 @@ yaml_app_field_order = [ '\n', 'Builds', '\n', - 'AllowedAPKSigningKeys', - '\n', 'MaintainerNotes', '\n', 'ArchivePolicy', @@ -107,6 +109,7 @@ yaml_app_fields = [x for x in yaml_app_field_order if x != '\n'] class App(dict): + def __init__(self, copydict=None): if copydict: super().__init__(copydict) @@ -114,7 +117,7 @@ class App(dict): super().__init__() self.Disabled = None - self.AntiFeatures = dict() + self.AntiFeatures = [] self.Provides = None self.Categories = [] self.License = 'Unknown' @@ -127,7 +130,9 @@ class App(dict): self.Translation = '' self.Changelog = '' self.Donate = None + self.FlattrID = None self.Liberapay = None + self.LiberapayID = None self.OpenCollective = None self.Bitcoin = None self.Litecoin = None @@ -139,13 +144,12 @@ class App(dict): self.RepoType = '' self.Repo = '' self.Binaries = None - self.AllowedAPKSigningKeys = [] self.MaintainerNotes = '' self.ArchivePolicy = None self.AutoUpdateMode = 'None' self.UpdateCheckMode = 'None' self.UpdateCheckIgnore = None - self.VercodeOperation = [] + self.VercodeOperation = None self.UpdateCheckName = None self.UpdateCheckData = None self.CurrentVersion = '' @@ -154,7 +158,8 @@ class App(dict): self.id = None self.metadatapath = None - self.Builds = [] + self.builds = [] + self.comments = {} self.added = None self.lastUpdated = None @@ -173,7 +178,15 @@ class App(dict): else: raise AttributeError("No such attribute: " + name) + def get_last_build(self): + if len(self.builds) > 0: + return self.builds[-1] + else: + return Build() + +TYPE_UNKNOWN = 0 +TYPE_OBSOLETE = 1 TYPE_STRING = 2 TYPE_BOOL = 3 TYPE_LIST = 4 @@ -181,19 +194,15 @@ TYPE_SCRIPT = 5 TYPE_MULTILINE = 6 TYPE_BUILD = 7 TYPE_INT = 8 -TYPE_STRINGMAP = 9 fieldtypes = { 'Description': TYPE_MULTILINE, 'MaintainerNotes': TYPE_MULTILINE, 'Categories': TYPE_LIST, - 'AntiFeatures': TYPE_STRINGMAP, - 'RequiresRoot': TYPE_BOOL, - 'AllowedAPKSigningKeys': TYPE_LIST, - 'Builds': TYPE_BUILD, - 'VercodeOperation': TYPE_LIST, - 'CurrentVersionCode': TYPE_INT, - 'ArchivePolicy': TYPE_INT, + 'AntiFeatures': TYPE_LIST, + 'Build': TYPE_BUILD, + 'BuildVersion': TYPE_OBSOLETE, + 'UseBuilt': TYPE_OBSOLETE, } @@ -218,8 +227,8 @@ build_flags = [ 'patch', 'gradle', 'maven', + 'buildozer', 'output', - 'binary', 'srclibs', 'oldsdkloc', 'encoding', @@ -238,13 +247,13 @@ build_flags = [ 'preassemble', 'gradleprops', 'antcommands', - 'postbuild', 'novcheck', 'antifeatures', ] class Build(dict): + def __init__(self, copydict=None): super().__init__() self.disable = '' @@ -256,9 +265,9 @@ class Build(dict): self.init = '' self.patch = [] self.gradle = [] - self.maven = None + self.maven = False + self.buildozer = False self.output = None - self.binary = None self.srclibs = [] self.oldsdkloc = False self.encoding = None @@ -277,9 +286,8 @@ class Build(dict): self.preassemble = [] self.gradleprops = [] self.antcommands = [] - self.postbuild = '' self.novcheck = False - self.antifeatures = dict() + self.antifeatures = [] if copydict: super().__init__(copydict) return @@ -299,12 +307,8 @@ class Build(dict): else: raise AttributeError("No such attribute: " + name) - @classmethod - def to_yaml(cls, representer, node): - return representer.represent_dict(node) - def build_method(self): - for f in ['maven', 'gradle']: + for f in ['maven', 'gradle', 'buildozer']: if self.get(f): return f if self.output: @@ -315,25 +319,19 @@ class Build(dict): def output_method(self): if self.output: return 'raw' - for f in ['maven', 'gradle']: + for f in ['maven', 'gradle', 'buildozer']: if self.get(f): return f return 'ant' - def ndk_path(self) -> str: - """Return the path string of the first configured NDK or an empty string.""" - ndk = self.ndk - if isinstance(ndk, list): - ndk = self.ndk[0] - path = common.config['ndk_paths'].get(ndk) - if path and not isinstance(path, str): - raise TypeError('NDK path is not string') - if path: - return path - for vsn, path in common.config['ndk_paths'].items(): - if not vsn.endswith("_orig") and path and os.path.basename(path) == ndk: - return path - return '' + def ndk_path(self): + version = self.ndk + if not version: + version = 'r12b' # falls back to latest + paths = fdroidserver.common.config['ndk_paths'] + if version not in paths: + return '' + return paths[version] flagtypes = { @@ -354,13 +352,12 @@ flagtypes = { 'init': TYPE_SCRIPT, 'prebuild': TYPE_SCRIPT, 'build': TYPE_SCRIPT, - 'postbuild': TYPE_SCRIPT, 'submodules': TYPE_BOOL, 'oldsdkloc': TYPE_BOOL, 'forceversion': TYPE_BOOL, 'forcevercode': TYPE_BOOL, 'novcheck': TYPE_BOOL, - 'antifeatures': TYPE_STRINGMAP, + 'antifeatures': TYPE_LIST, 'timeout': TYPE_INT, } @@ -371,8 +368,9 @@ def flagtype(name): return TYPE_STRING -class FieldValidator: - """Designate App metadata field types and checks that it matches. +class FieldValidator(): + """ + Designates App metadata field types and checks that it matches 'name' - The long name of the field type 'matching' - List of possible values or regex expression @@ -395,21 +393,24 @@ class FieldValidator: values = [v] for v in values: if not self.compiled.match(v): - _warn_or_exception( - _( - "'{value}' is not a valid {field} in {appid}. Regex pattern: {pattern}" - ).format( - value=v, field=self.name, appid=appid, pattern=self.matching - ) - ) + _warn_or_exception(_("'{value}' is not a valid {field} in {appid}. Regex pattern: {pattern}") + .format(value=v, field=self.name, appid=appid, pattern=self.matching)) # Generic value types valuetypes = { + FieldValidator("Flattr ID", + r'^[0-9a-z]+$', + ['FlattrID']), + FieldValidator("Liberapay", VALID_USERNAME_REGEX, ['Liberapay']), + FieldValidator("Liberapay ID", + r'^[0-9]+$', + ['LiberapayID']), + FieldValidator("Open Collective", VALID_USERNAME_REGEX, ['OpenCollective']), @@ -427,7 +428,7 @@ valuetypes = { ["Bitcoin"]), FieldValidator("Litecoin address", - r'^([LM3][a-km-zA-HJ-NP-Z1-9]{26,33}|ltc1[a-z0-9]{39})$', + r'^[LM3][a-km-zA-HJ-NP-Z1-9]{26,33}$', ["Litecoin"]), FieldValidator("Repo Type", @@ -438,16 +439,20 @@ valuetypes = { r'^http[s]?://', ["Binaries"]), - FieldValidator("AllowedAPKSigningKeys", - r'^[a-fA-F0-9]{64}$', - ["AllowedAPKSigningKeys"]), + FieldValidator("Archive Policy", + r'^[0-9]+ versions$', + ["ArchivePolicy"]), + + FieldValidator("Anti-Feature", + r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets|KnownVuln|ApplicationDebuggable|NoSourceSince)$', + ["AntiFeatures"]), FieldValidator("Auto Update Mode", - r"^(Version.*|None)$", + r"^(Version .+|None)$", ["AutoUpdateMode"]), FieldValidator("Update Check Mode", - r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|HTTP|Static|None)$", + r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", ["UpdateCheckMode"]) } @@ -460,45 +465,38 @@ def check_metadata(app): def parse_yaml_srclib(metadatapath): - thisinfo = {'RepoType': '', 'Repo': '', 'Subdir': None, 'Prepare': None} - if not metadatapath.exists(): - _warn_or_exception( - _("Invalid scrlib metadata: '{file}' does not exist").format( - file=metadatapath - ) - ) + thisinfo = {'RepoType': '', + 'Repo': '', + 'Subdir': None, + 'Prepare': None} + + if not os.path.exists(metadatapath): + _warn_or_exception(_("Invalid scrlib metadata: '{file}' " + "does not exist" + .format(file=metadatapath))) return thisinfo - with metadatapath.open("r", encoding="utf-8") as f: + with open(metadatapath, "r", encoding="utf-8") as f: try: - data = yaml.load(f) + data = yaml.load(f, Loader=SafeLoader) if type(data) is not dict: - if platform.system() == 'Windows': - # Handle symlink on Windows - symlink = metadatapath.parent / metadatapath.read_text(encoding='utf-8') - if symlink.is_file(): - with symlink.open("r", encoding="utf-8") as s: - data = yaml.load(s) - if type(data) is not dict: - raise ruamel.yaml.YAMLError( - _('{file} is blank or corrupt!').format(file=metadatapath) - ) - except ruamel.yaml.YAMLError as e: + raise yaml.error.YAMLError(_('{file} is blank or corrupt!') + .format(file=metadatapath)) + except yaml.error.YAMLError as e: _warn_or_exception(_("Invalid srclib metadata: could not " "parse '{file}'") .format(file=metadatapath) + '\n' - + common.run_yamllint(metadatapath, indent=4), + + fdroidserver.common.run_yamllint(metadatapath, + indent=4), cause=e) return thisinfo - for key in data: - if key not in thisinfo: - _warn_or_exception( - _("Invalid srclib metadata: unknown key '{key}' in '{file}'").format( - key=key, file=metadatapath - ) - ) + for key in data.keys(): + if key not in thisinfo.keys(): + _warn_or_exception(_("Invalid srclib metadata: unknown key " + "'{key}' in '{file}'") + .format(key=key, file=metadatapath)) return thisinfo else: if key == 'Subdir': @@ -508,11 +506,8 @@ def parse_yaml_srclib(metadatapath): thisinfo[key] = data[key] elif data[key] is None: thisinfo[key] = [''] - elif key == 'Prepare' or flagtype(key) == TYPE_SCRIPT: - if isinstance(data[key], list): - thisinfo[key] = data[key] - else: - thisinfo[key] = [data[key]] if data[key] else [] + elif key == 'Prepare' and isinstance(data[key], list): + thisinfo[key] = ' && '.join(data[key]) else: thisinfo[key] = str(data[key] or '') @@ -537,24 +532,27 @@ def read_srclibs(): srclibs = {} - srclibs_dir = Path('srclibs') - srclibs_dir.mkdir(exist_ok=True) + srcdir = 'srclibs' + if not os.path.exists(srcdir): + os.makedirs(srcdir) - for metadatapath in sorted(srclibs_dir.glob('*.yml')): - srclibs[metadatapath.stem] = parse_yaml_srclib(metadatapath) + for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.yml'))): + srclibname = os.path.basename(metadatapath[:-4]) + srclibs[srclibname] = parse_yaml_srclib(metadatapath) -def read_metadata(appid_to_vercode={}, sort_by_time=False): - """Return a list of App instances sorted newest first. +def read_metadata(appids={}, refresh=True, sort_by_time=False): + """Return a list of App instances sorted newest first This reads all of the metadata files in a 'data' repository, then builds a list of App instances from those files. The list is sorted based on creation time, newest first. Most of the time, the newer files are the most interesting. - appid_to_vercode is a dict with appids a keys and versionCodes as values. + appids is a dict with appids a keys and versionCodes as values. """ + # Always read the srclibs before the apps, since they can use a srlib as # their source repository. read_srclibs() @@ -562,17 +560,28 @@ def read_metadata(appid_to_vercode={}, sort_by_time=False): apps = OrderedDict() for basedir in ('metadata', 'tmp'): - Path(basedir).mkdir(exist_ok=True) + if not os.path.exists(basedir): + os.makedirs(basedir) - if appid_to_vercode: - metadatafiles = common.get_metadata_files(appid_to_vercode) + if appids: + vercodes = fdroidserver.common.read_pkg_args(appids) + found_invalid = False + metadatafiles = [] + for appid in vercodes.keys(): + f = os.path.join('metadata', '%s.yml' % appid) + if os.path.exists(f): + metadatafiles.append(f) + else: + found_invalid = True + logging.critical(_("No such package: %s") % appid) + if found_invalid: + raise FDroidException(_("Found invalid appids in arguments")) else: - metadatafiles = list(Path('metadata').glob('*.yml')) + list( - Path('.').glob('.fdroid.yml') - ) + metadatafiles = (glob.glob(os.path.join('metadata', '*.yml')) + + glob.glob('.fdroid.yml')) if sort_by_time: - entries = ((path.stat().st_mtime, path) for path in metadatafiles) + entries = ((os.stat(path).st_mtime, path) for path in metadatafiles) metadatafiles = [] for _ignored, path in sorted(entries, reverse=True): metadatafiles.append(path) @@ -581,704 +590,403 @@ def read_metadata(appid_to_vercode={}, sort_by_time=False): metadatafiles = sorted(metadatafiles) for metadatapath in metadatafiles: - appid = metadatapath.stem - if appid != '.fdroid' and not common.is_valid_package_name(appid): - _warn_or_exception( - _("{appid} from {path} is not a valid Java Package Name!").format( - appid=appid, path=metadatapath - ) - ) + appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath)) + if appid != '.fdroid' and not fdroidserver.common.is_valid_package_name(appid): + _warn_or_exception(_("{appid} from {path} is not a valid Java Package Name!") + .format(appid=appid, path=metadatapath)) if appid in apps: - _warn_or_exception( - _("Found multiple metadata files for {appid}").format(appid=appid) - ) - app = parse_metadata(metadatapath) + _warn_or_exception(_("Found multiple metadata files for {appid}") + .format(appid=appid)) + app = parse_metadata(metadatapath, appid in appids, refresh) check_metadata(app) apps[app.id] = app return apps -def parse_metadata(metadatapath): - """Parse metadata file, also checking the source repo for .fdroid.yml. +# Port legacy ';' separators +list_sep = re.compile(r'[,;]') - This function finds the relevant files, gets them parsed, converts - dicts into App and Build instances, and combines the results into - a single App instance. - If this is a metadata file from fdroiddata, it will first load the - source repo type and URL from fdroiddata, then read .fdroid.yml if - it exists, then include the rest of the metadata as specified in - fdroiddata, so that fdroiddata has precedence over the metadata in - the source code. +def split_list_values(s): + res = [] + for v in re.split(list_sep, s): + if not v: + continue + v = v.strip() + if not v: + continue + res.append(v) + return res - .fdroid.yml is embedded in the app's source repo, so it is - "user-generated". That means that it can have weird things in it - that need to be removed so they don't break the overall process, - e.g. if the upstream developer includes some broken field, it can - be overridden in the metadata file. - Parameters - ---------- - metadatapath - The file path to read. The "Application ID" aka "Package Name" - for the application comes from this filename. +def sorted_builds(builds): + return sorted(builds, key=lambda build: int(build.versionCode)) - Raises - ------ - FDroidException when there are syntax errors. - Returns - ------- - Returns a dictionary containing all the details of the - application. There are two major kinds of information in the - dictionary. Keys beginning with capital letters correspond - directory to identically named keys in the metadata file. Keys - beginning with lower case letters are generated in one way or - another, and are not found verbatim in the metadata. +esc_newlines = re.compile(r'\\( |\n)') + + +def post_metadata_parse(app): + # TODO keep native types, convert only for .txt metadata + for k, v in app.items(): + if type(v) in (float, int): + app[k] = str(v) + + if 'Builds' in app: + app['builds'] = app.pop('Builds') + + if 'flavours' in app and app['flavours'] == [True]: + app['flavours'] = 'yes' + + for field, fieldtype in fieldtypes.items(): + if fieldtype != TYPE_LIST: + continue + value = app.get(field) + if isinstance(value, str): + app[field] = [value, ] + elif value is not None: + app[field] = [str(i) for i in value] + + def _yaml_bool_unmapable(v): + return v in (True, False, [True], [False]) + + def _yaml_bool_unmap(v): + if v is True: + return 'yes' + elif v is False: + return 'no' + elif v == [True]: + return ['yes'] + elif v == [False]: + return ['no'] + + _bool_allowed = ('maven', 'buildozer') + + builds = [] + if 'builds' in app: + for build in app['builds']: + if not isinstance(build, Build): + build = Build(build) + for k, v in build.items(): + if not (v is None): + if flagtype(k) == TYPE_LIST: + if _yaml_bool_unmapable(v): + build[k] = _yaml_bool_unmap(v) + + if isinstance(v, str): + build[k] = [v] + elif isinstance(v, bool): + if v: + build[k] = ['yes'] + else: + build[k] = [] + elif flagtype(k) is TYPE_INT: + build[k] = str(v) + elif flagtype(k) is TYPE_STRING: + if isinstance(v, bool) and k in _bool_allowed: + build[k] = v + else: + if _yaml_bool_unmapable(v): + build[k] = _yaml_bool_unmap(v) + else: + build[k] = str(v) + builds.append(build) + + app.builds = sorted_builds(builds) + + +# Parse metadata for a single application. +# +# 'metadatapath' - the filename to read. The "Application ID" aka +# "Package Name" for the application comes from this +# filename. Pass None to get a blank entry. +# +# Returns a dictionary containing all the details of the application. There are +# two major kinds of information in the dictionary. Keys beginning with capital +# letters correspond directory to identically named keys in the metadata file. +# Keys beginning with lower case letters are generated in one way or another, +# and are not found verbatim in the metadata. +# +# Known keys not originating from the metadata are: +# +# 'builds' - a list of dictionaries containing build information +# for each defined build +# 'comments' - a list of comments from the metadata file. Each is +# a list of the form [field, comment] where field is +# the name of the field it preceded in the metadata +# file. Where field is None, the comment goes at the +# end of the file. Alternatively, 'build:version' is +# for a comment before a particular build version. +# 'descriptionlines' - original lines of description as formatted in the +# metadata file. +# + + +bool_true = re.compile(r'([Yy]es|[Tt]rue)') +bool_false = re.compile(r'([Nn]o|[Ff]alse)') + + +def _decode_bool(s): + if bool_true.match(s): + return True + if bool_false.match(s): + return False + _warn_or_exception(_("Invalid boolean '%s'") % s) + + +def parse_metadata(metadatapath, check_vcs=False, refresh=True): + '''parse metadata file, optionally checking the git repo for metadata first''' - """ - metadatapath = Path(metadatapath) app = App() - app.metadatapath = metadatapath.as_posix() - if metadatapath.suffix == '.yml': - with metadatapath.open('r', encoding='utf-8') as mf: - app.update(parse_yaml_metadata(mf)) + app.metadatapath = metadatapath + name, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath)) + if name == '.fdroid': + check_vcs = False else: - _warn_or_exception( - _('Unknown metadata format: {path} (use: *.yml)').format(path=metadatapath) - ) + app.id = name - if metadatapath.stem != '.fdroid': - app.id = metadatapath.stem - parse_localized_antifeatures(app) + if metadatapath.endswith('.yml'): + with open(metadatapath, 'r') as mf: + parse_yaml_metadata(mf, app) + else: + _warn_or_exception(_('Unknown metadata format: {path} (use: *.yml)') + .format(path=metadatapath)) - if metadatapath.name != '.fdroid.yml' and app.Repo: - build_dir = common.get_build_dir(app) - metadata_in_repo = build_dir / '.fdroid.yml' - if metadata_in_repo.is_file(): - commit_id = common.get_head_commit_id(build_dir) - if commit_id is not None: - logging.debug( - _('Including metadata from %s@%s') % (metadata_in_repo, commit_id) - ) - else: - logging.debug( - _('Including metadata from {path}').format(path=metadata_in_repo) - ) + if check_vcs and app.Repo: + build_dir = fdroidserver.common.get_build_dir(app) + metadata_in_repo = os.path.join(build_dir, '.fdroid.yml') + if not os.path.isfile(metadata_in_repo): + vcs, build_dir = fdroidserver.common.setup_vcs(app) + if isinstance(vcs, fdroidserver.common.vcs_git): + vcs.gotorevision('HEAD', refresh) # HEAD since we can't know where else to go + if os.path.isfile(metadata_in_repo): + logging.debug('Including metadata from ' + metadata_in_repo) + # do not include fields already provided by main metadata file app_in_repo = parse_metadata(metadata_in_repo) for k, v in app_in_repo.items(): if k not in app: app[k] = v - builds = [] - for build in app.get('Builds', []): - builds.append(Build(build)) - if builds: - app['Builds'] = builds + post_metadata_parse(app) - # if only .fdroid.yml was found, then this finds the appid if not app.id: - if app.get('Builds'): - build = app['Builds'][-1] + if app.builds: + build = app.builds[-1] if build.subdir: - root_dir = Path(build.subdir) + root_dir = build.subdir else: - root_dir = Path('.') - paths = common.manifest_paths(root_dir, build.gradle) - _ignored, _ignored, app.id = common.parse_androidmanifests(paths, app) + root_dir = '.' + paths = fdroidserver.common.manifest_paths(root_dir, build.gradle) + _ignored, _ignored, app.id = fdroidserver.common.parse_androidmanifests(paths, app) return app -def parse_yaml_metadata(mf): - """Parse the .yml file and post-process it. - - This function handles parsing a metadata YAML file and converting - all the various data types into a consistent internal - representation. The results are meant to update an existing App - instance or used as a plain dict. - - Clean metadata .yml files can be used directly, but in order to - make a better user experience for people editing .yml files, there - is post processing. That makes the parsing perform something like - Strict YAML. - - """ +def parse_yaml_metadata(mf, app): try: - yamldata = common.yaml.load(mf) - except ruamel.yaml.YAMLError as e: - _warn_or_exception( - _("could not parse '{path}'").format(path=mf.name) - + '\n' - + common.run_yamllint(mf.name, indent=4), - cause=e, - ) - - if yamldata is None or yamldata == '': - yamldata = dict() - if not isinstance(yamldata, dict): - _warn_or_exception( - _("'{path}' has invalid format, it should be a dictionary!").format( - path=mf.name - ) - ) - logging.error(_('Using blank dictionary instead of contents of {path}!').format( - path=mf.name) - ) - yamldata = dict() + yamldata = yaml.load(mf, Loader=SafeLoader) + except yaml.YAMLError as e: + _warn_or_exception(_("could not parse '{path}'") + .format(path=mf.name) + '\n' + + fdroidserver.common.run_yamllint(mf.name, + indent=4), + cause=e) deprecated_in_yaml = ['Provides'] - for field in tuple(yamldata.keys()): - if field not in yaml_app_fields + deprecated_in_yaml: - msg = _("Unrecognised app field '{fieldname}' in '{path}'").format( - fieldname=field, path=mf.name - ) - if Path(mf.name).name == '.fdroid.yml': - logging.error(msg) - del yamldata[field] - else: - _warn_or_exception(msg) + if yamldata: + for field in yamldata: + if field not in yaml_app_fields: + if field not in deprecated_in_yaml: + _warn_or_exception(_("Unrecognised app field " + "'{fieldname}' in '{path}'") + .format(fieldname=field, + path=mf.name)) - for deprecated_field in deprecated_in_yaml: - if deprecated_field in yamldata: - del yamldata[deprecated_field] - logging.warning( - _( - "Ignoring '{field}' in '{metapath}' " - "metadata because it is deprecated." - ).format(field=deprecated_field, metapath=mf.name) - ) + for deprecated_field in deprecated_in_yaml: + if deprecated_field in yamldata: + logging.warning(_("Ignoring '{field}' in '{metapath}' " + "metadata because it is deprecated.") + .format(field=deprecated_field, + metapath=mf.name)) + del(yamldata[deprecated_field]) - msg = _("Unrecognised build flag '{build_flag}' in '{path}'") - for build in yamldata.get('Builds', []): - for build_flag in build: - if build_flag not in build_flags: - _warn_or_exception(msg.format(build_flag=build_flag, path=mf.name)) - - post_parse_yaml_metadata(yamldata) - return yamldata - - -def parse_localized_antifeatures(app): - """Read in localized Anti-Features files from the filesystem. - - To support easy integration with Weblate and other translation - systems, there is a special type of metadata that can be - maintained in a Fastlane-style directory layout, where each field - is represented by a text file on directories that specified which - app it belongs to, which locale, etc. This function reads those - in and puts them into the internal dict, to be merged with any - related data that came from the metadata.yml file. - - This needs to be run after parse_yaml_metadata() since that - normalizes the data structure. Also, these values are lower - priority than what comes from the metadata file. So this should - not overwrite anything parse_yaml_metadata() puts into the App - instance. - - metadata///antifeatures/_.txt - metadata///antifeatures/.txt - - └── metadata/ - └── / - ├── en-US/ - │ └── antifeatures/ - │ ├── 123_Ads.txt -> "includes ad lib" - │ ├── 123_Tracking.txt -> "standard suspects" - │ └── NoSourceSince.txt -> "it vanished" - │ - └── zh-CN/ - └── antifeatures/ - └── 123_Ads.txt -> "包括广告库" - - Gets parsed into the metadata data structure: - - AntiFeatures: - NoSourceSince: - en-US: it vanished - Builds: - - versionCode: 123 - antifeatures: - Ads: - en-US: includes ad lib - zh-CN: 包括广告库 - Tracking: - en-US: standard suspects - - """ - app_dir = Path('metadata', app['id']) - if not app_dir.is_dir(): - return - af_dup_msg = _('Duplicate Anti-Feature declaration at {path} was ignored!') - - if app.get('AntiFeatures'): - app_has_AntiFeatures = True - else: - app_has_AntiFeatures = False - - has_versionCode = re.compile(r'^-?[0-9]+_.*') - has_antifeatures_from_app = set() - for build in app.get('Builds', []): - antifeatures = build.get('antifeatures') - if antifeatures: - has_antifeatures_from_app.add(build['versionCode']) - - for f in sorted(app_dir.glob('*/antifeatures/*.txt')): - path = f.as_posix() - left = path.index('/', 9) # 9 is length of "metadata/" - right = path.index('/', left + 1) - locale = path[left + 1 : right] - description = f.read_text() - if has_versionCode.match(f.stem): - i = f.stem.index('_') - versionCode = int(f.stem[:i]) - antifeature = f.stem[i + 1 :] - if versionCode in has_antifeatures_from_app: - logging.error(af_dup_msg.format(path=f)) - continue - if 'Builds' not in app: - app['Builds'] = [] - found = False - for build in app['Builds']: - # loop though builds again, there might be duplicate versionCodes - if versionCode == build['versionCode']: - found = True - if 'antifeatures' not in build: - build['antifeatures'] = dict() - if antifeature not in build['antifeatures']: - build['antifeatures'][antifeature] = dict() - build['antifeatures'][antifeature][locale] = description - if not found: - app['Builds'].append( - { - 'versionCode': versionCode, - 'antifeatures': { - antifeature: {locale: description}, - }, - } - ) - elif app_has_AntiFeatures: - logging.error(af_dup_msg.format(path=f)) - continue - else: - if 'AntiFeatures' not in app: - app['AntiFeatures'] = dict() - if f.stem not in app['AntiFeatures']: - app['AntiFeatures'][f.stem] = dict() - app['AntiFeatures'][f.stem][locale] = f.read_text() - - -def _normalize_type_int(k, v): - """Normalize anything that can be reliably converted to an integer.""" - if isinstance(v, int) and not isinstance(v, bool): - return v - if v is None: - return None - if isinstance(v, str): - try: - return int(v) - except ValueError: - pass - msg = _('{build_flag} must be an integer, found: {value}') - _warn_or_exception(msg.format(build_flag=k, value=v)) - - -def _normalize_type_string(v): - """Normalize any data to TYPE_STRING. - - YAML 1.2's booleans are all lowercase. - - Things like versionName are strings, but without quotes can be - numbers. Like "versionName: 1.0" would be a YAML float, but - should be a string. - - SHA-256 values are string values, but YAML 1.2 can interpret some - unquoted values as decimal ints. This converts those to a string - if they are over 50 digits. In the wild, the longest 0 padding on - a SHA-256 key fingerprint I found was 8 zeros. - - """ - if isinstance(v, bool): - if v: - return 'true' - return 'false' - if isinstance(v, float): - # YAML 1.2 values for NaN, Inf, and -Inf - if math.isnan(v): - return '.nan' - if math.isinf(v): - if v > 0: - return '.inf' - return '-.inf' - if v and isinstance(v, int): - if math.log10(v) > 50: # only if the int has this many digits - return '%064d' % v - return str(v) - - -def _normalize_type_stringmap(k, v): - """Normalize any data to TYPE_STRINGMAP. - - The internal representation of this format is a dict of dicts, - where the outer dict's keys are things like tag names of - Anti-Features, the inner dict's keys are locales, and the ultimate - values are human readable text. - - Metadata entries like AntiFeatures: can be written in many - forms, including a simple one-entry string, a list of strings, - a dict with keys and descriptions as values, or a dict with - localization. - - Returns - ------- - A dictionary with string keys, where each value is either a string - message or a dict with locale keys and string message values. - - """ - if v is None: - return dict() - if isinstance(v, str) or isinstance(v, int) or isinstance(v, float): - return {_normalize_type_string(v): dict()} - if isinstance(v, list) or isinstance(v, tuple) or isinstance(v, set): - retdict = dict() - for i in v: - if isinstance(i, dict): - # transitional format - if len(i) != 1: - _warn_or_exception( - _( - "'{value}' is not a valid {field}, should be {pattern}" - ).format(field=k, value=v, pattern='key: value') - ) - afname = _normalize_type_string(next(iter(i))) - desc = _normalize_type_string(next(iter(i.values()))) - retdict[afname] = {common.DEFAULT_LOCALE: desc} - else: - retdict[_normalize_type_string(i)] = {} - return retdict - - retdict = dict() - for af, afdict in v.items(): - key = _normalize_type_string(af) - if afdict: - if isinstance(afdict, dict): - retdict[key] = afdict - else: - retdict[key] = {common.DEFAULT_LOCALE: _normalize_type_string(afdict)} - else: - retdict[key] = dict() - - return retdict - - -def _normalize_type_list(k, v): - """Normalize any data to TYPE_LIST, which is always a list of strings.""" - if isinstance(v, dict): - msg = _('{build_flag} must be list or string, found: {value}') - _warn_or_exception(msg.format(build_flag=k, value=v)) - elif type(v) not in (list, tuple, set): - v = [v] - return [_normalize_type_string(i) for i in v] + if yamldata.get('Builds', None): + for build in yamldata.get('Builds', []): + # put all build flag keywords into a set to avoid + # excessive looping action + build_flag_set = set() + for build_flag in build.keys(): + build_flag_set.add(build_flag) + for build_flag in build_flag_set: + if build_flag not in build_flags: + _warn_or_exception( + _("Unrecognised build flag '{build_flag}' " + "in '{path}'").format(build_flag=build_flag, + path=mf.name)) + post_parse_yaml_metadata(yamldata) + app.update(yamldata) + return app def post_parse_yaml_metadata(yamldata): - """Convert human-readable metadata data structures into consistent data structures. - - "Be conservative in what is written out, be liberal in what is parsed." - https://en.wikipedia.org/wiki/Robustness_principle - - This also handles conversions that make metadata YAML behave - something like StrictYAML. Specifically, a field should have a - fixed value type, regardless of YAML 1.2's type auto-detection. - - TODO: None values should probably be treated as the string 'null', - since YAML 1.2 uses that for nulls - - """ - for k, v in yamldata.items(): - _fieldtype = fieldtype(k) - if _fieldtype == TYPE_LIST: - if v or v == 0: - yamldata[k] = _normalize_type_list(k, v) - elif _fieldtype == TYPE_INT: - # ArchivePolicy used to require " versions" in the value. - if k == 'ArchivePolicy' and isinstance(v, str): - v = v.split(' ', maxsplit=1)[0] - v = _normalize_type_int(k, v) - if v or v == 0: - yamldata[k] = v - elif _fieldtype == TYPE_STRING: - if v or v == 0: - yamldata[k] = _normalize_type_string(v) - elif _fieldtype == TYPE_STRINGMAP: - if v or v == 0: # TODO probably want just `if v:` - yamldata[k] = _normalize_type_stringmap(k, v) - elif _fieldtype == TYPE_BOOL: - yamldata[k] = bool(v) - else: - if type(v) in (float, int): - yamldata[k] = str(v) - - builds = [] + """transform yaml metadata to our internal data format""" for build in yamldata.get('Builds', []): - for k, v in build.items(): - if v is None: - continue + for flag in build.keys(): + _flagtype = flagtype(flag) - _flagtype = flagtype(k) - if _flagtype == TYPE_STRING: - if v or v == 0: - build[k] = _normalize_type_string(v) - elif _flagtype == TYPE_INT: - v = _normalize_type_int(k, v) - if v or v == 0: - build[k] = v - elif _flagtype in (TYPE_LIST, TYPE_SCRIPT): - if v or v == 0: - build[k] = _normalize_type_list(k, v) - elif _flagtype == TYPE_STRINGMAP: - if v or v == 0: - build[k] = _normalize_type_stringmap(k, v) - elif _flagtype == TYPE_BOOL: - build[k] = bool(v) - - builds.append(build) - - if builds: - yamldata['Builds'] = sorted(builds, key=lambda build: build['versionCode']) - - no_source_since = yamldata.get("NoSourceSince") - # do not overwrite the description if it is there - if no_source_since and not yamldata.get('AntiFeatures', {}).get('NoSourceSince'): - if 'AntiFeatures' not in yamldata: - yamldata['AntiFeatures'] = dict() - yamldata['AntiFeatures']['NoSourceSince'] = { - common.DEFAULT_LOCALE: no_source_since - } - - -def _format_multiline(value): - """TYPE_MULTILINE with newlines in them are saved as YAML literal strings.""" - if '\n' in value: - return ruamel.yaml.scalarstring.preserve_literal(str(value)) - return str(value) - - -def _format_list(value): - """TYPE_LIST should not contain null values.""" - return [v for v in value if v] - - -def _format_script(value): - """TYPE_SCRIPT with one value are converted to YAML string values.""" - value = [v for v in value if v] - if len(value) == 1: - return value[0] - return value - - -def _format_stringmap(appid, field, stringmap, versionCode=None): - """Format TYPE_STRINGMAP taking into account localized files in the metadata dir. - - If there are any localized versions on the filesystem already, - then move them all there. Otherwise, keep them in the .yml file. - - The directory for the localized files that is named after the - field is all lower case, following the convention set by Fastlane - metadata, and used by fdroidserver. - - """ - app_dir = Path('metadata', appid) - try: - next(app_dir.glob('*/%s/*.txt' % field.lower())) - files = [] - overwrites = [] - for name, descdict in stringmap.items(): - for locale, desc in descdict.items(): - outdir = app_dir / locale / field.lower() - if versionCode: - filename = '%d_%s.txt' % (versionCode, name) - else: - filename = '%s.txt' % name - outfile = outdir / filename - files.append(str(outfile)) - if outfile.exists(): - if desc != outfile.read_text(): - overwrites.append(str(outfile)) - else: - if not outfile.parent.exists(): - outfile.parent.mkdir(parents=True) - outfile.write_text(desc) - if overwrites: - _warn_or_exception( - _( - 'Conflicting "{field}" definitions between .yml and localized files:' - ).format(field=field) - + '\n' - + '\n'.join(sorted(overwrites)) - ) - logging.warning( - _('Moving Anti-Features declarations to localized files:') - + '\n' - + '\n'.join(sorted(files)) - ) - return - except StopIteration: - pass - make_list = True - outlist = [] - for name in sorted(stringmap): - outlist.append(name) - descdict = stringmap.get(name) - if descdict and any(descdict.values()): - make_list = False - break - if make_list: - return sorted(outlist, key=str.lower) - return stringmap - - -def _del_duplicated_NoSourceSince(app): - # noqa: D403 NoSourceSince is the word. - """NoSourceSince gets auto-added to AntiFeatures, but can also be manually added.""" - key = 'NoSourceSince' - if key in app: - no_source_since = app.get(key) - af_no_source_since = app.get('AntiFeatures', dict()).get(key) - if af_no_source_since == {common.DEFAULT_LOCALE: no_source_since}: - del app['AntiFeatures'][key] - - -def _builds_to_yaml(app): - """Reformat Builds: flags for output to YAML 1.2. - - This will strip any flag/value that is not set or is empty. - TYPE_BOOL fields are removed when they are false. 0 is valid - value, it should not be stripped, so there are special cases to - handle that. - - """ - builds = ruamel.yaml.comments.CommentedSeq() - for build in app.get('Builds', []): - b = ruamel.yaml.comments.CommentedMap() - for field in build_flags: - v = build.get(field) - if v is None or v is False or v == '' or v == dict() or v == list(): - continue - _flagtype = flagtype(field) - if _flagtype == TYPE_MULTILINE: - v = _format_multiline(v) - elif _flagtype == TYPE_LIST: - v = _format_list(v) - elif _flagtype == TYPE_SCRIPT: - v = _format_script(v) - elif _flagtype == TYPE_STRINGMAP: - v = _format_stringmap(app['id'], field, v, build['versionCode']) - - if v or v == 0: - b[field] = v - - builds.append(b) - - # insert extra empty lines between build entries - for i in range(1, len(builds)): - builds.yaml_set_comment_before_after_key(i, 'bogus') - builds.ca.items[i][1][-1].value = '\n' - - return builds - - -def _app_to_yaml(app): - cm = ruamel.yaml.comments.CommentedMap() - insert_newline = False - for field in yaml_app_field_order: - if field == '\n': - # next iteration will need to insert a newline - insert_newline = True - else: - value = app.get(field) - if value or field in ('Builds', 'ArchivePolicy'): - _fieldtype = fieldtype(field) - if field == 'Builds': - if app.get('Builds'): - cm.update({field: _builds_to_yaml(app)}) - elif field == 'Categories': - cm[field] = sorted(value, key=str.lower) - elif field == 'AntiFeatures': - v = _format_stringmap(app['id'], field, value) - if v: - cm[field] = v - elif field == 'AllowedAPKSigningKeys': - value = [str(i).lower() for i in value] - if len(value) == 1: - cm[field] = value[0] - else: - cm[field] = value - elif field == 'ArchivePolicy': - if value is None: - continue - cm[field] = value - elif _fieldtype == TYPE_MULTILINE: - v = _format_multiline(value) - if v: - cm[field] = v - elif _fieldtype == TYPE_SCRIPT: - v = _format_script(value) - if v: - cm[field] = v - else: - if value: - cm[field] = value - - if insert_newline: - # we need to prepend a newline in front of this field - insert_newline = False - # inserting empty lines is not supported so we add a - # bogus comment and over-write its value - cm.yaml_set_comment_before_after_key(field, 'bogus') - cm.ca.items[field][1][-1].value = '\n' - return cm + if _flagtype is TYPE_SCRIPT: + # concatenate script flags into a single string if they are stored as list + if isinstance(build[flag], list): + build[flag] = ' && '.join(build[flag]) + elif _flagtype is TYPE_STRING: + # things like versionNames are strings, but without quotes can be numbers + if isinstance(build[flag], float) or isinstance(build[flag], int): + build[flag] = str(build[flag]) + elif _flagtype is TYPE_INT: + # versionCode must be int + if not isinstance(build[flag], int): + _warn_or_exception(_('{build_flag} must be an integer, found: {value}') + .format(build_flag=flag, value=build[flag])) def write_yaml(mf, app): """Write metadata in yaml format. - This requires the 'rt' round trip dumper to maintain order and needs - custom indent settings, so it needs to instantiate its own YAML - instance. Therefore, this function deliberately avoids using any of - the common YAML parser setups. - - Parameters - ---------- - mf - active file discriptor for writing - app - app metadata to written to the YAML file - + :param mf: active file discriptor for writing + :param app: app metadata to written to the yaml file """ - _del_duplicated_NoSourceSince(app) + + # import rumael.yaml and check version + try: + import ruamel.yaml + except ImportError as e: + raise FDroidException('ruamel.yaml not installed, can not write metadata.') from e + if not ruamel.yaml.__version__: + raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..') + m = re.match(r'(?P[0-9]+)\.(?P[0-9]+)\.(?P[0-9]+)(-.+)?', + ruamel.yaml.__version__) + if not m: + raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml') + if int(m.group('major')) < 0 or int(m.group('minor')) < 13: + raise FDroidException('currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.'.format(ruamel.yaml.__version__)) + # suiteable version ruamel.yaml imported successfully + + _yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES', + 'true', 'True', 'TRUE', + 'on', 'On', 'ON') + _yaml_bools_false = ('n', 'N', 'no', 'No', 'NO', + 'false', 'False', 'FALSE', + 'off', 'Off', 'OFF') + _yaml_bools_plus_lists = [] + _yaml_bools_plus_lists.extend(_yaml_bools_true) + _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true]) + _yaml_bools_plus_lists.extend(_yaml_bools_false) + _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false]) + + def _class_as_dict_representer(dumper, data): + '''Creates a YAML representation of a App/Build instance''' + return dumper.represent_dict(data) + + def _field_to_yaml(typ, value): + if typ is TYPE_STRING: + if value in _yaml_bools_plus_lists: + return ruamel.yaml.scalarstring.SingleQuotedScalarString(str(value)) + return str(value) + elif typ is TYPE_INT: + return int(value) + elif typ is TYPE_MULTILINE: + if '\n' in value: + return ruamel.yaml.scalarstring.preserve_literal(str(value)) + else: + return str(value) + elif typ is TYPE_SCRIPT: + if type(value) == list: + if len(value) == 1: + return value[0] + else: + return value + else: + script_lines = value.split(' && ') + if len(script_lines) > 1: + return script_lines + else: + return value + else: + return value + + def _app_to_yaml(app): + cm = ruamel.yaml.comments.CommentedMap() + insert_newline = False + for field in yaml_app_field_order: + if field == '\n': + # next iteration will need to insert a newline + insert_newline = True + else: + if app.get(field) or field == 'Builds': + # .txt called it 'builds' internally, everywhere else its 'Builds' + if field == 'Builds': + if app.get('builds'): + cm.update({field: _builds_to_yaml(app)}) + elif field == 'CurrentVersionCode': + cm.update({field: _field_to_yaml(TYPE_INT, getattr(app, field))}) + else: + cm.update({field: _field_to_yaml(fieldtype(field), getattr(app, field))}) + + if insert_newline: + # we need to prepend a newline in front of this field + insert_newline = False + # inserting empty lines is not supported so we add a + # bogus comment and over-write its value + cm.yaml_set_comment_before_after_key(field, 'bogus') + cm.ca.items[field][1][-1].value = '\n' + return cm + + def _builds_to_yaml(app): + builds = ruamel.yaml.comments.CommentedSeq() + for build in app.builds: + b = ruamel.yaml.comments.CommentedMap() + for field in build_flags: + value = getattr(build, field) + if hasattr(build, field) and value: + if field == 'gradle' and value == ['off']: + value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')] + if field in ('maven', 'buildozer'): + if value == 'no': + continue + elif value == 'yes': + value = 'yes' + b.update({field: _field_to_yaml(flagtype(field), value)}) + builds.append(b) + + # insert extra empty lines between build entries + for i in range(1, len(builds)): + builds.yaml_set_comment_before_after_key(i, 'bogus') + builds.ca.items[i][1][-1].value = '\n' + + return builds + yaml_app = _app_to_yaml(app) - yamlmf = ruamel.yaml.YAML(typ='rt') - yamlmf.indent(mapping=2, sequence=4, offset=2) - yamlmf.dump(yaml_app, stream=mf) + ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2) + + +build_line_sep = re.compile(r'(? -# Copyright (C) 2022 FC Stegerman # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -17,160 +16,38 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import copy -import logging import os -import random -import tempfile -import time -import urllib - import requests -import urllib3 -from requests.adapters import HTTPAdapter, Retry - -from . import _, common - -logger = logging.getLogger(__name__) HEADERS = {'User-Agent': 'F-Droid'} -def download_file(url, local_filename=None, dldir='tmp', retries=3, backoff_factor=0.1): - """Try hard to download the file, including retrying on failures. - - This has two retry cycles, one inside of the requests session, the - other provided by this function. The requests retry logic applies - to failed DNS lookups, socket connections and connection timeouts, - never to requests where data has made it to the server. This - handles ChunkedEncodingError during transfer in its own retry - loop. This can result in more retries than are specified in the - retries parameter. - - """ - filename = urllib.parse.urlparse(url).path.split('/')[-1] +def download_file(url, local_filename=None, dldir='tmp'): + filename = url.split('/')[-1] if local_filename is None: local_filename = os.path.join(dldir, filename) - for i in range(retries + 1): - if retries: - max_retries = Retry(total=retries - i, backoff_factor=backoff_factor) - adapter = HTTPAdapter(max_retries=max_retries) - session = requests.Session() - session.mount('http://', adapter) - session.mount('https://', adapter) - else: - session = requests - # the stream=True parameter keeps memory usage low - r = session.get( - url, stream=True, allow_redirects=True, headers=HEADERS, timeout=300 - ) - r.raise_for_status() - try: - with open(local_filename, 'wb') as f: - for chunk in r.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - f.flush() - return local_filename - except requests.exceptions.ChunkedEncodingError as err: - if i == retries: - raise err - logger.warning('Download interrupted, retrying...') - time.sleep(backoff_factor * 2**i) - raise ValueError("retries must be >= 0") - - -def download_using_mirrors(mirrors, local_filename=None): - """Try to download the file from any working mirror. - - Download the file that all URLs in the mirrors list point to, - trying all the tricks, starting with the most private methods - first. The list of mirrors is converted into a list of mirror - configurations to try, in order that the should be attempted. - - This builds mirror_configs_to_try using all possible combos to - try. If a mirror is marked with worksWithoutSNI: True, then this - logic will try it twice: first without SNI, then again with SNI. - - """ - mirrors = common.parse_list_of_dicts(mirrors) - mirror_configs_to_try = [] - for mirror in mirrors: - mirror_configs_to_try.append(mirror) - if mirror.get('worksWithoutSNI'): - m = copy.deepcopy(mirror) - del m['worksWithoutSNI'] - mirror_configs_to_try.append(m) - - if not local_filename: - for mirror in mirrors: - filename = urllib.parse.urlparse(mirror['url']).path.split('/')[-1] - if filename: - break - if filename: - local_filename = os.path.join(common.get_cachedir(), filename) - else: - local_filename = tempfile.mkstemp(prefix='fdroid-') - - timeouts = (2, 10, 100) - last_exception = None - for timeout in timeouts: - for mirror in mirror_configs_to_try: - last_exception = None - urllib3.util.ssl_.HAS_SNI = not mirror.get('worksWithoutSNI') - try: - # the stream=True parameter keeps memory usage low - r = requests.get( - mirror['url'], - stream=True, - allow_redirects=False, - headers=HEADERS, - # add jitter to the timeout to be less predictable - timeout=timeout + random.randint(0, timeout), # nosec B311 - ) - if r.status_code != 200: - raise requests.exceptions.HTTPError(r.status_code, response=r) - with open(local_filename, 'wb') as f: - for chunk in r.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - f.flush() - return local_filename - except ( - ConnectionError, - requests.exceptions.ChunkedEncodingError, - requests.exceptions.ConnectionError, - requests.exceptions.ContentDecodingError, - requests.exceptions.HTTPError, - requests.exceptions.SSLError, - requests.exceptions.StreamConsumedError, - requests.exceptions.Timeout, - requests.exceptions.UnrewindableBodyError, - ) as e: - last_exception = e - logger.debug(_('Retrying failed download: %s') % str(e)) - # if it hasn't succeeded by now, then give up and raise last exception - if last_exception: - raise last_exception + # the stream=True parameter keeps memory usage low + r = requests.get(url, stream=True, allow_redirects=True, headers=HEADERS) + r.raise_for_status() + with open(local_filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + f.flush() + return local_filename def http_get(url, etag=None, timeout=600): - """Download the content from the given URL by making a GET request. + """ + Downloads the content from the given URL by making a GET request. If an ETag is given, it will do a HEAD request first, to see if the content changed. - Parameters - ---------- - url - The URL to download from. - etag - The last ETag to be used for the request (optional). - - Returns - ------- - A tuple consisting of: - - The raw content that was downloaded or None if it did not change - - The new eTag as returned by the HTTP request + :param url: The URL to download from. + :param etag: The last ETag to be used for the request (optional). + :return: A tuple consisting of: + - The raw content that was downloaded or None if it did not change + - The new eTag as returned by the HTTP request """ # TODO disable TLS Session IDs and TLS Session Tickets # (plain text cookie visible to anyone who can see the network traffic) diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 372390ea..63e7d131 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -"""Set up an app build for a nightly build repo.""" # # nightly.py - part of the FDroid server tools # Copyright (C) 2017 Hans-Christoph Steiner @@ -19,26 +18,23 @@ import base64 import datetime +import git import hashlib -import inspect import logging import os +import paramiko import platform import shutil -import ssl import subprocess import sys import tempfile -from argparse import ArgumentParser -from typing import Optional -from urllib.parse import urlparse - -import git -import paramiko import yaml +from urllib.parse import urlparse +from argparse import ArgumentParser + +from . import _ +from . import common -from . import _, common -from .exception import VCSException # hard coded defaults for Android ~/.android/debug.keystore files # https://developers.google.com/android/guides/client-auth @@ -51,121 +47,34 @@ DISTINGUISHED_NAME = 'CN=Android Debug,O=Android,C=US' NIGHTLY = '-nightly' -def _get_keystore_secret_var(keystore: str) -> str: - """Get keystore secret as base64. - - Parameters - ---------- - keystore - The path of the keystore. - - Returns - ------- - base64_secret - The keystore secret as base64 string. - """ - with open(keystore, 'rb') as fp: - return base64.standard_b64encode(fp.read()).decode('ascii') - - -def _ssh_key_from_debug_keystore(keystore: Optional[str] = None) -> str: - """Convert a debug keystore to an SSH private key. - - This leaves the original keystore file in place. - - Parameters - ---------- - keystore - The keystore to convert to a SSH private key. - - Returns - ------- - key_path - The SSH private key file path in the temporary directory. - """ - if keystore is None: - # set this here so it can be overridden in the tests - # TODO convert this to a class to get rid of this nonsense - keystore = KEYSTORE_FILE +def _ssh_key_from_debug_keystore(keystore=KEYSTORE_FILE): tmp_dir = tempfile.mkdtemp(prefix='.') privkey = os.path.join(tmp_dir, '.privkey') key_pem = os.path.join(tmp_dir, '.key.pem') p12 = os.path.join(tmp_dir, '.keystore.p12') _config = dict() common.fill_config_defaults(_config) - subprocess.check_call( - [ - _config['keytool'], - '-importkeystore', - '-srckeystore', - keystore, - '-srcalias', - KEY_ALIAS, - '-srcstorepass', - PASSWORD, - '-srckeypass', - PASSWORD, - '-destkeystore', - p12, - '-destalias', - KEY_ALIAS, - '-deststorepass', - PASSWORD, - '-destkeypass', - PASSWORD, - '-deststoretype', - 'PKCS12', - ], - env={'LC_ALL': 'C.UTF-8'}, - ) - subprocess.check_call( - [ - 'openssl', - 'pkcs12', - '-in', - p12, - '-out', - key_pem, - '-passin', - 'pass:' + PASSWORD, - '-passout', - 'pass:' + PASSWORD, - ], - env={'LC_ALL': 'C.UTF-8'}, - ) - - # OpenSSL 3.0 changed the default output format from PKCS#1 to - # PKCS#8, which paramiko does not support. - # https://www.openssl.org/docs/man3.0/man1/openssl-rsa.html#traditional - # https://github.com/paramiko/paramiko/issues/1015 - openssl_rsa_cmd = ['openssl', 'rsa'] - if ssl.OPENSSL_VERSION_INFO[0] >= 3: - openssl_rsa_cmd += ['-traditional'] - subprocess.check_call( - openssl_rsa_cmd - + [ - '-in', - key_pem, - '-out', - privkey, - '-passin', - 'pass:' + PASSWORD, - ], - env={'LC_ALL': 'C.UTF-8'}, - ) + subprocess.check_call([_config['keytool'], '-importkeystore', + '-srckeystore', keystore, '-srcalias', KEY_ALIAS, + '-srcstorepass', PASSWORD, '-srckeypass', PASSWORD, + '-destkeystore', p12, '-destalias', KEY_ALIAS, + '-deststorepass', PASSWORD, '-destkeypass', PASSWORD, + '-deststoretype', 'PKCS12'], + env={'LC_ALL': 'C.UTF-8'}) + subprocess.check_call(['openssl', 'pkcs12', '-in', p12, '-out', key_pem, + '-passin', 'pass:' + PASSWORD, '-passout', 'pass:' + PASSWORD], + env={'LC_ALL': 'C.UTF-8'}) + subprocess.check_call(['openssl', 'rsa', '-in', key_pem, '-out', privkey, + '-passin', 'pass:' + PASSWORD], + env={'LC_ALL': 'C.UTF-8'}) os.remove(key_pem) os.remove(p12) os.chmod(privkey, 0o600) # os.umask() should cover this, but just in case rsakey = paramiko.RSAKey.from_private_key_file(privkey) - fingerprint = ( - base64.b64encode(hashlib.sha256(rsakey.asbytes()).digest()) - .decode('ascii') - .rstrip('=') - ) - ssh_private_key_file = os.path.join( - tmp_dir, 'debug_keystore_' + fingerprint.replace('/', '_') + '_id_rsa' - ) + fingerprint = base64.b64encode(hashlib.sha256(rsakey.asbytes()).digest()).decode('ascii').rstrip('=') + ssh_private_key_file = os.path.join(tmp_dir, 'debug_keystore_' + + fingerprint.replace('/', '_') + '_id_rsa') shutil.move(privkey, ssh_private_key_file) pub = rsakey.get_name() + ' ' + rsakey.get_base64() + ' ' + ssh_private_key_file @@ -177,127 +86,26 @@ def _ssh_key_from_debug_keystore(keystore: Optional[str] = None) -> str: return ssh_private_key_file -def get_repo_base_url( - clone_url: str, repo_git_base: str, force_type: Optional[str] = None -) -> str: - """Generate the base URL for the F-Droid repository. - - Parameters - ---------- - clone_url - The URL to clone the Git repository. - repo_git_base - The project path of the Git repository at the Git forge. - force_type - The Git forge of the project. - - Returns - ------- - repo_base_url - The base URL of the F-Droid repository. - """ - if force_type is None: - force_type = urlparse(clone_url).netloc - if force_type == 'gitlab.com': - return clone_url + '/-/raw/master/fdroid' - if force_type == 'github.com': - return 'https://raw.githubusercontent.com/%s/master/fdroid' % repo_git_base - print(_('ERROR: unsupported git host "%s", patches welcome!') % force_type) - sys.exit(1) - - -def clone_git_repo(clone_url, git_mirror_path): - """Clone a git repo into the given path, failing if a password is required. - - If GitPython's safe mode is present, this will use that. Otherwise, - this includes a very limited version of the safe mode just to ensure - this won't hang on password prompts. - - https://github.com/gitpython-developers/GitPython/pull/2029 - - """ - logging.debug(_('cloning {url}').format(url=clone_url)) - try: - sig = inspect.signature(git.Repo.clone_from) - if 'safe' in sig.parameters: - git.Repo.clone_from(clone_url, git_mirror_path, safe=True) - else: - git.Repo.clone_from( - clone_url, - git_mirror_path, - env={ - 'GIT_ASKPASS': '/bin/true', - 'SSH_ASKPASS': '/bin/true', - 'GIT_USERNAME': 'u', - 'GIT_PASSWORD': 'p', - 'GIT_HTTP_USERNAME': 'u', - 'GIT_HTTP_PASSWORD': 'p', - 'GIT_SSH': '/bin/false', # for git < 2.3 - 'GIT_TERMINAL_PROMPT': '0', - }, - ) - except git.exc.GitCommandError as e: - logging.warning(_('WARNING: only public git repos are supported!')) - raise VCSException(f'git clone {clone_url} failed:', str(e)) from e - - def main(): - """Deploy to F-Droid repository or generate SSH private key from keystore. - The behaviour of this function is influenced by the configuration file as - well as command line parameters. - - Raises - ------ - :exc:`~fdroidserver.exception.VCSException` - If the nightly Git repository could not be cloned during an attempt to - deploy. - """ parser = ArgumentParser() common.setup_global_opts(parser) - parser.add_argument( - "--keystore", - default=KEYSTORE_FILE, - help=_("Specify which debug keystore file to use."), - ) - parser.add_argument( - "--show-secret-var", - action="store_true", - default=False, - help=_("Print the secret variable to the terminal for easy copy/paste"), - ) - parser.add_argument( - "--keep-private-keys", - action="store_true", - default=False, - help=_("Do not remove the private keys generated from the keystore"), - ) - parser.add_argument( - "--no-deploy", - action="store_true", - default=False, - help=_("Do not deploy the new files to the repo"), - ) - parser.add_argument( - "--file", - default='app/build/outputs/apk/*.apk', - help=_('The file to be included in the repo (path or glob)'), - ) - parser.add_argument( - "--no-checksum", - action="store_true", - default=False, - help=_("Don't use rsync checksums"), - ) - archive_older_unset = -1 - parser.add_argument( - "--archive-older", - type=int, - default=archive_older_unset, - help=_("Set maximum releases in repo before older ones are archived"), - ) + parser.add_argument("--keystore", default=KEYSTORE_FILE, + help=_("Specify which debug keystore file to use.")) + parser.add_argument("--show-secret-var", action="store_true", default=False, + help=_("Print the secret variable to the terminal for easy copy/paste")) + parser.add_argument("--keep-private-keys", action="store_true", default=False, + help=_("Do not remove the private keys generated from the keystore")) + parser.add_argument("--no-deploy", action="store_true", default=False, + help=_("Do not deploy the new files to the repo")) + parser.add_argument("--file", default='app/build/outputs/apk/*.apk', + help=_('The file to be included in the repo (path or glob)')) + parser.add_argument("--no-checksum", action="store_true", default=False, + help=_("Don't use rsync checksums")) + parser.add_argument("--archive-older", type=int, default=20, + help=_("Set maximum releases in repo before older ones are archived")) # TODO add --with-btlog - options = common.parse_args(parser) + options = parser.parse_args() # force a tighter umask since this writes private key material umask = os.umask(0o077) @@ -321,86 +129,56 @@ def main(): cibase = os.getcwd() os.makedirs(repodir, exist_ok=True) - # the 'master' branch is hardcoded in fdroidserver/deploy.py if 'CI_PROJECT_PATH' in os.environ and 'CI_PROJECT_URL' in os.environ: # we are in GitLab CI repo_git_base = os.getenv('CI_PROJECT_PATH') + NIGHTLY clone_url = os.getenv('CI_PROJECT_URL') + NIGHTLY - repo_base = get_repo_base_url( - clone_url, repo_git_base, force_type='gitlab.com' - ) + repo_base = clone_url + '/raw/master/fdroid' servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base - deploy_key_url = ( - f'{clone_url}/-/settings/repository#js-deploy-keys-settings' - ) + deploy_key_url = clone_url + '/settings/repository' git_user_name = os.getenv('GITLAB_USER_NAME') git_user_email = os.getenv('GITLAB_USER_EMAIL') elif 'TRAVIS_REPO_SLUG' in os.environ: # we are in Travis CI repo_git_base = os.getenv('TRAVIS_REPO_SLUG') + NIGHTLY clone_url = 'https://github.com/' + repo_git_base - repo_base = get_repo_base_url( - clone_url, repo_git_base, force_type='github.com' - ) + _branch = os.getenv('TRAVIS_BRANCH') + repo_base = 'https://raw.githubusercontent.com/' + repo_git_base + '/' + _branch + '/fdroid' servergitmirror = 'git@github.com:' + repo_git_base - deploy_key_url = ( - f'https://github.com/{repo_git_base}/settings/keys' - + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' - ) + deploy_key_url = ('https://github.com/' + repo_git_base + '/settings/keys' + + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys') git_user_name = repo_git_base git_user_email = os.getenv('USER') + '@' + platform.node() - elif ( - 'CIRCLE_REPOSITORY_URL' in os.environ - and 'CIRCLE_PROJECT_USERNAME' in os.environ - and 'CIRCLE_PROJECT_REPONAME' in os.environ - ): + elif 'CIRCLE_REPOSITORY_URL' in os.environ \ + and 'CIRCLE_PROJECT_USERNAME' in os.environ \ + and 'CIRCLE_PROJECT_REPONAME' in os.environ: # we are in Circle CI - repo_git_base = ( - os.getenv('CIRCLE_PROJECT_USERNAME') - + '/' - + os.getenv('CIRCLE_PROJECT_REPONAME') - + NIGHTLY - ) + repo_git_base = (os.getenv('CIRCLE_PROJECT_USERNAME') + + '/' + os.getenv('CIRCLE_PROJECT_REPONAME') + NIGHTLY) clone_url = os.getenv('CIRCLE_REPOSITORY_URL') + NIGHTLY - repo_base = get_repo_base_url( - clone_url, repo_git_base, force_type='github.com' - ) + repo_base = clone_url + '/raw/master/fdroid' servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base - deploy_key_url = ( - f'https://github.com/{repo_git_base}/settings/keys' - + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' - ) + deploy_key_url = ('https://github.com/' + repo_git_base + '/settings/keys' + + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys') git_user_name = os.getenv('CIRCLE_USERNAME') git_user_email = git_user_name + '@' + platform.node() - elif 'GITHUB_ACTIONS' in os.environ: - # we are in Github actions - repo_git_base = os.getenv('GITHUB_REPOSITORY') + NIGHTLY - clone_url = os.getenv('GITHUB_SERVER_URL') + '/' + repo_git_base - repo_base = get_repo_base_url( - clone_url, repo_git_base, force_type='github.com' - ) - servergitmirror = 'git@' + urlparse(clone_url).netloc + ':' + repo_git_base - deploy_key_url = ( - f'https://github.com/{repo_git_base}/settings/keys' - + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys' - ) - git_user_name = os.getenv('GITHUB_ACTOR') - git_user_email = git_user_name + '@' + platform.node() else: print(_('ERROR: unsupported CI type, patches welcome!')) sys.exit(1) repo_url = repo_base + '/repo' git_mirror_path = os.path.join(repo_basedir, 'git-mirror') - git_mirror_fdroiddir = os.path.join(git_mirror_path, 'fdroid') - git_mirror_repodir = os.path.join(git_mirror_fdroiddir, 'repo') - git_mirror_metadatadir = os.path.join(git_mirror_fdroiddir, 'metadata') + git_mirror_repodir = os.path.join(git_mirror_path, 'fdroid', 'repo') + git_mirror_metadatadir = os.path.join(git_mirror_path, 'fdroid', 'metadata') + git_mirror_statsdir = os.path.join(git_mirror_path, 'fdroid', 'stats') if not os.path.isdir(git_mirror_repodir): - clone_git_repo(clone_url, git_mirror_path) + logging.debug(_('cloning {url}').format(url=clone_url)) + try: + git.Repo.clone_from(clone_url, git_mirror_path) + except Exception: + pass if not os.path.isdir(git_mirror_repodir): os.makedirs(git_mirror_repodir, mode=0o755) - if os.path.exists('LICENSE'): - shutil.copy2('LICENSE', git_mirror_path) mirror_git_repo = git.Repo.init(git_mirror_path) writer = mirror_git_repo.config_writer() @@ -414,31 +192,34 @@ def main(): readme = ''' # {repo_git_base} -This is an app repository for nightly versions. -You can use it with the [F-Droid](https://f-droid.org/) Android app. +[![{repo_url}](icon.png)]({repo_url}) -[![{repo_url}]({repo_url}/icons/icon.png)](https://fdroid.link/#{repo_url}) - -Last updated: {date}'''.format( - repo_git_base=repo_git_base, - repo_url=repo_url, - date=datetime.datetime.now(datetime.timezone.utc).strftime( - '%Y-%m-%d %H:%M:%S UTC' - ), - ) +Last updated: {date}'''.format(repo_git_base=repo_git_base, + repo_url=repo_url, + date=datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')) with open(readme_path, 'w') as fp: fp.write(readme) mirror_git_repo.git.add(all=True) mirror_git_repo.index.commit("update README") + icon_path = os.path.join(git_mirror_path, 'icon.png') + try: + import qrcode + qrcode.make(repo_url).save(icon_path) + except Exception: + exampleicon = os.path.join(common.get_examples_dir(), 'fdroid-icon.png') + shutil.copy(exampleicon, icon_path) mirror_git_repo.git.add(all=True) mirror_git_repo.index.commit("update repo/website icon") + shutil.copy(icon_path, repo_basedir) os.chdir(repo_basedir) if os.path.isdir(git_mirror_repodir): - common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/') + common.local_rsync(options, git_mirror_repodir + '/', 'repo/') if os.path.isdir(git_mirror_metadatadir): - common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/') + common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/') + if os.path.isdir(git_mirror_statsdir): + common.local_rsync(options, git_mirror_statsdir + '/', 'stats/') ssh_private_key_file = _ssh_key_from_debug_keystore() # this is needed for GitPython to find the SSH key @@ -449,89 +230,57 @@ Last updated: {date}'''.format( with open(ssh_config, 'a') as fp: fp.write('\n\nHost *\n\tIdentityFile %s\n' % ssh_private_key_file) - if options.archive_older == archive_older_unset: - fdroid_size = common.get_dir_size(git_mirror_fdroiddir) - max_size = common.GITLAB_COM_PAGES_MAX_SIZE - if fdroid_size < max_size: - options.archive_older = 20 - else: - options.archive_older = 3 - print( - 'WARNING: repo is %s over the GitLab Pages limit (%s)' - % (fdroid_size - max_size, max_size) - ) - print('Setting --archive-older to 3') - - config = { - 'identity_file': ssh_private_key_file, - 'repo_name': repo_git_base, - 'repo_url': repo_url, - 'repo_description': 'Nightly builds from %s' % git_user_email, - 'archive_name': repo_git_base + ' archive', - 'archive_url': repo_base + '/archive', - 'archive_description': 'Old nightly builds that have been archived.', - 'archive_older': options.archive_older, - 'servergitmirrors': [{"url": servergitmirror}], - 'keystore': KEYSTORE_FILE, - 'repo_keyalias': KEY_ALIAS, - 'keystorepass': PASSWORD, - 'keypass': PASSWORD, - 'keydname': DISTINGUISHED_NAME, - 'make_current_version_link': False, - } - with open(common.CONFIG_FILE, 'w', encoding='utf-8') as fp: - yaml.dump(config, fp, default_flow_style=False) - os.chmod(common.CONFIG_FILE, 0o600) - config = common.read_config() + config = '' + config += "identity_file = '%s'\n" % ssh_private_key_file + config += "repo_name = '%s'\n" % repo_git_base + config += "repo_url = '%s'\n" % repo_url + config += "repo_icon = 'icon.png'\n" + config += "repo_description = 'Nightly builds from %s'\n" % git_user_email + config += "archive_name = '%s'\n" % (repo_git_base + ' archive') + config += "archive_url = '%s'\n" % (repo_base + '/archive') + config += "archive_icon = 'icon.png'\n" + config += "archive_description = 'Old nightly builds that have been archived.'\n" + config += "archive_older = %i\n" % options.archive_older + config += "servergitmirrors = '%s'\n" % servergitmirror + config += "keystore = '%s'\n" % KEYSTORE_FILE + config += "repo_keyalias = '%s'\n" % KEY_ALIAS + config += "keystorepass = '%s'\n" % PASSWORD + config += "keypass = '%s'\n" % PASSWORD + config += "keydname = '%s'\n" % DISTINGUISHED_NAME + config += "make_current_version_link = False\n" + config += "update_stats = True\n" + with open('config.py', 'w') as fp: + fp.write(config) + os.chmod('config.py', 0o600) + config = common.read_config(options) common.assert_config_keystore(config) - logging.debug( - _( - 'Run over {cibase} to find -debug.apk. and skip repo_basedir {repo_basedir}' - ).format(cibase=cibase, repo_basedir=repo_basedir) - ) - for root, dirs, files in os.walk(cibase): - for d in ('.git', '.gradle'): - if d in dirs: + for d in dirs: + if d == '.git' or d == '.gradle' or (d == 'fdroid' and root == cibase): dirs.remove(d) - if root == cibase and 'fdroid' in dirs: - dirs.remove('fdroid') - for f in files: if f.endswith('-debug.apk'): apkfilename = os.path.join(root, f) - logging.debug( - _('Stripping mystery signature from {apkfilename}').format( - apkfilename=apkfilename - ) - ) + logging.debug(_('Stripping mystery signature from {apkfilename}') + .format(apkfilename=apkfilename)) destapk = os.path.join(repodir, os.path.basename(f)) os.chmod(apkfilename, 0o644) - logging.debug( - _( - 'Resigning {apkfilename} with provided debug.keystore' - ).format(apkfilename=os.path.basename(apkfilename)) - ) + logging.debug(_('Resigning {apkfilename} with provided debug.keystore') + .format(apkfilename=os.path.basename(apkfilename))) + common.apk_strip_v1_signatures(apkfilename, strip_manifest=True) common.sign_apk(apkfilename, destapk, KEY_ALIAS) if options.verbose: logging.debug(_('attempting bare SSH connection to test deploy key:')) try: - subprocess.check_call( - [ - 'ssh', - '-Tvi', - ssh_private_key_file, - '-oIdentitiesOnly=yes', - '-oStrictHostKeyChecking=no', - servergitmirror.split(':')[0], - ] - ) + subprocess.check_call(['ssh', '-Tvi', ssh_private_key_file, + '-oIdentitiesOnly=yes', '-oStrictHostKeyChecking=no', + servergitmirror.split(':')[0]]) except subprocess.CalledProcessError: pass - app_url = clone_url[: -len(NIGHTLY)] + app_url = clone_url[:-len(NIGHTLY)] template = dict() template['AuthorName'] = clone_url.split('/')[4] template['AuthorWebSite'] = '/'.join(clone_url.split('/')[:4]) @@ -543,13 +292,10 @@ Last updated: {date}'''.format( with open('template.yml', 'w') as fp: yaml.dump(template, fp) - subprocess.check_call( - ['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'], - cwd=repo_basedir, - ) - common.local_rsync( - options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/' - ) + subprocess.check_call(['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'], + cwd=repo_basedir) + common.local_rsync(options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/') + common.local_rsync(options, repo_basedir + '/stats/', git_mirror_statsdir + '/') mirror_git_repo.git.add(all=True) mirror_git_repo.index.commit("update app metadata") @@ -558,11 +304,8 @@ Last updated: {date}'''.format( cmd = ['fdroid', 'deploy', '--verbose', '--no-keep-git-mirror-archive'] subprocess.check_call(cmd, cwd=repo_basedir) except subprocess.CalledProcessError: - logging.error( - _('cannot publish update, did you set the deploy key?') - + '\n' - + deploy_key_url - ) + logging.error(_('cannot publish update, did you set the deploy key?') + + '\n' + deploy_key_url) sys.exit(1) if not options.keep_private_keys: @@ -576,33 +319,25 @@ Last updated: {date}'''.format( if not os.path.exists(androiddir): os.mkdir(androiddir) logging.info(_('created {path}').format(path=androiddir)) - logging.error( - _('{path} does not exist! Create it by running:').format( - path=options.keystore - ) - + '\n keytool -genkey -v -keystore ' - + options.keystore - + ' -storepass android \\' - + '\n -alias androiddebugkey -keypass android -keyalg RSA -keysize 2048 -validity 10000 \\' - + '\n -dname "CN=Android Debug,O=Android,C=US"' - ) + logging.error(_('{path} does not exist! Create it by running:').format(path=options.keystore) + + '\n keytool -genkey -v -keystore ' + options.keystore + ' -storepass android \\' + + '\n -alias androiddebugkey -keypass android -keyalg RSA -keysize 2048 -validity 10000 \\' + + '\n -dname "CN=Android Debug,O=Android,C=US"') sys.exit(1) ssh_dir = os.path.join(os.getenv('HOME'), '.ssh') + os.makedirs(os.path.dirname(ssh_dir), exist_ok=True) privkey = _ssh_key_from_debug_keystore(options.keystore) - if os.path.exists(ssh_dir): - ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey)) - shutil.move(privkey, ssh_private_key_file) - shutil.move(privkey + '.pub', ssh_private_key_file + '.pub') + ssh_private_key_file = os.path.join(ssh_dir, os.path.basename(privkey)) + shutil.move(privkey, ssh_private_key_file) + shutil.move(privkey + '.pub', ssh_private_key_file + '.pub') if shutil.rmtree.avoids_symlink_attacks: shutil.rmtree(os.path.dirname(privkey)) if options.show_secret_var: - debug_keystore = _get_keystore_secret_var(options.keystore) - print( - _('\n{path} encoded for the DEBUG_KEYSTORE secret variable:').format( - path=options.keystore - ) - ) + with open(options.keystore, 'rb') as fp: + debug_keystore = base64.standard_b64encode(fp.read()).decode('ascii') + print(_('\n{path} encoded for the DEBUG_KEYSTORE secret variable:') + .format(path=options.keystore)) print(debug_keystore) os.umask(umask) diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 42945166..41d70300 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -3,7 +3,6 @@ # publish.py - part of the FDroid server tools # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com # Copyright (C) 2013-2014 Daniel Martí -# Copyright (C) 2021 Felix C. Stegerman # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -18,40 +17,34 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Sign APKs using keys or via reproducible builds signature copying. - -This command takes unsigned APKs and signs them. It looks for -unsigned APKs in the unsigned/ directory and puts successfully signed -APKs into the repo/ directory. The default is to run in a kind of -batch mode, where it will only quit on certain kinds of errors. It -mostly reports success by moving an APK from unsigned/ to repo/ - -""" - -import glob -import hashlib -import json -import logging +import sys import os import re import shutil -import sys -import time -import zipfile +import glob +import hashlib from argparse import ArgumentParser from collections import OrderedDict +import logging from gettext import ngettext +import json +import time +import zipfile -from . import _, common, metadata +from . import _ +from . import common +from . import metadata from .common import FDroidPopen from .exception import BuildException, FDroidException config = None +options = None start_timestamp = time.gmtime() def publish_source_tarball(apkfilename, unsigned_dir, output_dir): """Move the source tarball into the output directory...""" + tarfilename = apkfilename[:-4] + '_src.tar.gz' tarfile = os.path.join(unsigned_dir, tarfilename) if os.path.exists(tarfile): @@ -62,9 +55,7 @@ def publish_source_tarball(apkfilename, unsigned_dir, output_dir): def key_alias(appid): - """No summary. - - Get the alias which F-Droid uses to indentify the singing key + """Get the alias which F-Droid uses to indentify the singing key for this App in F-Droids keystore. """ if config and 'keyaliases' in config and appid in config['keyaliases']: @@ -82,27 +73,24 @@ def key_alias(appid): def read_fingerprints_from_keystore(): - """Obtain a dictionary containing all singning-key fingerprints which are managed by F-Droid, grouped by appid.""" - env_vars = {'LC_ALL': 'C.UTF-8', 'FDROID_KEY_STORE_PASS': config['keystorepass']} - cmd = [ - config['keytool'], - '-list', - '-v', - '-keystore', - config['keystore'], - '-storepass:env', - 'FDROID_KEY_STORE_PASS', - ] + """Obtain a dictionary containing all singning-key fingerprints which + are managed by F-Droid, grouped by appid. + """ + env_vars = {'LC_ALL': 'C.UTF-8', + 'FDROID_KEY_STORE_PASS': config['keystorepass']} + cmd = [config['keytool'], '-list', + '-v', '-keystore', config['keystore'], + '-storepass:env', 'FDROID_KEY_STORE_PASS'] if config['keystore'] == 'NONE': cmd += config['smartcardoptions'] p = FDroidPopen(cmd, envs=env_vars, output=False) if p.returncode != 0: raise FDroidException('could not read keystore {}'.format(config['keystore'])) - realias = re.compile('Alias name: (?P.+)' + os.linesep) - resha256 = re.compile(r'\s+SHA256: (?P[:0-9A-F]{95})' + os.linesep) + realias = re.compile('Alias name: (?P.+)\n') + resha256 = re.compile(r'\s+SHA256: (?P[:0-9A-F]{95})\n') fps = {} - for block in p.output.split(('*' * 43) + os.linesep + '*' * 43): + for block in p.output.split(('*' * 43) + '\n' + '*' * 43): s_alias = realias.search(block) s_sha256 = resha256.search(block) if s_alias and s_sha256: @@ -112,9 +100,8 @@ def read_fingerprints_from_keystore(): def sign_sig_key_fingerprint_list(jar_file): - """Sign the list of app-signing key fingerprints. - - This is used primaryily by fdroid update to determine which APKs + """sign the list of app-signing key fingerprints which is + used primaryily by fdroid update to determine which APKs where built and signed by F-Droid and which ones were manually added by users. """ @@ -128,22 +115,19 @@ def sign_sig_key_fingerprint_list(jar_file): cmd += config['smartcardoptions'] else: # smardcards never use -keypass cmd += '-keypass:env', 'FDROID_KEY_PASS' - env_vars = { - 'FDROID_KEY_STORE_PASS': config['keystorepass'], - 'FDROID_KEY_PASS': config.get('keypass', ""), - } + env_vars = {'FDROID_KEY_STORE_PASS': config['keystorepass'], + 'FDROID_KEY_PASS': config.get('keypass', "")} p = common.FDroidPopen(cmd, envs=env_vars) if p.returncode != 0: raise FDroidException("Failed to sign '{}'!".format(jar_file)) -def store_publish_signer_fingerprints(appids, indent=None): +def store_stats_fdroid_signing_key_fingerprints(appids, indent=None): """Store list of all signing-key fingerprints for given appids to HD. - This list will later on be needed by fdroid update. """ - if not os.path.exists('repo'): - os.makedirs('repo') + if not os.path.exists('stats'): + os.makedirs('stats') data = OrderedDict() fps = read_fingerprints_from_keystore() for appid in sorted(appids): @@ -151,22 +135,17 @@ def store_publish_signer_fingerprints(appids, indent=None): if alias in fps: data[appid] = {'signer': fps[key_alias(appid)]} - jar_file = os.path.join('repo', 'signer-index.jar') - output = json.dumps(data, indent=indent) + jar_file = os.path.join('stats', 'publishsigkeys.jar') with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar: - jar.writestr('signer-index.json', output) - with open(os.path.join('repo', 'signer-index.json'), 'w') as fp: - fp.write(output) + jar.writestr('publishsigkeys.json', json.dumps(data, indent=indent)) sign_sig_key_fingerprint_list(jar_file) def status_update_json(generatedKeys, signedApks): - """Output a JSON file with metadata about this run.""" + """Output a JSON file with metadata about this run""" + logging.debug(_('Outputting JSON')) output = common.setup_status_output(start_timestamp) - output['apksigner'] = shutil.which(config.get('apksigner', '')) - output['jarsigner'] = shutil.which(config.get('jarsigner', '')) - output['keytool'] = shutil.which(config.get('keytool', '')) if generatedKeys: output['generatedKeys'] = generatedKeys if signedApks: @@ -175,8 +154,8 @@ def status_update_json(generatedKeys, signedApks): def check_for_key_collisions(allapps): - """Make sure there's no collision in keyaliases from apps. - + """ + Make sure there's no collision in keyaliases from apps. It was suggested at https://dev.guardianproject.info/projects/bazaar/wiki/FDroid_Audit that a package could be crafted, such that it would use the same signing @@ -185,16 +164,9 @@ def check_for_key_collisions(allapps): the colliding ID would be something that would be a) a valid package ID, and b) a sane-looking ID that would make its way into the repo. Nonetheless, to be sure, before publishing we check that there are no - collisions, and refuse to do any publishing if that's the case. - - Parameters - ---------- - allapps - a dict of all apps to process - - Returns - ------- - a list of all aliases corresponding to allapps + collisions, and refuse to do any publishing if that's the case... + :param allapps a dict of all apps to process + :return: a list of all aliases corresponding to allapps """ allaliases = [] for appid in allapps: @@ -209,53 +181,30 @@ def check_for_key_collisions(allapps): def create_key_if_not_existing(keyalias): - """Ensure a signing key with the given keyalias exists. - - Returns - ------- - boolean - True if a new key was created, False otherwise + """ + Ensures a signing key with the given keyalias exists + :return: boolean, True if a new key was created, false otherwise """ # See if we already have a key for this application, and # if not generate one... - env_vars = { - 'LC_ALL': 'C.UTF-8', - 'FDROID_KEY_STORE_PASS': config['keystorepass'], - 'FDROID_KEY_PASS': config.get('keypass', ""), - } - cmd = [ - config['keytool'], - '-list', - '-alias', - keyalias, - '-keystore', - config['keystore'], - '-storepass:env', - 'FDROID_KEY_STORE_PASS', - ] + env_vars = {'LC_ALL': 'C.UTF-8', + 'FDROID_KEY_STORE_PASS': config['keystorepass'], + 'FDROID_KEY_PASS': config.get('keypass', "")} + cmd = [config['keytool'], '-list', + '-alias', keyalias, '-keystore', config['keystore'], + '-storepass:env', 'FDROID_KEY_STORE_PASS'] if config['keystore'] == 'NONE': cmd += config['smartcardoptions'] p = FDroidPopen(cmd, envs=env_vars) if p.returncode != 0: logging.info("Key does not exist - generating...") - cmd = [ - config['keytool'], - '-genkey', - '-keystore', - config['keystore'], - '-alias', - keyalias, - '-keyalg', - 'RSA', - '-keysize', - '2048', - '-validity', - '10000', - '-storepass:env', - 'FDROID_KEY_STORE_PASS', - '-dname', - config['keydname'], - ] + cmd = [config['keytool'], '-genkey', + '-keystore', config['keystore'], + '-alias', keyalias, + '-keyalg', 'RSA', '-keysize', '2048', + '-validity', '10000', + '-storepass:env', 'FDROID_KEY_STORE_PASS', + '-dname', config['keydname']] if config['keystore'] == 'NONE': cmd += config['smartcardoptions'] else: @@ -269,35 +218,22 @@ def create_key_if_not_existing(keyalias): def main(): - global config + global config, options # Parse command line... - parser = ArgumentParser( - usage="%(prog)s [options] " "[APPID[:VERCODE] [APPID[:VERCODE] ...]]" - ) + parser = ArgumentParser(usage="%(prog)s [options] " + "[APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) - parser.add_argument( - "-e", - "--error-on-failed", - action="store_true", - default=False, - help=_("When signing or verifying fails, exit with an error code."), - ) - parser.add_argument( - "appid", - nargs='*', - help=_("application ID with optional versionCode in the form APPID[:VERCODE]"), - ) + parser.add_argument("appid", nargs='*', + help=_("application ID with optional versionCode in the form APPID[:VERCODE]")) metadata.add_metadata_arguments(parser) - options = common.parse_args(parser) + options = parser.parse_args() metadata.warnings_action = options.W - config = common.read_config() + config = common.read_config(options) if not ('jarsigner' in config and 'keytool' in config): - logging.critical( - _('Java JDK not found! Install in standard location or set java_paths!') - ) + logging.critical(_('Java JDK not found! Install in standard location or set java_paths!')) sys.exit(1) common.assert_config_keystore(config) @@ -329,22 +265,16 @@ def main(): allapps = metadata.read_metadata() vercodes = common.read_pkg_args(options.appid, True) - common.get_metadata_files(vercodes) # only check appids signed_apks = dict() generated_keys = dict() allaliases = check_for_key_collisions(allapps) - logging.info( - ngettext( - '{0} app, {1} key aliases', '{0} apps, {1} key aliases', len(allapps) - ).format(len(allapps), len(allaliases)) - ) + logging.info(ngettext('{0} app, {1} key aliases', + '{0} apps, {1} key aliases', len(allapps)).format(len(allapps), len(allaliases))) - failed = 0 # Process any APKs or ZIPs that are waiting to be signed... - for apkfile in sorted( - glob.glob(os.path.join(unsigned_dir, '*.apk')) - + glob.glob(os.path.join(unsigned_dir, '*.zip')) - ): + for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk')) + + glob.glob(os.path.join(unsigned_dir, '*.zip'))): + appid, vercode = common.publishednameinfo(apkfile) apkfilename = os.path.basename(apkfile) if vercodes and appid not in vercodes: @@ -357,17 +287,13 @@ def main(): # There ought to be valid metadata for this app, otherwise why are we # trying to publish it? if appid not in allapps: - logging.error( - "Unexpected {0} found in unsigned directory".format(apkfilename) - ) + logging.error("Unexpected {0} found in unsigned directory" + .format(apkfilename)) sys.exit(1) app = allapps[appid] - build = None - for b in app.get("Builds", ()): - if b.get("versionCode") == vercode: - build = b - if app.Binaries or (build and build.binary): + if app.Binaries: + # It's an app where we build from source, and verify the apk # contents against a developer's binary, and then publish their # version if everything checks out. @@ -378,22 +304,14 @@ def main(): srcapk = srcapk.replace(unsigned_dir, binaries_dir) if not os.path.isfile(srcapk): - logging.error( - "...reference binary missing - publish skipped: '{refpath}'".format( - refpath=srcapk - ) - ) - failed += 1 + logging.error("...reference binary missing - publish skipped: " + "'{refpath}'".format(refpath=srcapk)) else: # Compare our unsigned one with the downloaded one... compare_result = common.verify_apks(srcapk, apkfile, tmp_dir) if compare_result: - logging.error( - "...verification failed - publish skipped : {result}".format( - result=compare_result - ) - ) - failed += 1 + logging.error("...verification failed - publish skipped : " + "{result}".format(result=compare_result)) else: # Success! So move the downloaded file to the repo, and remove # our built version. @@ -404,6 +322,7 @@ def main(): logging.info('Published ' + apkfilename) elif apkfile.endswith('.zip'): + # OTA ZIPs built by fdroid do not need to be signed by jarsigner, # just to be moved into place in the repo shutil.move(apkfile, os.path.join(output_dir, apkfilename)) @@ -411,6 +330,7 @@ def main(): logging.info('Published ' + apkfilename) else: + # It's a 'normal' app, i.e. we sign and publish it... skipsigning = False @@ -421,23 +341,22 @@ def main(): # metadata. This means we're going to prepare both a locally # signed APK and a version signed with the developers key. - signature_file, _ignored, manifest, v2_files = signingfiles + signaturefile, signedfile, manifest = signingfiles - with open(signature_file, 'rb') as f: - devfp = common.signer_fingerprint_short( - common.get_certificate(f.read()) - ) + with open(signaturefile, 'rb') as f: + devfp = common.signer_fingerprint_short(common.get_certificate(f.read())) devsigned = '{}_{}_{}.apk'.format(appid, vercode, devfp) devsignedtmp = os.path.join(tmp_dir, devsigned) + shutil.copy(apkfile, devsignedtmp) - common.apk_implant_signatures(apkfile, devsignedtmp, manifest=manifest) + common.apk_implant_signatures(devsignedtmp, signaturefile, + signedfile, manifest) if common.verify_apk_signature(devsignedtmp): shutil.move(devsignedtmp, os.path.join(output_dir, devsigned)) else: os.remove(devsignedtmp) logging.error('...verification failed - skipping: %s', devsigned) skipsigning = True - failed += 1 # Now we sign with the F-Droid key. if not skipsigning: @@ -449,30 +368,25 @@ def main(): signed_apk_path = os.path.join(output_dir, apkfilename) if os.path.exists(signed_apk_path): - raise BuildException( - _( - "Refusing to sign '{path}', file exists in both {dir1} and {dir2} folder." - ).format(path=apkfilename, dir1=unsigned_dir, dir2=output_dir) - ) + raise BuildException("Refusing to sign '{0}' file exists in both " + "{1} and {2} folder.".format(apkfilename, + unsigned_dir, + output_dir)) - # Sign the application... + # Sign and zipalign the application... common.sign_apk(apkfile, signed_apk_path, keyalias) if appid not in signed_apks: signed_apks[appid] = [] - signed_apks[appid].append({"keyalias": keyalias, "filename": apkfile}) + signed_apks[appid].append({"keyalias": keyalias, + "filename": apkfile}) publish_source_tarball(apkfilename, unsigned_dir, output_dir) logging.info('Published ' + apkfilename) - store_publish_signer_fingerprints(allapps.keys()) + store_stats_fdroid_signing_key_fingerprints(allapps.keys()) status_update_json(generated_keys, signed_apks) logging.info('published list signing-key fingerprints') - if failed: - logging.error(_('%d APKs failed to be signed or verified!') % failed) - if options.error_on_failed: - sys.exit(failed) - if __name__ == "__main__": main() diff --git a/fdroidserver/readmeta.py b/fdroidserver/readmeta.py index b3ef7c3b..b45e9559 100644 --- a/fdroidserver/readmeta.py +++ b/fdroidserver/readmeta.py @@ -17,17 +17,20 @@ # along with this program. If not, see . from argparse import ArgumentParser +from . import common +from . import metadata -from . import common, metadata +options = None def main(): + parser = ArgumentParser() common.setup_global_opts(parser) metadata.add_metadata_arguments(parser) options = parser.parse_args() metadata.warnings_action = options.W - common.read_config() + common.read_config(None) metadata.read_metadata() diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py index 4bbe810d..94439b97 100644 --- a/fdroidserver/rewritemeta.py +++ b/fdroidserver/rewritemeta.py @@ -17,73 +17,56 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import io -import logging -import shutil -import tempfile from argparse import ArgumentParser -from pathlib import Path +import os +import logging +import io +import tempfile +import shutil -from . import _, common, metadata +from . import _ +from . import common +from . import metadata config = None +options = None def proper_format(app): s = io.StringIO() # TODO: currently reading entire file again, should reuse first # read in metadata.py - cur_content = Path(app.metadatapath).read_text(encoding='utf-8') - if Path(app.metadatapath).suffix == '.yml': + with open(app.metadatapath, 'r') as f: + cur_content = f.read() + if app.metadatapath.endswith('.yml'): metadata.write_yaml(s, app) content = s.getvalue() s.close() return content == cur_content -def remove_blank_flags_from_builds(builds): - """Remove unset entries from Builds so they are not written out.""" - if not builds: - return list() - newbuilds = list() - for build in builds: - new = dict() - for k in metadata.build_flags: - v = build.get(k) - # 0 is valid value, it should not be stripped - if v is None or v is False or v == '' or v == dict() or v == list(): - continue - new[k] = v - newbuilds.append(new) - return newbuilds - - def main(): - global config + + global config, options parser = ArgumentParser() common.setup_global_opts(parser) - parser.add_argument( - "-l", - "--list", - action="store_true", - default=False, - help=_("List files that would be reformatted (dry run)"), - ) - parser.add_argument( - "appid", nargs='*', help=_("application ID of file to operate on") - ) + parser.add_argument("-l", "--list", action="store_true", default=False, + help=_("List files that would be reformatted")) + parser.add_argument("appid", nargs='*', help=_("application ID of file to operate on")) metadata.add_metadata_arguments(parser) - options = common.parse_args(parser) + options = parser.parse_args() metadata.warnings_action = options.W - config = common.read_config() + config = common.read_config(options) - apps = common.read_app_args(options.appid) + # Get all apps... + allapps = metadata.read_metadata(options.appid) + apps = common.read_app_args(options.appid, allapps, False) for appid, app in apps.items(): - path = Path(app.metadatapath) - if path.suffix == '.yml': + path = app.metadatapath + if path.endswith('.yml'): logging.info(_("Rewriting '{appid}'").format(appid=appid)) else: logging.warning(_('Cannot rewrite "{path}"').format(path=path)) @@ -94,15 +77,21 @@ def main(): print(path) continue - # TODO these should be moved to metadata.write_yaml() - builds = remove_blank_flags_from_builds(app.get('Builds')) - if builds: - app['Builds'] = builds + newbuilds = [] + for build in app.builds: + new = metadata.Build() + for k in metadata.build_flags: + v = build[k] + if v is None or v is False or v == [] or v == '': + continue + new[k] = v + newbuilds.append(new) + app.builds = newbuilds - # rewrite to temporary file before overwriting existing + # rewrite to temporary file before overwriting existsing # file in case there's a bug in write_metadata with tempfile.TemporaryDirectory() as tmpdir: - tmp_path = Path(tmpdir) / path.name + tmp_path = os.path.join(tmpdir, os.path.basename(path)) metadata.write_metadata(tmp_path, app) shutil.move(tmp_path, path) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index f28e3803..8230831e 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -16,743 +16,147 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import itertools +import imghdr import json -import logging import os import re import sys import traceback -import urllib.parse -import urllib.request -import zipfile from argparse import ArgumentParser -from dataclasses import dataclass, field, fields -from datetime import datetime, timedelta, timezone -from enum import IntEnum -from pathlib import Path -from tempfile import TemporaryDirectory -from typing import Union +import logging +import itertools -try: - import magic -except ImportError: - import puremagic as magic +from . import _ +from . import common +from . import metadata +from .exception import BuildException, VCSException -if sys.version_info >= (3, 11): - import tomllib -else: - import tomli as tomllib +config = None +options = None -from . import _, common, metadata, scanner -from .exception import BuildException, ConfigurationException, VCSException +DEFAULT_JSON_PER_BUILD = {'errors': [], 'warnings': [], 'infos': []} +json_per_build = DEFAULT_JSON_PER_BUILD +MAVEN_URL_REGEX = re.compile(r"""\smaven\s*{.*?(?:setUrl|url)\s*=?\s*(?:uri)?\(?\s*["']?([^\s"']+)["']?[^}]*}""", + re.DOTALL) -@dataclass -class MessageStore: - infos: list = field(default_factory=list) - warnings: list = field(default_factory=list) - errors: list = field(default_factory=list) - - -MAVEN_URL_REGEX = re.compile( - r"""\smaven\s*(?:{.*?(?:setUrl|url)|\(\s*(?:url)?)\s*=?\s*(?:uri|URI|Uri\.create)?\(?\s*["']?([^\s"']+)["']?[^})]*[)}]""", - re.DOTALL, -) - -DEPFILE = { - "Cargo.toml": ["Cargo.lock"], - "pubspec.yaml": ["pubspec.lock"], - "package.json": ["package-lock.json", "yarn.lock", "pnpm-lock.yaml", "bun.lock"], +CODE_SIGNATURES = { + # The `apkanalyzer dex packages` output looks like this: + # M d 1 1 93 + # The first column has P/C/M/F for package, class, method or field + # The second column has x/k/r/d for removed, kept, referenced and defined. + # We already filter for defined only in the apkanalyzer call. 'r' will be + # for things referenced but not distributed in the apk. + exp: re.compile(r'.[\s]*d[\s]*[0-9]*[\s]*[0-9*][\s]*[0-9]*[\s]*' + exp, re.IGNORECASE) for exp in [ + r'(com\.google\.firebase[^\s]*)', + r'(com\.google\.android\.gms[^\s]*)', + r'(com\.google\.tagmanager[^\s]*)', + r'(com\.google\.analytics[^\s]*)', + r'(com\.android\.billing[^\s]*)', + ] } -SCANNER_CACHE_VERSION = 1 - -DEFAULT_CATALOG_PREFIX_REGEX = re.compile( - r'''defaultLibrariesExtensionName\s*=\s*['"](\w+)['"]''' -) -GRADLE_CATALOG_FILE_REGEX = re.compile( - r'''(?:create\()?['"]?(\w+)['"]?\)?\s*\{[^}]*from\(files\(['"]([^"]+)['"]\)\)''' -) -VERSION_CATALOG_REGEX = re.compile(r'versionCatalogs\s*\{') - -APK_SIGNING_BLOCK_IDS = { - # https://source.android.com/docs/security/features/apksigning/v2#apk-signing-block - # 0x7109871a: 'APK signature scheme v2', - # https://source.android.com/docs/security/features/apksigning/v3#apk-signing-block - # 0xf05368c0: 'APK signature scheme v3', - # See "Security metadata in early 2018" - # https://android-developers.googleblog.com/2017/12/improving-app-security-and-performance.html - 0x2146444E: 'Google Play Signature aka "Frosting"', - # 0x42726577: 'Verity padding', - # 0x6DFF800D: 'Source stamp V2 X509 cert', - # JSON with some metadata, used by Chinese company Meituan - 0x71777777: 'Meituan payload', - # Dependencies metadata generated by Gradle and encrypted by Google Play. - # '...The data is compressed, encrypted by a Google Play signing key...' - # https://developer.android.com/studio/releases/gradle-plugin#dependency-metadata - 0x504B4453: 'Dependency metadata', +# Common known non-free blobs (always lower case): +NON_FREE_GRADLE_LINES = { + exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [ + r'flurryagent', + r'paypal.*mpl', + r'admob.*sdk.*android', + r'google.*ad.*view', + r'google.*admob', + r'google.*play.*services', + r'crittercism', + r'heyzap', + r'jpct.*ae', + r'youtube.*android.*player.*api', + r'bugsense', + r'crashlytics', + r'ouya.*sdk', + r'libspen23', + r'firebase', + r'''["']com.facebook.android['":]''', + r'cloudrail', + r'com.tencent.bugly', + r'appcenter-push', + ] } -class ExitCode(IntEnum): - NONFREE_CODE = 1 - - -class GradleVersionCatalog: - """Parse catalog from libs.versions.toml. - - https://docs.gradle.org/current/userguide/platforms.html - """ - - def __init__(self, catalog): - self.version = { - alias: self.get_version(version) - for alias, version in catalog.get("versions", {}).items() - } - self.libraries = { - self.alias_to_accessor(alias): self.library_to_coordinate(library) - for alias, library in catalog.get("libraries", {}).items() - } - self.plugins = { - self.alias_to_accessor(alias): self.plugin_to_coordinate(plugin) - for alias, plugin in catalog.get("plugins", {}).items() - } - self.bundles = { - self.alias_to_accessor(alias): self.bundle_to_coordinates(bundle) - for alias, bundle in catalog.get("bundles", {}).items() - } - - @staticmethod - def alias_to_accessor(alias: str) -> str: - """Covert alias to accessor. - - https://docs.gradle.org/current/userguide/platforms.html#sub:mapping-aliases-to-accessors - Alias is used to define a lib in catalog. Accessor is used to access it. - """ - return alias.replace("-", ".").replace("_", ".") - - def get_version(self, version: Union[dict, str]) -> str: - if isinstance(version, str): - return version - ref = version.get("ref") - if ref: - return self.version.get(ref, "") - return ( - version.get("prefer", "") - or version.get("require", "") - or version.get("strictly", "") - ) - - def library_to_coordinate(self, library: Union[dict, str]) -> str: - """Generate the Gradle dependency coordinate from catalog.""" - if isinstance(library, str): - return library - module = library.get("module") - if not module: - group = library.get("group") - name = library.get("name") - if group and name: - module = f"{group}:{name}" - else: - return "" - - version = library.get("version") - if version: - return f"{module}:{self.get_version(version)}" - else: - return module - - def plugin_to_coordinate(self, plugin: Union[dict, str]) -> str: - """Generate the Gradle plugin coordinate from catalog.""" - if isinstance(plugin, str): - return plugin - id = plugin.get("id") - if not id: - return "" - - version = plugin.get("version") - if version: - return f"{id}:{self.get_version(version)}" - else: - return id - - def bundle_to_coordinates(self, bundle: list[str]) -> list[str]: - """Generate the Gradle dependency bundle coordinate from catalog.""" - coordinates = [] - for alias in bundle: - library = self.libraries.get(self.alias_to_accessor(alias)) - if library: - coordinates.append(library) - return coordinates - - def get_coordinate(self, accessor: str) -> list[str]: - """Get the Gradle coordinate from the catalog with an accessor.""" - if accessor.startswith("plugins."): - return [ - self.plugins.get(accessor[8:].removesuffix(".asLibraryDependency"), "") - ] - if accessor.startswith("bundles."): - return self.bundles.get(accessor[8:], []) - return [self.libraries.get(accessor, "")] - - -def get_catalogs(root: str) -> dict[str, GradleVersionCatalog]: - """Get all Gradle dependency catalogs from settings.gradle[.kts]. - - Returns a dict with the extension and the corresponding catalog. - The extension is used as the prefix of the accessor to access libs in the catalog. - """ - root = Path(root) - catalogs = {} - default_prefix = "libs" - catalog_files_m = [] - - def find_block_end(s, start): - pat = re.compile("[{}]") - depth = 1 - for m in pat.finditer(s, pos=start): - if m.group() == "{": - depth += 1 - else: - depth -= 1 - if depth == 0: - return m.start() - else: - return -1 - - groovy_file = root / "settings.gradle" - kotlin_file = root / "settings.gradle.kts" - if groovy_file.is_file(): - gradle_file = groovy_file - elif kotlin_file.is_file(): - gradle_file = kotlin_file - else: - return {} - - s = gradle_file.read_text(encoding="utf-8") - version_catalogs_m = VERSION_CATALOG_REGEX.search(s) - if version_catalogs_m: - start = version_catalogs_m.end() - end = find_block_end(s, start) - catalog_files_m = GRADLE_CATALOG_FILE_REGEX.finditer(s, start, end) - - m_default = DEFAULT_CATALOG_PREFIX_REGEX.search(s) - if m_default: - default_prefix = m_default.group(1) - default_catalog_file = Path(root) / "gradle/libs.versions.toml" - if default_catalog_file.is_file(): - with default_catalog_file.open("rb") as f: - catalogs[default_prefix] = GradleVersionCatalog(tomllib.load(f)) - for m in catalog_files_m: - catalog_file = Path(root) / m.group(2).replace("$rootDir/", "") - if catalog_file.is_file(): - with catalog_file.open("rb") as f: - catalogs[m.group(1)] = GradleVersionCatalog(tomllib.load(f)) - return catalogs - - def get_gradle_compile_commands(build): - compileCommands = [ - 'alias', - 'api', - 'apk', - 'classpath', - 'compile', - 'compileOnly', - 'id', - 'implementation', - 'provided', - 'runtimeOnly', - ] + compileCommands = ['compile', + 'provided', + 'apk', + 'implementation', + 'api', + 'compileOnly', + 'runtimeOnly'] buildTypes = ['', 'release'] + flavors = [''] if build.gradle and build.gradle != ['yes']: - flavors = common.calculate_gradle_flavor_combination(build.gradle) - else: - flavors = [''] + flavors += build.gradle - return [''.join(c) for c in itertools.product(flavors, buildTypes, compileCommands)] - - -def get_gradle_compile_commands_without_catalog(build): - return [ - re.compile(rf'''\s*{c}.*\s*\(?['"].*['"]''', re.IGNORECASE) - for c in get_gradle_compile_commands(build) - ] - - -def get_gradle_compile_commands_with_catalog(build, prefix): - return [ - re.compile(rf'\s*{c}.*\s*\(?{prefix}\.([a-z0-9.]+)', re.IGNORECASE) - for c in get_gradle_compile_commands(build) - ] - - -def get_embedded_classes(apkfile, depth=0): - """Get the list of Java classes embedded into all DEX files. - - :return: set of Java classes names as string - """ - if depth > 10: # zipbomb protection - return {_('Max recursion depth in ZIP file reached: %s') % apkfile} - - archive_regex = re.compile(r'.*\.(aab|aar|apk|apks|jar|war|xapk|zip)$') - class_regex = re.compile(r'classes.*\.dex') - classes = set() - - try: - with TemporaryDirectory() as tmp_dir, zipfile.ZipFile(apkfile, 'r') as apk_zip: - for info in apk_zip.infolist(): - # apk files can contain apk files, again - with apk_zip.open(info) as apk_fp: - if zipfile.is_zipfile(apk_fp): - classes = classes.union(get_embedded_classes(apk_fp, depth + 1)) - if not archive_regex.search(info.filename): - classes.add( - 'ZIP file without proper file extension: %s' - % info.filename - ) - continue - - with apk_zip.open(info.filename) as fp: - file_magic = fp.read(3) - if file_magic == b'dex': - if not class_regex.search(info.filename): - classes.add('DEX file with fake name: %s' % info.filename) - apk_zip.extract(info, tmp_dir) - run = common.SdkToolsPopen( - ["dexdump", '{}/{}'.format(tmp_dir, info.filename)], - output=False, - ) - classes = classes.union( - set(re.findall(r'[A-Z]+((?:\w+\/)+\w+)', run.output)) - ) - except zipfile.BadZipFile as ex: - return {_('Problem with ZIP file: %s, error %s') % (apkfile, ex)} - - return classes - - -def _datetime_now(): - """Get datetime.now(), using this funciton allows mocking it for testing.""" - return datetime.now(timezone.utc) - - -def _scanner_cachedir(): - """Get `Path` to fdroidserver cache dir.""" - cfg = common.get_config() - if not cfg: - raise ConfigurationException('config not initialized') - if "cachedir_scanner" not in cfg: - raise ConfigurationException("could not load 'cachedir_scanner' from config") - cachedir = Path(cfg["cachedir_scanner"]) - cachedir.mkdir(exist_ok=True, parents=True) - return cachedir - - -class SignatureDataMalformedException(Exception): - pass - - -class SignatureDataOutdatedException(Exception): - pass - - -class SignatureDataCacheMissException(Exception): - pass - - -class SignatureDataNoDefaultsException(Exception): - pass - - -class SignatureDataVersionMismatchException(Exception): - pass - - -class SignatureDataController: - def __init__(self, name, filename, url): - self.name = name - self.filename = filename - self.url = url - # by default we assume cache is valid indefinitely - self.cache_duration = timedelta(days=999999) - self.data = {} - - def check_data_version(self): - if self.data.get("version") != SCANNER_CACHE_VERSION: - raise SignatureDataVersionMismatchException() - - def check_last_updated(self): - """Check if the last_updated value is ok and raise an exception if expired or inaccessible. - - :raises SignatureDataMalformedException: when timestamp value is - inaccessible or not parse-able - :raises SignatureDataOutdatedException: when timestamp is older then - `self.cache_duration` - """ - last_updated = self.data.get("last_updated", None) - if last_updated: - try: - last_updated = datetime.fromtimestamp(last_updated, timezone.utc) - except ValueError as e: - raise SignatureDataMalformedException() from e - except TypeError as e: - raise SignatureDataMalformedException() from e - delta = (last_updated + self.cache_duration) - scanner._datetime_now() - if delta > timedelta(seconds=0): - logging.debug( - _('next {name} cache update due in {time}').format( - name=self.filename, time=delta - ) - ) - else: - raise SignatureDataOutdatedException() - - def fetch(self): - try: - self.fetch_signatures_from_web() - self.write_to_cache() - except Exception as e: - raise Exception( - _("downloading scanner signatures from '{}' failed").format(self.url) - ) from e - - def load(self): - try: - try: - self.load_from_cache() - self.verify_data() - self.check_last_updated() - except SignatureDataCacheMissException: - self.load_from_defaults() - except (SignatureDataOutdatedException, SignatureDataNoDefaultsException): - self.fetch_signatures_from_web() - self.write_to_cache() - except ( - SignatureDataMalformedException, - SignatureDataVersionMismatchException, - ) as e: - logging.critical( - _( - "scanner cache is malformed! You can clear it with: '{clear}'" - ).format( - clear='rm -r {}'.format(common.get_config()['cachedir_scanner']) - ) - ) - raise e - - def load_from_defaults(self): - raise SignatureDataNoDefaultsException() - - def load_from_cache(self): - sig_file = scanner._scanner_cachedir() / self.filename - if not sig_file.exists(): - raise SignatureDataCacheMissException() - with open(sig_file) as f: - self.set_data(json.load(f)) - - def write_to_cache(self): - sig_file = scanner._scanner_cachedir() / self.filename - with open(sig_file, "w", encoding="utf-8") as f: - json.dump(self.data, f, indent=2) - logging.debug("write '{}' to cache".format(self.filename)) - - def verify_data(self): - """Clean and validate `self.data`. - - Right now this function does just a basic key sanitation. - """ - self.check_data_version() - valid_keys = [ - 'timestamp', - 'last_updated', - 'version', - 'signatures', - 'cache_duration', - ] - - for k in list(self.data.keys()): - if k not in valid_keys: - del self.data[k] - - def set_data(self, new_data): - self.data = new_data - if 'cache_duration' in new_data: - self.cache_duration = timedelta(seconds=new_data['cache_duration']) - - def fetch_signatures_from_web(self): - if not self.url.startswith("https://"): - raise Exception(_("can't open non-https url: '{};".format(self.url))) - logging.debug(_("downloading '{}'").format(self.url)) - with urllib.request.urlopen(self.url) as f: # nosec B310 scheme filtered above - self.set_data(json.load(f)) - self.data['last_updated'] = scanner._datetime_now().timestamp() - - -class ExodusSignatureDataController(SignatureDataController): - def __init__(self): - super().__init__( - 'Exodus signatures', - 'exodus.json', - 'https://reports.exodus-privacy.eu.org/api/trackers', - ) - self.cache_duration = timedelta(days=1) # refresh exodus cache after one day - self.has_trackers_json_key = True - - def fetch_signatures_from_web(self): - logging.debug(_("downloading '{}'").format(self.url)) - - data = { - "signatures": {}, - "timestamp": scanner._datetime_now().timestamp(), - "last_updated": scanner._datetime_now().timestamp(), - "version": SCANNER_CACHE_VERSION, - } - - if not self.url.startswith("https://"): - raise Exception(_("can't open non-https url: '{};".format(self.url))) - with urllib.request.urlopen(self.url) as f: # nosec B310 scheme filtered above - trackerlist = json.load(f) - if self.has_trackers_json_key: - trackerlist = trackerlist["trackers"].values() - for tracker in trackerlist: - if tracker.get('code_signature'): - data["signatures"][tracker["name"]] = { - "name": tracker["name"], - "warn_code_signatures": [tracker["code_signature"]], - # exodus also provides network signatures, unused atm. - # "network_signatures": [tracker["network_signature"]], - "AntiFeatures": ["Tracking"], # TODO - "license": "NonFree", # We assume all trackers in exodus - # are non-free, although free - # trackers like piwik, acra, - # etc. might be listed by exodus - # too. - } - self.set_data(data) - - -class EtipSignatureDataController(ExodusSignatureDataController): - def __init__(self): - super().__init__() - self.name = 'ETIP signatures' - self.filename = 'etip.json' - self.url = 'https://etip.exodus-privacy.eu.org/api/trackers/?format=json' - self.has_trackers_json_key = False - - -class SUSSDataController(SignatureDataController): - def __init__(self): - super().__init__( - 'SUSS', 'suss.json', 'https://fdroid.gitlab.io/fdroid-suss/suss.json' - ) - - def load_from_defaults(self): - self.set_data(json.loads(SUSS_DEFAULT)) - - -class ScannerTool: - refresh_allowed = True - - def __init__(self): - # we could add support for loading additional signature source - # definitions from config.yml here - - self.scanner_data_lookup() - - options = common.get_options() - options_refresh_scanner = ( - hasattr(options, "refresh_scanner") - and options.refresh_scanner - and ScannerTool.refresh_allowed - ) - if options_refresh_scanner or common.get_config().get('refresh_scanner'): - self.refresh() - - self.load() - self.compile_regexes() - - def scanner_data_lookup(self): - sigsources = common.get_config().get('scanner_signature_sources', []) - logging.debug( - "scanner is configured to use signature data from: '{}'".format( - "', '".join(sigsources) - ) - ) - self.sdcs = [] - for i, source_url in enumerate(sigsources): - if source_url.lower() == 'suss': - self.sdcs.append(SUSSDataController()) - elif source_url.lower() == 'exodus': - self.sdcs.append(ExodusSignatureDataController()) - elif source_url.lower() == 'etip': - self.sdcs.append(EtipSignatureDataController()) - else: - u = urllib.parse.urlparse(source_url) - if u.scheme != 'https' or u.path == "": - raise ConfigurationException( - "Invalid 'scanner_signature_sources' configuration: '{}'. " - "Has to be a valid HTTPS-URL or match a predefined " - "constants: 'suss', 'exodus'".format(source_url) - ) - self.sdcs.append( - SignatureDataController( - source_url, - '{}_{}'.format(i, os.path.basename(u.path)), - source_url, - ) - ) - - def load(self): - for sdc in self.sdcs: - sdc.load() - - def compile_regexes(self): - self.regexs = { - 'err_code_signatures': {}, - 'err_gradle_signatures': {}, - 'warn_code_signatures': {}, - 'warn_gradle_signatures': {}, - } - for sdc in self.sdcs: - for signame, sigdef in sdc.data.get('signatures', {}).items(): - for sig in sigdef.get('code_signatures', []): - self.regexs['err_code_signatures'][sig] = re.compile( - '.*' + sig, re.IGNORECASE - ) - for sig in sigdef.get('gradle_signatures', []): - self.regexs['err_gradle_signatures'][sig] = re.compile( - '.*' + sig, re.IGNORECASE - ) - for sig in sigdef.get('warn_code_signatures', []): - self.regexs['warn_code_signatures'][sig] = re.compile( - '.*' + sig, re.IGNORECASE - ) - for sig in sigdef.get('warn_gradle_signatures', []): - self.regexs['warn_gradle_signatures'][sig] = re.compile( - '.*' + sig, re.IGNORECASE - ) - - def refresh(self): - for sdc in self.sdcs: - sdc.fetch_signatures_from_web() - sdc.write_to_cache() - - def add(self, new_controller: SignatureDataController): - self.sdcs.append(new_controller) - self.compile_regexes() - - -# TODO: change this from singleton instance to dependency injection -# use `_get_tool()` instead of accessing this directly -_SCANNER_TOOL = None - - -def _get_tool(): - """Lazy loading function for getting a ScannerTool instance. - - ScannerTool initialization need to access `common.config` values. Those are only available after initialization through `common.read_config()`. So this factory assumes config was called at an erlier point in time. - """ - if not scanner._SCANNER_TOOL: - scanner._SCANNER_TOOL = ScannerTool() - return scanner._SCANNER_TOOL + commands = [''.join(c) for c in itertools.product(flavors, buildTypes, compileCommands)] + return [re.compile(r'\s*' + c, re.IGNORECASE) for c in commands] def scan_binary(apkfile): - """Scan output of dexdump for known non-free classes.""" - logging.info(_('Scanning APK with dexdump for known non-free classes.')) - result = get_embedded_classes(apkfile) - problems, warnings = 0, 0 - for classname in result: - for suspect, regexp in _get_tool().regexs['warn_code_signatures'].items(): - if regexp.match(classname): - logging.debug("Warning: found class '%s'" % classname) - warnings += 1 - for suspect, regexp in _get_tool().regexs['err_code_signatures'].items(): - if regexp.match(classname): - logging.debug("Problem: found class '%s'" % classname) - problems += 1 - - logging.info(_('Scanning APK for extra signing blocks.')) - a = common.get_androguard_APK(str(apkfile)) - a.parse_v2_v3_signature() - for b in a._v2_blocks: - if b in APK_SIGNING_BLOCK_IDS: - logging.debug( - f"Problem: found extra signing block '{APK_SIGNING_BLOCK_IDS[b]}'" - ) + logging.info("Scanning APK for known non-free classes.") + result = common.SdkToolsPopen(["apkanalyzer", "dex", "packages", "--defined-only", apkfile], output=False) + problems = 0 + for suspect, regexp in CODE_SIGNATURES.items(): + matches = regexp.findall(result.output) + if matches: + for m in set(matches): + logging.debug("Found class '%s'" % m) problems += 1 - - if warnings: - logging.warning( - _("Found {count} warnings in {filename}").format( - count=warnings, filename=apkfile - ) - ) if problems: - logging.critical( - _("Found {count} problems in {filename}").format( - count=problems, filename=apkfile - ) - ) + logging.critical("Found problems in %s" % apkfile) return problems -def scan_source(build_dir, build=metadata.Build(), json_per_build=None): - """Scan the source code in the given directory (and all subdirectories). - - Returns - ------- - the number of fatal problems encountered. - +def scan_source(build_dir, build=metadata.Build()): + """Scan the source code in the given directory (and all subdirectories) + and return the number of fatal problems encountered """ + count = 0 - if not json_per_build: - json_per_build = MessageStore() + whitelisted = [ + 'firebase-jobdispatcher', # https://github.com/firebase/firebase-jobdispatcher-android/blob/master/LICENSE + 'com.firebaseui', # https://github.com/firebase/FirebaseUI-Android/blob/master/LICENSE + 'geofire-android' # https://github.com/firebase/geofire-java/blob/master/LICENSE + ] + + def is_whitelisted(s): + return any(wl in s for wl in whitelisted) def suspects_found(s): - for n, r in _get_tool().regexs['err_gradle_signatures'].items(): - if r.match(s): + for n, r in NON_FREE_GRADLE_LINES.items(): + if r.match(s) and not is_whitelisted(s): yield n - allowed_repos = [ - re.compile(r'^https://' + re.escape(repo) + r'/*') - for repo in [ - 'repo1.maven.org/maven2', # mavenCentral() - 'jitpack.io', - 'www.jitpack.io', - 'repo.maven.apache.org/maven2', - 'oss.jfrog.org/artifactory/oss-snapshot-local', - 'central.sonatype.com/repository/maven-snapshots', - 'oss.sonatype.org/content/repositories/snapshots', - 'oss.sonatype.org/content/repositories/releases', - 'oss.sonatype.org/content/groups/public', - 'oss.sonatype.org/service/local/staging/deploy/maven2', - 's01.oss.sonatype.org/content/repositories/snapshots', - 's01.oss.sonatype.org/content/repositories/releases', - 's01.oss.sonatype.org/content/groups/public', - 's01.oss.sonatype.org/service/local/staging/deploy/maven2', - 'clojars.org/repo', # Clojure free software libs - 'repo.clojars.org', # Clojure free software libs - 's3.amazonaws.com/repo.commonsware.com', # CommonsWare - 'plugins.gradle.org/m2', # Gradle plugin repo - 'maven.google.com', # google() + allowed_repos = [re.compile(r'^https://' + re.escape(repo) + r'/*') for repo in [ + 'repo1.maven.org/maven2', # mavenCentral() + 'jcenter.bintray.com', # jcenter() + 'jitpack.io', + 'www.jitpack.io', + 'repo.maven.apache.org/maven2', + 'oss.jfrog.org/artifactory/oss-snapshot-local', + 'oss.sonatype.org/content/repositories/snapshots', + 'oss.sonatype.org/content/repositories/releases', + 'oss.sonatype.org/content/groups/public', + 'clojars.org/repo', # Clojure free software libs + 's3.amazonaws.com/repo.commonsware.com', # CommonsWare + 'plugins.gradle.org/m2', # Gradle plugin repo + 'maven.google.com', # Google Maven Repo, https://developer.android.com/studio/build/dependencies.html#google-maven ] - ] + [ - re.compile(r'^file://' + re.escape(repo) + r'/*') - for repo in [ - '/usr/share/maven-repo', # local repo on Debian installs + ] + [re.compile(r'^file://' + re.escape(repo) + r'/*') for repo in [ + '/usr/share/maven-repo', # local repo on Debian installs ] ] - scanignore, scanignore_not_found_paths = common.getpaths_map( - build_dir, build.scanignore - ) - scandelete, scandelete_not_found_paths = common.getpaths_map( - build_dir, build.scandelete - ) + scanignore = common.getpaths_map(build_dir, build.scanignore) + scandelete = common.getpaths_map(build_dir, build.scandelete) scanignore_worked = set() scandelete_worked = set() @@ -773,48 +177,29 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None): return True return False - def ignoreproblem(what, path_in_build_dir, json_per_build): - """No summary. - - Parameters - ---------- - what: string - describing the problem, will be printed in log messages - path_in_build_dir - path to the file relative to `build`-dir - - Returns - ------- - 0 as we explicitly ignore the file, so don't count an error - + def ignoreproblem(what, path_in_build_dir): """ - msg = 'Ignoring %s at %s' % (what, path_in_build_dir) + :param what: string describing the problem, will be printed in log messages + :param path_in_build_dir: path to the file relative to `build`-dir + "returns: 0 as we explicitly ignore the file, so don't count an error + """ + msg = ('Ignoring %s at %s' % (what, path_in_build_dir)) logging.info(msg) if json_per_build is not None: - json_per_build.infos.append([msg, path_in_build_dir]) + json_per_build['infos'].append([msg, path_in_build_dir]) return 0 - def removeproblem(what, path_in_build_dir, filepath, json_per_build): - """No summary. - - Parameters - ---------- - what: string - describing the problem, will be printed in log messages - path_in_build_dir - path to the file relative to `build`-dir - filepath - Path (relative to our current path) to the file - - Returns - ------- - 0 as we deleted the offending file - + def removeproblem(what, path_in_build_dir, filepath): """ - msg = 'Removing %s at %s' % (what, path_in_build_dir) + :param what: string describing the problem, will be printed in log messages + :param path_in_build_dir: path to the file relative to `build`-dir + :param filepath: Path (relative to our current path) to the file + "returns: 0 as we deleted the offending file + """ + msg = ('Removing %s at %s' % (what, path_in_build_dir)) logging.info(msg) if json_per_build is not None: - json_per_build.infos.append([msg, path_in_build_dir]) + json_per_build['infos'].append([msg, path_in_build_dir]) try: os.remove(filepath) except FileNotFoundError: @@ -824,66 +209,44 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None): pass return 0 - def warnproblem(what, path_in_build_dir, json_per_build): - """No summary. - - Parameters - ---------- - what: string - describing the problem, will be printed in log messages - path_in_build_dir - path to the file relative to `build`-dir - - Returns - ------- - 0, as warnings don't count as errors - + def warnproblem(what, path_in_build_dir): + """ + :param what: string describing the problem, will be printed in log messages + :param path_in_build_dir: path to the file relative to `build`-dir + :returns: 0, as warnings don't count as errors """ if toignore(path_in_build_dir): return 0 logging.warning('Found %s at %s' % (what, path_in_build_dir)) if json_per_build is not None: - json_per_build.warnings.append([what, path_in_build_dir]) + json_per_build['warnings'].append([what, path_in_build_dir]) return 0 - def handleproblem(what, path_in_build_dir, filepath, json_per_build): - """Dispatches to problem handlers (ignore, delete, warn). - - Or returns 1 for increasing the error count. - - Parameters - ---------- - what: string - describing the problem, will be printed in log messages - path_in_build_dir - path to the file relative to `build`-dir - filepath - Path (relative to our current path) to the file - - Returns - ------- - 0 if the problem was ignored/deleted/is only a warning, 1 otherwise + def handleproblem(what, path_in_build_dir, filepath): + """Dispatches to problem handlers (ignore, delete, warn) or returns 1 + for increasing the error count + :param what: string describing the problem, will be printed in log messages + :param path_in_build_dir: path to the file relative to `build`-dir + :param filepath: Path (relative to our current path) to the file + :returns: 0 if the problem was ignored/deleted/is only a warning, 1 otherwise """ - options = common.get_options() if toignore(path_in_build_dir): - return ignoreproblem(what, path_in_build_dir, json_per_build) + return ignoreproblem(what, path_in_build_dir) if todelete(path_in_build_dir): - return removeproblem(what, path_in_build_dir, filepath, json_per_build) - if 'src/test' in path_in_build_dir or '/test/' in path_in_build_dir: - return warnproblem(what, path_in_build_dir, json_per_build) + return removeproblem(what, path_in_build_dir, filepath) + if 'src/test' in filepath or '/test/' in path_in_build_dir: + return warnproblem(what, path_in_build_dir) if options and 'json' in vars(options) and options.json: - json_per_build.errors.append([what, path_in_build_dir]) - if options and ( - options.verbose or not ('json' in vars(options) and options.json) - ): + json_per_build['errors'].append([what, path_in_build_dir]) + if options and (options.verbose or not ('json' in vars(options) and options.json)): logging.error('Found %s at %s' % (what, path_in_build_dir)) return 1 def is_executable(path): return os.path.exists(path) and os.access(path, os.X_OK) - textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f}) # fmt: skip + textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f}) def is_binary(path): d = None @@ -892,21 +255,15 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None): return bool(d.translate(None, textchars)) # False positives patterns for files that are binary and executable. - safe_paths = [ - re.compile(r) - for r in [ - r".*/drawable[^/]*/.*\.png$", # png drawables - r".*/mipmap[^/]*/.*\.png$", # png mipmaps + safe_paths = [re.compile(r) for r in [ + r".*/drawable[^/]*/.*\.png$", # png drawables + r".*/mipmap[^/]*/.*\.png$", # png mipmaps ] ] def is_image_file(path): - try: - mimetype = magic.from_file(path, mime=True) - if mimetype and mimetype.startswith('image/'): - return True - except Exception as e: - logging.info(e) + if imghdr.what(path) is not None: + return True def safe_path(path_in_build_dir): for sp in safe_paths: @@ -914,32 +271,21 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None): return True return False - def is_used_by_gradle_without_catalog(line): - return any( - command.match(line) - for command in get_gradle_compile_commands_without_catalog(build) - ) + gradle_compile_commands = get_gradle_compile_commands(build) - def is_used_by_gradle_with_catalog(line, prefix): - for m in ( - command.match(line) - for command in get_gradle_compile_commands_with_catalog(build, prefix) - ): - if m: - return m + def is_used_by_gradle(line): + return any(command.match(line) for command in gradle_compile_commands) - all_catalogs = {} # Iterate through all files in the source code for root, dirs, files in os.walk(build_dir, topdown=True): + # It's topdown, so checking the basename is enough for ignoredir in ('.hg', '.git', '.svn', '.bzr'): if ignoredir in dirs: dirs.remove(ignoredir) - if "settings.gradle" in files or "settings.gradle.kts" in files: - all_catalogs[str(root)] = get_catalogs(root) - for curfile in files: + if curfile in ['.DS_Store']: continue @@ -950,179 +296,70 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None): continue path_in_build_dir = os.path.relpath(filepath, build_dir) + extension = os.path.splitext(path_in_build_dir)[1] if curfile in ('gradle-wrapper.jar', 'gradlew', 'gradlew.bat'): - removeproblem(curfile, path_in_build_dir, filepath, json_per_build) - elif curfile.endswith('.apk'): - removeproblem( - _('Android APK file'), path_in_build_dir, filepath, json_per_build - ) + removeproblem(curfile, path_in_build_dir, filepath) + elif extension == '.apk': + removeproblem(_('Android APK file'), path_in_build_dir, filepath) - elif curfile.endswith('.a'): - count += handleproblem( - _('static library'), path_in_build_dir, filepath, json_per_build - ) - elif curfile.endswith('.aar'): - count += handleproblem( - _('Android AAR library'), - path_in_build_dir, - filepath, - json_per_build, - ) - elif curfile.endswith('.class'): - count += handleproblem( - _('Java compiled class'), - path_in_build_dir, - filepath, - json_per_build, - ) - elif curfile.endswith('.dex'): - count += handleproblem( - _('Android DEX code'), path_in_build_dir, filepath, json_per_build - ) - elif curfile.endswith('.gz') or curfile.endswith('.tgz'): - count += handleproblem( - _('gzip file archive'), path_in_build_dir, filepath, json_per_build - ) - # We use a regular expression here to also match versioned shared objects like .so.0.0.0 - elif re.match(r'.*\.so(\..+)*$', curfile): - count += handleproblem( - _('shared library'), path_in_build_dir, filepath, json_per_build - ) - elif curfile.endswith('.zip'): - count += handleproblem( - _('ZIP file archive'), path_in_build_dir, filepath, json_per_build - ) - elif curfile.endswith('.jar'): + elif extension == '.a': + count += handleproblem(_('static library'), path_in_build_dir, filepath) + elif extension == '.aar': + count += handleproblem(_('Android AAR library'), path_in_build_dir, filepath) + elif extension == '.class': + count += handleproblem(_('Java compiled class'), path_in_build_dir, filepath) + elif extension == '.dex': + count += handleproblem(_('Android DEX code'), path_in_build_dir, filepath) + elif extension == '.gz': + count += handleproblem(_('gzip file archive'), path_in_build_dir, filepath) + elif extension == '.so': + count += handleproblem(_('shared library'), path_in_build_dir, filepath) + elif extension == '.zip': + count += handleproblem(_('ZIP file archive'), path_in_build_dir, filepath) + elif extension == '.jar': for name in suspects_found(curfile): - count += handleproblem( - 'usual suspect \'%s\'' % name, - path_in_build_dir, - filepath, - json_per_build, - ) - count += handleproblem( - _('Java JAR file'), path_in_build_dir, filepath, json_per_build - ) - elif curfile.endswith('.wasm'): - count += handleproblem( - _('WebAssembly binary file'), - path_in_build_dir, - filepath, - json_per_build, - ) + count += handleproblem('usual suspect \'%s\'' % name, path_in_build_dir, filepath) + count += handleproblem(_('Java JAR file'), path_in_build_dir, filepath) - elif curfile.endswith('.java'): + elif extension == '.java': if not os.path.isfile(filepath): continue with open(filepath, 'r', errors='replace') as f: for line in f: if 'DexClassLoader' in line: - count += handleproblem( - 'DexClassLoader', - path_in_build_dir, - filepath, - json_per_build, - ) + count += handleproblem('DexClassLoader', path_in_build_dir, filepath) break - elif curfile.endswith('.gradle') or curfile.endswith('.gradle.kts'): - catalog_path = str(build_dir) - # Find the longest path of dir that the curfile is in - for p in all_catalogs: - if os.path.commonpath([root, p]) == p: - catalog_path = p - catalogs = all_catalogs.get(catalog_path, {}) - + elif extension == '.gradle': if not os.path.isfile(filepath): continue with open(filepath, 'r', errors='replace') as f: lines = f.readlines() for i, line in enumerate(lines): - if is_used_by_gradle_without_catalog(line): + if is_used_by_gradle(line): for name in suspects_found(line): - count += handleproblem( - f"usual suspect '{name}'", - path_in_build_dir, - filepath, - json_per_build, - ) - for prefix, catalog in catalogs.items(): - m = is_used_by_gradle_with_catalog(line, prefix) - if not m: - continue - accessor = m[1] - coordinates = catalog.get_coordinate(accessor) - for coordinate in coordinates: - for name in suspects_found(coordinate): - count += handleproblem( - f"usual suspect '{prefix}.{accessor}: {name}'", - path_in_build_dir, - filepath, - json_per_build, - ) - noncomment_lines = [ - line for line in lines if not common.gradle_comment.match(line) - ] - no_comments = re.sub( - r'/\*.*?\*/', '', ''.join(noncomment_lines), flags=re.DOTALL - ) + count += handleproblem("usual suspect \'%s\'" % (name), + path_in_build_dir, filepath) + noncomment_lines = [line for line in lines if not common.gradle_comment.match(line)] + no_comments = re.sub(r'/\*.*?\*/', '', ''.join(noncomment_lines), flags=re.DOTALL) for url in MAVEN_URL_REGEX.findall(no_comments): if not any(r.match(url) for r in allowed_repos): - count += handleproblem( - 'unknown maven repo \'%s\'' % url, - path_in_build_dir, - filepath, - json_per_build, - ) + count += handleproblem('unknown maven repo \'%s\'' % url, path_in_build_dir, filepath) - elif os.path.splitext(path_in_build_dir)[1] in ['', '.bin', '.out', '.exe']: + elif extension in ['', '.bin', '.out', '.exe']: if is_binary(filepath): - count += handleproblem( - 'binary', path_in_build_dir, filepath, json_per_build - ) - - elif curfile in DEPFILE: - d = root - while d.startswith(str(build_dir)): - for lockfile in DEPFILE[curfile]: - if os.path.isfile(os.path.join(d, lockfile)): - break - else: - d = os.path.dirname(d) - continue - break - else: - count += handleproblem( - _('dependency file without lock'), - path_in_build_dir, - filepath, - json_per_build, - ) + count += handleproblem('binary', path_in_build_dir, filepath) elif is_executable(filepath): - if is_binary(filepath) and not ( - safe_path(path_in_build_dir) or is_image_file(filepath) - ): - warnproblem( - _('executable binary, possibly code'), - path_in_build_dir, - json_per_build, - ) - - for p in scanignore_not_found_paths: - logging.error(_("Non-exist scanignore path: %s") % p) - count += 1 + if is_binary(filepath) and not (safe_path(path_in_build_dir) or is_image_file(filepath)): + warnproblem(_('executable binary, possibly code'), path_in_build_dir) for p in scanignore: if p not in scanignore_worked: logging.error(_('Unused scanignore path: %s') % p) count += 1 - for p in scandelete_not_found_paths: - logging.error(_("Non-exist scandelete path: %s") % p) - count += 1 - for p in scandelete: if p not in scandelete_worked: logging.error(_('Unused scandelete path: %s') % p) @@ -1132,42 +369,18 @@ def scan_source(build_dir, build=metadata.Build(), json_per_build=None): def main(): - parser = ArgumentParser( - usage="%(prog)s [options] [(APPID[:VERCODE] | path/to.apk) ...]" - ) + global config, options, json_per_build + + # Parse command line... + parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) - parser.add_argument( - "appid", - nargs='*', - help=_("application ID with optional versionCode in the form APPID[:VERCODE]"), - ) - parser.add_argument( - "-f", - "--force", - action="store_true", - default=False, - help=_("Force scan of disabled apps and builds."), - ) - parser.add_argument( - "--json", action="store_true", default=False, help=_("Output JSON to stdout.") - ) - parser.add_argument( - "-r", - "--refresh", - dest="refresh_scanner", - action="store_true", - default=False, - help=_("fetch the latest version of signatures from the web"), - ) - parser.add_argument( - "-e", - "--exit-code", - action="store_true", - default=False, - help=_("Exit with a non-zero code if problems were found"), - ) + parser.add_argument("appid", nargs='*', help=_("application ID with optional versionCode in the form APPID[:VERCODE]")) + parser.add_argument("-f", "--force", action="store_true", default=False, + help=_("Force scan of disabled apps and builds.")) + parser.add_argument("--json", action="store_true", default=False, + help=_("Output JSON to stdout.")) metadata.add_metadata_arguments(parser) - options = common.parse_args(parser) + options = parser.parse_args() metadata.warnings_action = options.W json_output = dict() @@ -1177,34 +390,14 @@ def main(): else: logging.getLogger().setLevel(logging.ERROR) - # initialize/load configuration values - common.get_config() + config = common.read_config(options) + + # Read all app and srclib metadata + allapps = metadata.read_metadata() + apps = common.read_app_args(options.appid, allapps, True) probcount = 0 - appids = [] - for apk in options.appid: - if os.path.isfile(apk): - count = scanner.scan_binary(apk) - if count > 0: - logging.warning( - _('Scanner found {count} problems in {apk}').format( - count=count, apk=apk - ) - ) - probcount += count - else: - appids.append(apk) - - if not appids: - if options.exit_code and probcount > 0: - sys.exit(ExitCode.NONFREE_CODE) - if options.refresh_scanner: - _get_tool() - return - - apps = common.read_app_args(appids, allow_version_codes=True) - build_dir = 'build' if not os.path.isdir(build_dir): logging.info("Creating build directory") @@ -1213,13 +406,12 @@ def main(): extlib_dir = os.path.join(build_dir, 'extlib') for appid, app in apps.items(): + json_per_appid = dict() if app.Disabled and not options.force: logging.info(_("Skipping {appid}: disabled").format(appid=appid)) - json_per_appid['disabled'] = MessageStore().infos.append( - 'Skipping: disabled' - ) + json_per_appid['disabled'] = json_per_build['infos'].append('Skipping: disabled') continue try: @@ -1228,1776 +420,66 @@ def main(): else: build_dir = os.path.join('build', appid) - if app.get('Builds'): + if app.builds: logging.info(_("Processing {appid}").format(appid=appid)) # Set up vcs interface and make sure we have the latest code... vcs = common.getvcs(app.RepoType, app.Repo, build_dir) else: - logging.info( - _( - "{appid}: no builds specified, running on current source state" - ).format(appid=appid) - ) - json_per_build = MessageStore() + logging.info(_("{appid}: no builds specified, running on current source state") + .format(appid=appid)) + json_per_build = DEFAULT_JSON_PER_BUILD json_per_appid['current-source-state'] = json_per_build - count = scan_source(build_dir, json_per_build=json_per_build) + count = scan_source(build_dir) if count > 0: - logging.warning( - _('Scanner found {count} problems in {appid}:').format( - count=count, appid=appid - ) - ) + logging.warning(_('Scanner found {count} problems in {appid}:') + .format(count=count, appid=appid)) probcount += count - app['Builds'] = [] + app.builds = [] - for build in app.get('Builds', []): - json_per_build = MessageStore() + for build in app.builds: + json_per_build = DEFAULT_JSON_PER_BUILD json_per_appid[build.versionCode] = json_per_build if build.disable and not options.force: - logging.info( - "...skipping version %s - %s" - % (build.versionName, build.get('disable', build.commit[1:])) - ) + logging.info("...skipping version %s - %s" % ( + build.versionName, build.get('disable', build.commit[1:]))) continue logging.info("...scanning version " + build.versionName) # Prepare the source code... - common.prepare_source( - vcs, app, build, build_dir, srclib_dir, extlib_dir, False - ) + common.prepare_source(vcs, app, build, + build_dir, srclib_dir, + extlib_dir, False) - count = scan_source(build_dir, build, json_per_build=json_per_build) + count = scan_source(build_dir, build) if count > 0: - logging.warning( - _( - 'Scanner found {count} problems in {appid}:{versionCode}:' - ).format( - count=count, appid=appid, versionCode=build.versionCode - ) - ) + logging.warning(_('Scanner found {count} problems in {appid}:{versionCode}:') + .format(count=count, appid=appid, versionCode=build.versionCode)) probcount += count except BuildException as be: - logging.warning( - 'Could not scan app %s due to BuildException: %s' % (appid, be) - ) + logging.warning('Could not scan app %s due to BuildException: %s' % ( + appid, be)) probcount += 1 except VCSException as vcse: logging.warning('VCS error while scanning app %s: %s' % (appid, vcse)) probcount += 1 except Exception: - logging.warning( - 'Could not scan app %s due to unknown error: %s' - % (appid, traceback.format_exc()) - ) + logging.warning('Could not scan app %s due to unknown error: %s' % ( + appid, traceback.format_exc())) probcount += 1 for k, v in json_per_appid.items(): - if len(v.errors) or len(v.warnings) or len(v.infos): - json_output[appid] = { - k: dict((field.name, getattr(v, field.name)) for field in fields(v)) - for k, v in json_per_appid.items() - } + if len(v['errors']) or len(v['warnings']) or len(v['infos']): + json_output[appid] = json_per_appid break logging.info(_("Finished")) if options.json: print(json.dumps(json_output)) - elif probcount or options.verbose: + else: print(_("%d problems found") % probcount) if __name__ == "__main__": main() - - -SUSS_DEFAULT = r'''{ - "cache_duration": 86400, - "signatures": { - "com.amazon.device.ads": { - "anti_features": [ - "Ads", - "NonFreeComp" - ], - "code_signatures": [ - "com/amazon/device/ads" - ], - "description": "an interface for views used to retrieve and display Amazon ads.", - "license": "NonFree" - }, - "com.amazon.device.associates": { - "anti_features": [ - "Ads", - "NonFreeComp" - ], - "code_signatures": [ - "com/amazon/device/associates" - ], - "description": "library for Amazon\u2019s affiliate marketing program.", - "license": "NonFree" - }, - "com.amazon.device.iap": { - "anti_features": [ - "NonFreeComp", - "NonFreeNet" - ], - "code_signatures": [ - "com/amazon/device/iap" - ], - "description": "allows an app to present, process, and fulfill purchases of digital content and subscriptions within your app.", - "license": "NonFree" - }, - "com.amazonaws": { - "code_signatures": [ - "com/amazonaws/AbortedException", - "com/amazonaws/AmazonClientException", - "com/amazonaws/AmazonServiceException$ErrorType", - "com/amazonaws/AmazonServiceException", - "com/amazonaws/AmazonWebServiceClient", - "com/amazonaws/AmazonWebServiceRequest", - "com/amazonaws/AmazonWebServiceResponse", - "com/amazonaws/async", - "com/amazonaws/auth", - "com/amazonaws/ClientConfiguration", - "com/amazonaws/cognito", - "com/amazonaws/DefaultRequest", - "com/amazonaws/event", - "com/amazonaws/handlers", - "com/amazonaws/http", - "com/amazonaws/HttpMethod", - "com/amazonaws/internal", - "com/amazonaws/logging", - "com/amazonaws/metrics", - "com/amazonaws/mobile", - "com/amazonaws/mobileconnectors", - "com/amazonaws/Protocol", - "com/amazonaws/regions", - "com/amazonaws/RequestClientOptions$Marker", - "com/amazonaws/RequestClientOptions", - "com/amazonaws/Request", - "com/amazonaws/ResponseMetadata", - "com/amazonaws/Response", - "com/amazonaws/retry", - "com/amazonaws/SDKGlobalConfiguration", - "com/amazonaws/ServiceNameFactory", - "com/amazonaws/services", - "com/amazonaws/transform", - "com/amazonaws/util" - ], - "gradle_signatures": [ - "com.amazonaws:amazon-kinesis-aggregator", - "com.amazonaws:amazon-kinesis-connectors", - "com.amazonaws:amazon-kinesis-deaggregator", - "com.amazonaws:aws-android-sdk-apigateway-core", - "com.amazonaws:aws-android-sdk-auth-core", - "com.amazonaws:aws-android-sdk-auth-facebook", - "com.amazonaws:aws-android-sdk-auth-google", - "com.amazonaws:aws-android-sdk-auth-ui", - "com.amazonaws:aws-android-sdk-auth-userpools", - "com.amazonaws:aws-android-sdk-cognito", - "com.amazonaws:aws-android-sdk-cognitoauth", - "com.amazonaws:aws-android-sdk-cognitoidentityprovider-asf", - "com.amazonaws:aws-android-sdk-comprehend", - "com.amazonaws:aws-android-sdk-core", - "com.amazonaws:aws-android-sdk-ddb", - "com.amazonaws:aws-android-sdk-ddb-document", - "com.amazonaws:aws-android-sdk-iot", - "com.amazonaws:aws-android-sdk-kinesis", - "com.amazonaws:aws-android-sdk-kinesisvideo", - "com.amazonaws:aws-android-sdk-kinesisvideo-archivedmedia", - "com.amazonaws:aws-android-sdk-kms", - "com.amazonaws:aws-android-sdk-lambda", - "com.amazonaws:aws-android-sdk-lex", - "com.amazonaws:aws-android-sdk-location", - "com.amazonaws:aws-android-sdk-logs", - "com.amazonaws:aws-android-sdk-mobileanalytics", - "com.amazonaws:aws-android-sdk-mobile-client", - "com.amazonaws:aws-android-sdk-pinpoint", - "com.amazonaws:aws-android-sdk-polly", - "com.amazonaws:aws-android-sdk-rekognition", - "com.amazonaws:aws-android-sdk-s3", - "com.amazonaws:aws-android-sdk-ses", - "com.amazonaws:aws-android-sdk-sns", - "com.amazonaws:aws-android-sdk-sqs", - "com.amazonaws:aws-android-sdk-textract", - "com.amazonaws:aws-android-sdk-transcribe", - "com.amazonaws:aws-android-sdk-translate", - "com.amazonaws:dynamodb-key-diagnostics-library", - "com.amazonaws:DynamoDBLocal", - "com.amazonaws:dynamodb-lock-client", - "com.amazonaws:ivs-broadcast", - "com.amazonaws:ivs-player", - "com.amazonaws:kinesis-storm-spout" - ], - "license": "NonFree", - "name": "AmazonAWS" - }, - "com.android.billingclient": { - "code_signatures": [ - "com/android/billingclient" - ], - "documentation": [ - "https://developer.android.com/google/play/billing/integrate" - ], - "gradle_signatures": [ - "com.android.billingclient", - "com.google.androidbrowserhelper:billing", - "com.anjlab.android.iab.v3:library", - "com.github.penn5:donations", - "me.proton.core:payment-iap" - ], - "license": "NonFree", - "name": "BillingClient" - }, - "com.android.installreferrer": { - "anti_features": [ - "NonFreeDep", - "NonFreeNet" - ], - "code_signatures": [ - "com/android/installreferrer" - ], - "documentation": [ - "https://developer.android.com/google/play/installreferrer/library" - ], - "gradle_signatures": [ - "com.android.installreferrer" - ], - "license": "NonFree", - "name": "Play Install Referrer Library" - }, - "com.anychart": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/anychart" - ], - "description": "a data visualization library for easily creating interactive charts in Android apps.", - "license": "NonFree" - }, - "com.appboy": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/appboy" - ], - "description": "Targets customers based on personal interests, location, past purchases, and more; profiles users, segments audiences, and utilizes analytics for targeted advertisements.", - "license": "NonFree" - }, - "com.appbrain": { - "anti_features": [ - "Ads", - "NonFreeComp" - ], - "code_signatures": [ - "com/appbrain" - ], - "description": "See Exodus Privacy.", - "license": "NonFree" - }, - "com.applause.android": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/applause/android" - ], - "description": "crowd-sourced testing. See Crunchbase and Exodus Privacy.", - "license": "NonFree" - }, - "com.applovin": { - "anti_features": [ - "Ads" - ], - "code_signatures": [ - "com/applovin" - ], - "description": "a mobile advertising technology company that enables brands to create mobile marketing campaigns that are fueled by data. Primary targets games.", - "license": "NonFree" - }, - "com.appsflyer": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/appsflyer" - ], - "description": "a mobile & attribution analytics platform.", - "license": "NonFree" - }, - "com.apptentive": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/apptentive" - ], - "description": "See Exodus Privacy.", - "license": "NonFree" - }, - "com.apptimize": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/apptimize" - ], - "description": "See Exodus Privacy and Crunchbase.", - "license": "NonFree" - }, - "com.askingpoint": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/askingpoint" - ], - "description": "complete mobile user engagement solution (power local, In-application evaluations and audits, input, user support, mobile reviews and informing).", - "license": "NonFree" - }, - "com.baidu.mobstat": { - "code_signatures": [ - "com/baidu/mobstat" - ], - "documentation": [ - "https://mtj.baidu.com/web/sdk/index" - ], - "gradle_signatures": [ - "com.baidu.mobstat" - ], - "license": "NonFree", - "name": "\u767e\u5ea6\u79fb\u52a8\u7edf\u8ba1SDK" - }, - "com.batch": { - "anti_features": [ - "Ads", - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/batch" - ], - "description": "mobile engagement platform to execute CRM tactics over iOS, Android & mobile websites.", - "license": "NonFree" - }, - "com.bosch.mtprotocol": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/bosch/mtprotocol" - ], - "description": "simplify and manage use of Bosch GLM and PLR laser rangefinders with Bluetooth connectivity.", - "license": "NonFree" - }, - "com.bugsee.library.Bugsee": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/bugsee/library/Bugsee" - ], - "description": "see video, network and logs that led to bugs and crashes in live apps. No need to reproduce intermittent bugs. With Bugsee, all the crucial data is always there.", - "license": "NonFree" - }, - "com.bugsense": { - "code_signatures": [ - "com/bugsense" - ], - "documentation": [ - "https://github.com/bugsense/docs/blob/master/android.md" - ], - "gradle_signatures": [ - "com.bugsense" - ], - "license": "NonFree", - "name": "BugSense" - }, - "com.chartboost.sdk": { - "anti_features": [ - "Ads", - "NonFreeComp" - ], - "code_signatures": [ - "com/chartboost/sdk" - ], - "description": "create customized interstitial and video ads, promote new games, and swap traffic with one another. For more details, see Wikipedia.", - "license": "NonFree" - }, - "com.cloudrail": { - "code_signature": [ - "com/cloudrail" - ], - "documentation": [ - "https://cloudrail.com/" - ], - "gradle_signatures": [ - "com.cloudrail" - ], - "license": "NonFree", - "name": "CloudRail" - }, - "com.comscore.analytics": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/comscore" - ], - "description": "See Wikipedia for details.", - "license": "NonFree" - }, - "com.crashlytics.sdk.android": { - "code_signatures": [ - "com/crashlytics" - ], - "documentation": [ - "https://firebase.google.com/docs/crashlytics" - ], - "gradle_signatures": [ - "crashlytics" - ], - "license": "NonFree", - "name": "Firebase Crashlytics" - }, - "com.crittercism": { - "code_signatures": [ - "com/crittercism" - ], - "documentation": [ - "https://github.com/crittercism/crittercism-unity-android" - ], - "gradle_signatures": [ - "com.crittercism" - ], - "license": "NonFree", - "name": "Crittercism Plugin for Unity Crash Reporting" - }, - "com.criware": { - "anti_features": [ - "NonFreeComp", - "NonFreeAssets" - ], - "code_signatures": [ - "com/criware" - ], - "description": "audio and video solutions that can be integrated with popular game engines.", - "license": "NonFree" - }, - "com.deezer.sdk": { - "anti_features": [ - "NonFreeComp", - "NonFreeNet" - ], - "code_signatures": [ - "com/deezer/sdk" - ], - "description": "a closed-source API for the Deezer music streaming service.", - "license": "NonFree" - }, - "com.dynamicyield": { - "anti_features": [ - "Ads", - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/dynamicyield" - ], - "description": "targeted advertising. Tracks user via location (GPS, WiFi, location data). Collects PII, profiling. See Exodus Privacy for more details.", - "license": "NonFree" - }, - "com.dynatrace.android.app": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/dynatrace/android/app" - ], - "description": "See Crunchbase and Exodus Privacy.", - "license": "NonFree" - }, - "com.ensighten": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/ensighten" - ], - "description": "organizations can leverage first-party customer data and profiles to fuel omni-channel action and insight using their existing technology investments. See Crunchbase and Exodus Privacy.", - "license": "NonFree" - }, - "com.epicgames.mobile.eossdk": { - "anti_features": [ - "NonFreeComp", - "NonFreeNet" - ], - "code_signatures": [ - "com/epicgames/mobile/eossdk" - ], - "description": "integrate games with Epic Account Services and Epic Games Store", - "license": "NonFree" - }, - "com.facebook.android": { - "code_signatures": [ - "com/facebook/AccessToken", - "com/facebook/AccessTokenCache", - "com/facebook/AccessTokenManager", - "com/facebook/AccessTokenSource", - "com/facebook/AccessTokenTracker", - "com/facebook/all/All", - "com/facebook/appevents/aam/MetadataIndexer", - "com/facebook/appevents/aam/MetadataMatcher", - "com/facebook/appevents/aam/MetadataRule", - "com/facebook/appevents/aam/MetadataViewObserver", - "com/facebook/appevents/AccessTokenAppIdPair", - "com/facebook/appevents/AnalyticsUserIDStore", - "com/facebook/appevents/AppEvent", - "com/facebook/appevents/AppEventCollection", - "com/facebook/appevents/AppEventDiskStore", - "com/facebook/appevents/AppEventQueue", - "com/facebook/appevents/AppEventsConstants", - "com/facebook/appevents/AppEventsLogger", - "com/facebook/appevents/AppEventsLoggerImpl", - "com/facebook/appevents/AppEventsManager", - "com/facebook/appevents/AppEventStore", - "com/facebook/appevents/cloudbridge/AppEventsCAPIManager", - "com/facebook/appevents/cloudbridge/AppEventsConversionsAPITransformer", - "com/facebook/appevents/cloudbridge/AppEventsConversionsAPITransformerWebRequests", - "com/facebook/appevents/codeless/CodelessLoggingEventListener", - "com/facebook/appevents/codeless/CodelessManager", - "com/facebook/appevents/codeless/CodelessMatcher", - "com/facebook/appevents/codeless/internal/Constants", - "com/facebook/appevents/codeless/internal/EventBinding", - "com/facebook/appevents/codeless/internal/ParameterComponent", - "com/facebook/appevents/codeless/internal/PathComponent", - "com/facebook/appevents/codeless/internal/SensitiveUserDataUtils", - "com/facebook/appevents/codeless/internal/UnityReflection", - "com/facebook/appevents/codeless/internal/ViewHierarchy", - "com/facebook/appevents/codeless/RCTCodelessLoggingEventListener", - "com/facebook/appevents/codeless/ViewIndexer", - "com/facebook/appevents/codeless/ViewIndexingTrigger", - "com/facebook/appevents/eventdeactivation/EventDeactivationManager", - "com/facebook/appevents/FacebookSDKJSInterface", - "com/facebook/appevents/FlushReason", - "com/facebook/appevents/FlushResult", - "com/facebook/appevents/FlushStatistics", - "com/facebook/appevents/iap/InAppPurchaseActivityLifecycleTracker", - "com/facebook/appevents/iap/InAppPurchaseAutoLogger", - "com/facebook/appevents/iap/InAppPurchaseBillingClientWrapper", - "com/facebook/appevents/iap/InAppPurchaseEventManager", - "com/facebook/appevents/iap/InAppPurchaseLoggerManager", - "com/facebook/appevents/iap/InAppPurchaseManager", - "com/facebook/appevents/iap/InAppPurchaseSkuDetailsWrapper", - "com/facebook/appevents/iap/InAppPurchaseUtils", - "com/facebook/appevents/integrity/BlocklistEventsManager", - "com/facebook/appevents/integrity/IntegrityManager", - "com/facebook/appevents/integrity/MACARuleMatchingManager", - "com/facebook/appevents/integrity/ProtectedModeManager", - "com/facebook/appevents/integrity/RedactedEventsManager", - "com/facebook/appevents/internal/ActivityLifecycleTracker", - "com/facebook/appevents/InternalAppEventsLogger", - "com/facebook/appevents/internal/AppEventsLoggerUtility", - "com/facebook/appevents/internal/AppEventUtility", - "com/facebook/appevents/internal/AutomaticAnalyticsLogger", - "com/facebook/appevents/internal/Constants", - "com/facebook/appevents/internal/FileDownloadTask", - "com/facebook/appevents/internal/HashUtils", - "com/facebook/appevents/internal/SessionInfo", - "com/facebook/appevents/internal/SessionLogger", - "com/facebook/appevents/internal/SourceApplicationInfo", - "com/facebook/appevents/internal/ViewHierarchyConstants", - "com/facebook/appevents/ml/Model", - "com/facebook/appevents/ml/ModelManager", - "com/facebook/appevents/ml/MTensor", - "com/facebook/appevents/ml/Operator", - "com/facebook/appevents/ml/Utils", - "com/facebook/appevents/ondeviceprocessing/OnDeviceProcessingManager", - "com/facebook/appevents/ondeviceprocessing/RemoteServiceParametersHelper", - "com/facebook/appevents/ondeviceprocessing/RemoteServiceWrapper", - "com/facebook/appevents/PersistedEvents", - "com/facebook/appevents/restrictivedatafilter/RestrictiveDataManager", - "com/facebook/appevents/SessionEventsState", - "com/facebook/appevents/suggestedevents/FeatureExtractor", - "com/facebook/appevents/suggestedevents/PredictionHistoryManager", - "com/facebook/appevents/suggestedevents/SuggestedEventsManager", - "com/facebook/appevents/suggestedevents/SuggestedEventViewHierarchy", - "com/facebook/appevents/suggestedevents/ViewObserver", - "com/facebook/appevents/suggestedevents/ViewOnClickListener", - "com/facebook/appevents/UserDataStore", - "com/facebook/applinks/AppLinkData", - "com/facebook/applinks/AppLinks", - "com/facebook/applinks/FacebookAppLinkResolver", - "com/facebook/AuthenticationToken", - "com/facebook/AuthenticationTokenCache", - "com/facebook/AuthenticationTokenClaims", - "com/facebook/AuthenticationTokenHeader", - "com/facebook/AuthenticationTokenManager", - "com/facebook/AuthenticationTokenTracker", - "com/facebook/bolts/AggregateException", - "com/facebook/bolts/AndroidExecutors", - "com/facebook/bolts/AppLink", - "com/facebook/bolts/AppLinkResolver", - "com/facebook/bolts/AppLinks", - "com/facebook/bolts/BoltsExecutors", - "com/facebook/bolts/CancellationToken", - "com/facebook/bolts/CancellationTokenRegistration", - "com/facebook/bolts/CancellationTokenSource", - "com/facebook/bolts/Continuation", - "com/facebook/bolts/ExecutorException", - "com/facebook/bolts/Task", - "com/facebook/bolts/TaskCompletionSource", - "com/facebook/bolts/UnobservedErrorNotifier", - "com/facebook/bolts/UnobservedTaskException", - "com/facebook/CallbackManager", - "com/facebook/common/Common", - "com/facebook/core/Core", - "com/facebook/CurrentAccessTokenExpirationBroadcastReceiver", - "com/facebook/CustomTabActivity", - "com/facebook/CustomTabMainActivity", - "com/facebook/devicerequests/internal/DeviceRequestsHelper", - "com/facebook/FacebookActivity", - "com/facebook/FacebookAuthorizationException", - "com/facebook/FacebookBroadcastReceiver", - "com/facebook/FacebookButtonBase", - "com/facebook/FacebookCallback", - "com/facebook/FacebookContentProvider", - "com/facebook/FacebookDialog", - "com/facebook/FacebookDialogException", - "com/facebook/FacebookException", - "com/facebook/FacebookGraphResponseException", - "com/facebook/FacebookOperationCanceledException", - "com/facebook/FacebookRequestError", - "com/facebook/FacebookSdk", - "com/facebook/FacebookSdkNotInitializedException", - "com/facebook/FacebookSdkVersion", - "com/facebook/FacebookServiceException", - "com/facebook/gamingservices/cloudgaming/AppToUserNotificationSender", - "com/facebook/gamingservices/cloudgaming/CloudGameLoginHandler", - "com/facebook/gamingservices/cloudgaming/DaemonReceiver", - "com/facebook/gamingservices/cloudgaming/DaemonRequest", - "com/facebook/gamingservices/cloudgaming/GameFeaturesLibrary", - "com/facebook/gamingservices/cloudgaming/InAppAdLibrary", - "com/facebook/gamingservices/cloudgaming/InAppPurchaseLibrary", - "com/facebook/gamingservices/cloudgaming/internal/SDKAnalyticsEvents", - "com/facebook/gamingservices/cloudgaming/internal/SDKConstants", - "com/facebook/gamingservices/cloudgaming/internal/SDKLogger", - "com/facebook/gamingservices/cloudgaming/internal/SDKMessageEnum", - "com/facebook/gamingservices/cloudgaming/internal/SDKShareIntentEnum", - "com/facebook/gamingservices/cloudgaming/PlayableAdsLibrary", - "com/facebook/gamingservices/ContextChooseDialog", - "com/facebook/gamingservices/ContextCreateDialog", - "com/facebook/gamingservices/ContextSwitchDialog", - "com/facebook/gamingservices/CustomUpdate", - "com/facebook/gamingservices/FriendFinderDialog", - "com/facebook/gamingservices/GameRequestDialog", - "com/facebook/gamingservices/GamingContext", - "com/facebook/gamingservices/GamingGroupIntegration", - "com/facebook/gamingservices/GamingImageUploader", - "com/facebook/gamingservices/GamingPayload", - "com/facebook/gamingservices/GamingServices", - "com/facebook/gamingservices/GamingVideoUploader", - "com/facebook/gamingservices/internal/DateFormatter", - "com/facebook/gamingservices/internal/GamingMediaUploader", - "com/facebook/gamingservices/internal/TournamentJoinDialogURIBuilder", - "com/facebook/gamingservices/internal/TournamentScoreType", - "com/facebook/gamingservices/internal/TournamentShareDialogURIBuilder", - "com/facebook/gamingservices/internal/TournamentSortOrder", - "com/facebook/gamingservices/model/ContextChooseContent", - "com/facebook/gamingservices/model/ContextCreateContent", - "com/facebook/gamingservices/model/ContextSwitchContent", - "com/facebook/gamingservices/model/CustomUpdateContent", - "com/facebook/gamingservices/OpenGamingMediaDialog", - "com/facebook/gamingservices/Tournament", - "com/facebook/gamingservices/TournamentConfig", - "com/facebook/gamingservices/TournamentFetcher", - "com/facebook/gamingservices/TournamentJoinDialog", - "com/facebook/gamingservices/TournamentShareDialog", - "com/facebook/gamingservices/TournamentUpdater", - "com/facebook/GraphRequest", - "com/facebook/GraphRequestAsyncTask", - "com/facebook/GraphRequestBatch", - "com/facebook/GraphResponse", - "com/facebook/HttpMethod", - "com/facebook/internal/AnalyticsEvents", - "com/facebook/internal/AppCall", - "com/facebook/internal/AttributionIdentifiers", - "com/facebook/internal/BoltsMeasurementEventListener", - "com/facebook/internal/BundleJSONConverter", - "com/facebook/internal/CallbackManagerImpl", - "com/facebook/internal/CollectionMapper", - "com/facebook/internal/CustomTab", - "com/facebook/internal/CustomTabUtils", - "com/facebook/internal/DialogFeature", - "com/facebook/internal/DialogPresenter", - "com/facebook/internal/FacebookDialogBase", - "com/facebook/internal/FacebookDialogFragment", - "com/facebook/internal/FacebookGamingAction", - "com/facebook/internal/FacebookInitProvider", - "com/facebook/internal/FacebookRequestErrorClassification", - "com/facebook/internal/FacebookSignatureValidator", - "com/facebook/internal/FacebookWebFallbackDialog", - "com/facebook/internal/FeatureManager", - "com/facebook/internal/FetchedAppGateKeepersManager", - "com/facebook/internal/FetchedAppSettings", - "com/facebook/internal/FetchedAppSettingsManager", - "com/facebook/internal/FileLruCache", - "com/facebook/internal/FragmentWrapper", - "com/facebook/internal/gatekeeper/GateKeeper", - "com/facebook/internal/gatekeeper/GateKeeperRuntimeCache", - "com/facebook/internal/ImageDownloader", - "com/facebook/internal/ImageRequest", - "com/facebook/internal/ImageResponse", - "com/facebook/internal/ImageResponseCache", - "com/facebook/internal/InstagramCustomTab", - "com/facebook/internal/InstallReferrerUtil", - "com/facebook/internal/instrument/anrreport/ANRDetector", - "com/facebook/internal/instrument/anrreport/ANRHandler", - "com/facebook/internal/instrument/crashreport/CrashHandler", - "com/facebook/internal/instrument/crashshield/AutoHandleExceptions", - "com/facebook/internal/instrument/crashshield/CrashShieldHandler", - "com/facebook/internal/instrument/crashshield/NoAutoExceptionHandling", - "com/facebook/internal/instrument/errorreport/ErrorReportData", - "com/facebook/internal/instrument/errorreport/ErrorReportHandler", - "com/facebook/internal/instrument/ExceptionAnalyzer", - "com/facebook/internal/instrument/InstrumentData", - "com/facebook/internal/instrument/InstrumentManager", - "com/facebook/internal/instrument/InstrumentUtility", - "com/facebook/internal/instrument/threadcheck/ThreadCheckHandler", - "com/facebook/internal/InternalSettings", - "com/facebook/internal/LockOnGetVariable", - "com/facebook/internal/Logger", - "com/facebook/internal/logging/dumpsys/EndToEndDumper", - "com/facebook/internal/Mutable", - "com/facebook/internal/NativeAppCallAttachmentStore", - "com/facebook/internal/NativeProtocol", - "com/facebook/internal/PlatformServiceClient", - "com/facebook/internal/ProfileInformationCache", - "com/facebook/internal/qualityvalidation/Excuse", - "com/facebook/internal/qualityvalidation/ExcusesForDesignViolations", - "com/facebook/internal/security/CertificateUtil", - "com/facebook/internal/security/OidcSecurityUtil", - "com/facebook/internal/ServerProtocol", - "com/facebook/internal/SmartLoginOption", - "com/facebook/internal/UrlRedirectCache", - "com/facebook/internal/Utility", - "com/facebook/internal/Validate", - "com/facebook/internal/WebDialog", - "com/facebook/internal/WorkQueue", - "com/facebook/LegacyTokenHelper", - "com/facebook/LoggingBehavior", - "com/facebook/login/CodeChallengeMethod", - "com/facebook/login/CustomTabLoginMethodHandler", - "com/facebook/login/CustomTabPrefetchHelper", - "com/facebook/login/DefaultAudience", - "com/facebook/login/DeviceAuthDialog", - "com/facebook/login/DeviceAuthMethodHandler", - "com/facebook/login/DeviceLoginManager", - "com/facebook/login/GetTokenClient", - "com/facebook/login/GetTokenLoginMethodHandler", - "com/facebook/login/InstagramAppLoginMethodHandler", - "com/facebook/login/KatanaProxyLoginMethodHandler", - "com/facebook/login/Login", - "com/facebook/login/LoginBehavior", - "com/facebook/login/LoginClient", - "com/facebook/login/LoginConfiguration", - "com/facebook/login/LoginFragment", - "com/facebook/login/LoginLogger", - "com/facebook/login/LoginManager", - "com/facebook/login/LoginMethodHandler", - "com/facebook/login/LoginResult", - "com/facebook/login/LoginStatusClient", - "com/facebook/login/LoginTargetApp", - "com/facebook/login/NativeAppLoginMethodHandler", - "com/facebook/login/NonceUtil", - "com/facebook/login/PKCEUtil", - "com/facebook/login/StartActivityDelegate", - "com/facebook/LoginStatusCallback", - "com/facebook/login/WebLoginMethodHandler", - "com/facebook/login/WebViewLoginMethodHandler", - "com/facebook/login/widget/DeviceLoginButton", - "com/facebook/login/widget/LoginButton", - "com/facebook/login/widget/ProfilePictureView", - "com/facebook/login/widget/ToolTipPopup", - "com/facebook/messenger/Messenger", - "com/facebook/messenger/MessengerThreadParams", - "com/facebook/messenger/MessengerUtils", - "com/facebook/messenger/ShareToMessengerParams", - "com/facebook/messenger/ShareToMessengerParamsBuilder", - "com/facebook/Profile", - "com/facebook/ProfileCache", - "com/facebook/ProfileManager", - "com/facebook/ProfileTracker", - "com/facebook/ProgressNoopOutputStream", - "com/facebook/ProgressOutputStream", - "com/facebook/RequestOutputStream", - "com/facebook/RequestProgress", - "com/facebook/share/internal/CameraEffectFeature", - "com/facebook/share/internal/CameraEffectJSONUtility", - "com/facebook/share/internal/GameRequestValidation", - "com/facebook/share/internal/LegacyNativeDialogParameters", - "com/facebook/share/internal/MessageDialogFeature", - "com/facebook/share/internal/NativeDialogParameters", - "com/facebook/share/internal/ResultProcessor", - "com/facebook/share/internal/ShareConstants", - "com/facebook/share/internal/ShareContentValidation", - "com/facebook/share/internal/ShareDialogFeature", - "com/facebook/share/internal/ShareFeedContent", - "com/facebook/share/internal/ShareInternalUtility", - "com/facebook/share/internal/ShareStoryFeature", - "com/facebook/share/internal/VideoUploader", - "com/facebook/share/internal/WebDialogParameters", - "com/facebook/share/model/AppGroupCreationContent", - "com/facebook/share/model/CameraEffectArguments", - "com/facebook/share/model/CameraEffectTextures", - "com/facebook/share/model/GameRequestContent", - "com/facebook/share/model/ShareCameraEffectContent", - "com/facebook/share/model/ShareContent", - "com/facebook/share/model/ShareHashtag", - "com/facebook/share/model/ShareLinkContent", - "com/facebook/share/model/ShareMedia", - "com/facebook/share/model/ShareMediaContent", - "com/facebook/share/model/ShareMessengerActionButton", - "com/facebook/share/model/ShareMessengerURLActionButton", - "com/facebook/share/model/ShareModel", - "com/facebook/share/model/ShareModelBuilder", - "com/facebook/share/model/SharePhoto", - "com/facebook/share/model/SharePhotoContent", - "com/facebook/share/model/ShareStoryContent", - "com/facebook/share/model/ShareVideo", - "com/facebook/share/model/ShareVideoContent", - "com/facebook/share/Share", - "com/facebook/share/ShareApi", - "com/facebook/share/ShareBuilder", - "com/facebook/share/Sharer", - "com/facebook/share/widget/GameRequestDialog", - "com/facebook/share/widget/MessageDialog", - "com/facebook/share/widget/SendButton", - "com/facebook/share/widget/ShareButton", - "com/facebook/share/widget/ShareButtonBase", - "com/facebook/share/widget/ShareDialog", - "com/facebook/UserSettingsManager", - "com/facebook/WebDialog" - ], - "documentation": [ - "https://developers.facebook.com/docs/android" - ], - "gradle_signatures": [ - "com.facebook.android" - ], - "license": "NonFree", - "name": "Facebook Android SDK" - }, - "com.flurry.android": { - "code_signature": [ - "com/flurry" - ], - "documentation": [ - "https://www.flurry.com/" - ], - "gradle_signatures": [ - "com.flurry.android" - ], - "license": "NonFree", - "name": "Flurry Android SDK" - }, - "com.garmin.android.connectiq": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/garmin/android/apps/connectmobile/connectiq" - ], - "description": "SDK to build unique wearable experiences leveraging Garmin device sensors and features.", - "license": "NonFree" - }, - "com.garmin.connectiq": { - "code_signatures": [ - "com/garmin/android/connectiq" - ], - "documentation": [ - "https://developer.garmin.com/connect-iq/core-topics/mobile-sdk-for-android/" - ], - "gradle_signatures": [ - "com.garmin.connectiq:ciq-companion-app-sdk" - ], - "license": "NonFree", - "name": "Connect IQ Mobile SDK for Android" - }, - "com.garmin.fit": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/garmin/fit" - ], - "description": "SDK to access the Garmin Fit.", - "license": "NonFree" - }, - "com.geetest": { - "code_signatures": [ - "com/geetest" - ], - "documentation": [ - "https://docs.geetest.com/" - ], - "gradle_signatures": [ - "com.geetest" - ], - "license": "NonFree", - "name": "GeeTest" - }, - "com.github.junrar": { - "code_signatures": [ - "com/github/junrar" - ], - "documentation": [ - "https://github.com/junrar/junrar" - ], - "gradle_signatures": [ - "com.github.junrar:junrar" - ], - "license": "NonFree", - "name": "Junrar" - }, - "com.github.omicronapps.7-Zip-JBinding-4Android": { - "documentation": [ - "https://github.com/omicronapps/7-Zip-JBinding-4Android" - ], - "gradle_signatures": [ - "com.github.omicronapps:7-Zip-JBinding-4Android" - ], - "license": "NonFree", - "name": "7-Zip-JBinding-4Android" - }, - "com.google.ads": { - "code_signatures": [ - "com/google/ads" - ], - "documentation": [ - "https://developers.google.com/interactive-media-ads/docs/sdks/android/client-side" - ], - "gradle_signatures": [ - "com.google.ads", - "com.google.android.exoplayer:extension-ima", - "androidx.media3:media3-exoplayer-ima" - ], - "license": "NonFree", - "name": "IMA SDK for Android" - }, - "com.google.android.apps.auto.sdk": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/google/android/apps/auto/sdk" - ], - "description": "Framework to develop apps for Android Auto", - "license": "NonFree" - }, - "com.google.android.gcm": { - "anti_features": [ - "NonFreeComp", - "NonFreeNet" - ], - "code_signatures": [ - "com/google/android/gcm" - ], - "description": "Google Cloud Messaging is a mobile notification service developed by Google that enables third-party application developers to send notification data or information from developer-run servers to app.", - "license": "NonFree" - }, - "com.google.android.gms": { - "code_signatures": [ - "com/google/android/gms" - ], - "documentation": [ - "https://www.android.com/gms/" - ], - "gradle_signatures": [ - "com.google.android.gms(?!.(oss-licenses-plugin|strict-version-matcher-plugin))", - "com.google.android.ump", - "androidx.core:core-google-shortcuts", - "androidx.credentials:credentials-play-services-auth", - "androidx.media3:media3-cast", - "androidx.media3:media3-datasource-cronet", - "androidx.wear:wear-remote-interactions", - "androidx.work:work-gcm", - "com.google.android.exoplayer:extension-cast", - "com.google.android.exoplayer:extension-cronet", - "com.evernote:android-job", - "com.cloudinary:cloudinary-android.*:2\\.[12]\\.", - "com.pierfrancescosoffritti.androidyoutubeplayer:chromecast-sender", - "com.yayandroid:locationmanager", - "(?Home channels for mobile apps.", - "license": "NonFree" - }, - "com.google.android.play": { - "anti_features": [ - "NonFreeDep", - "NonFreeNet" - ], - "code_signatures": [ - "com/google/android/play/core" - ], - "documentation": [ - "https://developer.android.com/guide/playcore" - ], - "gradle_signatures": [ - "com.google.android.play:app-update", - "com.google.android.play:asset-delivery", - "com.google.android.play:core.*", - "com.google.android.play:feature-delivery", - "com.google.android.play:review", - "androidx.navigation:navigation-dynamic-features", - "com.github.SanojPunchihewa:InAppUpdater", - "com.suddenh4x.ratingdialog:awesome-app-rating" - ], - "license": "NonFree", - "name": "Google Play Core" - }, - "com.google.android.play.appupdate": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/google/android/play/appupdate" - ], - "description": "manages operations that allow an app to initiate its own updates.", - "license": "NonFree" - }, - "com.google.android.play.integrity": { - "anti_features": [ - "NonFreeComp", - "NonFreeNet" - ], - "code_signatures": [ - "com/google/android/play/integrity" - ], - "description": "helps you check that interactions and server requests are coming from your genuine app binary running on a genuine Android device.", - "license": "NonFree" - }, - "com.google.android.play.review": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/google/android/play/review" - ], - "description": "lets you prompt users to submit Play Store ratings and reviews without the inconvenience of leaving your app or game.", - "license": "NonFree" - }, - "com.google.android.vending": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/google/android/vending/(?!licensing|expansion)" - ], - "description": "the Google Play Store app and its libaries, parts are FOSS and get vendored in libs as they are", - "documentation": [ - "https://github.com/google/play-licensing/tree/master/lvl_library/src/main", - "https://github.com/googlearchive/play-apk-expansion/tree/master/zip_file/src/com/google/android/vending/expansion/zipfile", - "https://github.com/googlearchive/play-apk-expansion/tree/master/apkx_library/src/com/google/android/vending/expansion/downloader" - ], - "license": "NonFree" - }, - "com.google.android.wearable": { - "code_signatures": [ - "com/google/android/wearable/(?!compat/WearableActivityController)" - ], - "description": "an API for the Android Wear platform, note that androidx.wear:wear has a stub https://android.googlesource.com/platform/frameworks/support/+/refs/heads/androidx-master-release/wear/wear/src/androidTest/java/com/google/android/wearable/compat/WearableActivityController.java#26", - "gradle_signatures": [ - "com.google.android.support:wearable", - "com.google.android.wearable:wearable" - ], - "license": "NonFree" - }, - "com.google.android.youtube.player": { - "anti_features": [ - "NonFreeComp", - "NonFreeNet" - ], - "code_signatures": [ - "com/google/android/youtube/player" - ], - "description": "enables you to easily play YouTube videos and display thumbnails of YouTube videos in your Android application.", - "license": "NonFree" - }, - "com.google.mlkit": { - "code_signatures": [ - "com/google/mlkit" - ], - "documentation": [ - "https://developers.google.com/ml-kit" - ], - "gradle_signatures": [ - "com.google.mlkit", - "io.github.g00fy2.quickie" - ], - "license": "NonFree", - "name": "ML Kit" - }, - "com.google.vr": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/google/vr" - ], - "description": "enables Daydream and Cardboard app development on Android.", - "license": "NonFree" - }, - "com.heapanalytics": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/heapanalytics" - ], - "description": "automatically captures every web, mobile, and cloud interaction: clicks, submits, transactions, emails, and more. Retroactively analyze your data without writing code.", - "license": "NonFree" - }, - "com.heyzap": { - "code_signatures": [ - "com/heyzap" - ], - "documentation": [ - "https://www.digitalturbine.com/" - ], - "license": "NonFree", - "name": "Heyzap" - }, - "com.huawei.hms": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/huawei/hms" - ], - "description": "Huawei's pendant to GMS (Google Mobile Services)", - "license": "NonFree" - }, - "com.hypertrack": { - "code_signatures": [ - "com/hypertrack/(?!hyperlog)" - ], - "documentation": [ - "https://github.com/hypertrack/sdk-android" - ], - "gradle_signatures": [ - "com.hypertrack(?!:hyperlog)" - ], - "gradle_signatures_negative_examples": [ - "com.hypertrack:hyperlog" - ], - "license": "NonFree", - "name": "HyperTrack SDK for Android" - }, - "com.instabug": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/instabug" - ], - "description": "In-App Feedback and Bug Reporting for Mobile Apps.", - "license": "NonFree" - }, - "com.kiddoware.kidsplace.sdk": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/kiddoware/kidsplace/sdk" - ], - "description": "parental control", - "license": "NonFree" - }, - "com.kochava.android.tracker": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/kochava/android/tracker" - ], - "description": "provides holistic, unbiased measurement for precise, real-time visualization of app performance through the funnel. See Crunchbase and Exodus Privacy.", - "license": "NonFree" - }, - "com.mapbox": { - "MaintainerNotes": "It seems that all libs in https://github.com/mapbox/mapbox-java is fully FOSS\nsince 3.0.0.\n", - "documentation": [ - "https://docs.mapbox.com/android/java/overview/", - "https://github.com/mapbox/mapbox-java" - ], - "gradle_signatures": [ - "com\\.mapbox(?!\\.mapboxsdk:mapbox-sdk-(services|geojson|turf):([3-5]))" - ], - "gradle_signatures_negative_examples": [ - "com.mapbox.mapboxsdk:mapbox-sdk-services:5.0.0", - "com.github.johan12345:mapbox-events-android:a21c324501", - "implementation(\"com.github.johan12345.AnyMaps:anymaps-mapbox:$anyMapsVersion\")" - ], - "gradle_signatures_positive_examples": [ - "com.mapbox.mapboxsdk:mapbox-android-plugin-annotation-v7:0.6.0", - "com.mapbox.mapboxsdk:mapbox-android-plugin-annotation-v8:0.7.0", - "com.mapbox.mapboxsdk:mapbox-android-plugin-localization-v7:0.7.0", - "com.mapbox.mapboxsdk:mapbox-android-plugin-locationlayer:0.4.0", - "com.mapbox.mapboxsdk:mapbox-android-plugin-markerview-v8:0.3.0", - "com.mapbox.mapboxsdk:mapbox-android-plugin-places-v8:0.9.0", - "com.mapbox.mapboxsdk:mapbox-android-plugin-scalebar-v8:0.2.0", - "com.mapbox.mapboxsdk:mapbox-android-sdk:7.3.0" - ], - "license": "NonFree", - "name": "Mapbox Java SDK" - }, - "com.microblink": { - "anti_features": [ - "NonFreeComp", - "NonFreeNet", - "Tracking" - ], - "code_signatures": [ - "com/microblink" - ], - "description": "verify users at scale and automate your document-based workflow with computer vision tech built for a remote world.", - "license": "NonFree" - }, - "com.microsoft.band": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/microsoft/band" - ], - "description": "library to access the Microsoft Band smartwatch.", - "license": "NonFree" - }, - "com.mopub.mobileads": { - "anti_features": [ - "Ads", - "NonFreeComp" - ], - "code_signatures": [ - "com/mopub/mobileads" - ], - "description": "ad framework run by Twitter until 1/2022, then sold to AppLovin.", - "license": "NonFree" - }, - "com.newrelic.agent": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/newrelic/agent" - ], - "description": "delivering full-stack visibility and analytics to enterprises around the world. See Crunchbase and Exodus Privacy.", - "license": "NonFree" - }, - "com.onesignal": { - "code_signatures": [ - "com/onesignal" - ], - "documentation": [ - "https://github.com/OneSignal/OneSignal-Android-SDK" - ], - "gradle_signatures": [ - "com.onesignal:OneSignal" - ], - "license": "NonFree", - "name": "OneSignal Android Push Notification Plugin" - }, - "com.optimizely": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/optimizely" - ], - "description": "part of the comScore, Inc. market research community, a leading global market research effort that studies and reports on Internet trends and behavior.", - "license": "NonFree" - }, - "com.paypal.sdk": { - "code_signatures": [ - "com/paypal" - ], - "documentation": [ - "https://github.com/paypal/PayPal-Android-SDK", - "https://github.com/paypal/android-checkout-sdk" - ], - "gradle_signatures": [ - "com.paypal" - ], - "license": "NonFree", - "name": "PayPal Android SDK" - }, - "com.pushwoosh": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/pushwoosh" - ], - "description": "mobile analytics under the cover of push messaging.", - "license": "NonFree" - }, - "com.quantcast.measurement.service": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/quantcast/measurement/service" - ], - "description": "processes real-time data at the intersection of commerce and culture, providing useful, actionable insights for brands and publishers. See Crunchbase and Exodus Privacy.", - "license": "NonFree" - }, - "com.revenuecat.purchases": { - "code_signatures": [ - "com/revenuecat/purchases" - ], - "documentation": [ - "https://www.revenuecat.com/" - ], - "gradle_signatures": [ - "com.revenuecat.purchases" - ], - "license": "NonFree", - "name": "RevenueCat Purchases" - }, - "com.samsung.accessory": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/samsung/accessory" - ], - "description": "provides a stable environment in which you can use a variety features by connecting accessories to your mobile device.", - "license": "NonFree" - }, - "com.samsung.android.sdk.look": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/samsung/android/sdk/look" - ], - "description": "offers specialized widgets and service components for extended functions of the Samsung Android devices.", - "license": "NonFree" - }, - "com.sendbird.android": { - "anti_features": [ - "NonFreeComp", - "NonFreeNet", - "Tracking" - ], - "code_signatures": [ - "com/sendbird/android" - ], - "description": "an easy-to-use Chat API, native Chat SDKs, and a fully-managed chat platform on the backend means faster time-to-market.", - "license": "NonFree" - }, - "com.smaato.soma": { - "anti_features": [ - "Ads", - "NonFreeComp" - ], - "code_signatures": [ - "com/smaato/soma" - ], - "description": "a mobile ad platform that includes video ads.", - "license": "NonFree" - }, - "com.spotify.sdk": { - "anti_features": [ - "NonFreeComp", - "NonFreeNet" - ], - "code_signatures": [ - "com/spotify/sdk" - ], - "description": "allows your application to interact with the Spotify app service. (Note that while the SDK repo claims Apache license, the code is not available there)", - "license": "NonFree" - }, - "com.startapp.android": { - "anti_features": [ - "Ads", - "Tracking", - "NonFreeComp" - ], - "code_signatures": [ - "com/startapp" - ], - "description": "partly quite intrusive ad network.", - "license": "NonFree" - }, - "com.telerik.android": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "com/telerik/android" - ], - "description": "offers high quality Xamarin Forms UI components and Visual Studio item templates to enable every developer.", - "license": "NonFree" - }, - "com.tencent.bugly": { - "code_signatures": [ - "com/tencent/bugly" - ], - "documentation": [ - "https://bugly.qq.com/" - ], - "gradle_signatures": [ - "com.tencent.bugly" - ], - "license": "NonFree", - "name": "Bugly Android SDK" - }, - "com.tencent.mapsdk": { - "anti_features": [ - "NonFreeNet" - ], - "code_signatures": [ - "com/tencent/tencentmap" - ], - "description": "giving access to Tencent Maps.", - "license": "NonFree" - }, - "com.tenjin.android.TenjinSDK": { - "anti_features": [ - "Tracking" - ], - "code_signatures": [ - "com/tenjin/android/TenjinSDK" - ], - "description": "a marketing platform designed for mobile that features analytics, automated aggregation, and direct data visualization with direct SQL access.", - "license": "NonFree" - }, - "com.umeng.umsdk": { - "code_signatures": [ - "com/umeng" - ], - "documentation": [ - "https://developer.umeng.com/docs/119267/detail/118584" - ], - "gradle_signatures": [ - "com.umeng" - ], - "license": "NonFree", - "name": "Umeng SDK" - }, - "com.wei.android.lib": { - "code_signatures": [ - "com/wei/android/lib/fingerprintidentify" - ], - "documentation": [ - "https://github.com/uccmawei/FingerprintIdentify" - ], - "gradle_signatures": [ - "com.wei.android.lib:fingerprintidentify", - "com.github.uccmawei:FingerprintIdentify" - ], - "gradle_signatures_positive_examples": [ - "implementation \"com.github.uccmawei:fingerprintidentify:${safeExtGet(\"fingerprintidentify\", \"1.2.6\")}\"" - ], - "license": "NonFree", - "name": "FingerprintIdentify" - }, - "com.yandex.android": { - "code_signatures": [ - "com/yandex/android/(?!:authsdk)" - ], - "gradle_signatures": [ - "com\\.yandex\\.android(?!:authsdk)" - ], - "gradle_signatures_negative_examples": [ - "com.yandex.android:authsdk" - ], - "license": "NonFree", - "name": "Yandex SDK" - }, - "com.yandex.metrica": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "com/yandex/metrica" - ], - "description": "a mobile attribution and analytics platform developed by Yandex. It is free, real-time and has no data limits restriction. See Crunchbase and Exodus Privacy.", - "license": "NonFree" - }, - "com.yandex.mobile.ads": { - "anti_features": [ - "Ads", - "NonFreeComp" - ], - "code_signatures": [ - "com/yandex/mobile/ads" - ], - "description": "See Exodus Privacy.", - "license": "NonFree" - }, - "de.epgpaid": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "de/epgpaid" - ], - "description": "access paid EPG (Electronic Program Guide, for TV) data (after payment, of course). Part of TVBrowser.", - "license": "NonFree" - }, - "de.innosystec.unrar": { - "code_signatures": [ - "de/innosystec/unrar" - ], - "description": "java unrar util", - "license": "NonFree" - }, - "firebase": { - "code_signatures": [ - "com/google/firebase" - ], - "documentation": [ - "https://www.firebase.com" - ], - "gradle_signatures": [ - "com(\\.google)?\\.firebase[.:](?!firebase-jobdispatcher|geofire-java)", - "com.microsoft.appcenter:appcenter-push" - ], - "gradle_signatures_negative_examples": [ - " compile 'com.firebase:firebase-jobdispatcher:0.8.4'", - "implementation 'com.firebase:geofire-java:3.0.0'", - " compile 'com.firebaseui:firebase-ui-auth:3.1.3'", - "com.firebaseui:firebase-ui-database", - "com.firebaseui:firebase-ui-storage", - "com.github.axet:android-firebase-fake", - "com.github.b3er.rxfirebase:firebase-database", - "com.github.b3er.rxfirebase:firebase-database-kotlin", - "com.segment.analytics.android.integrations:firebase" - ], - "gradle_signatures_positive_examples": [ - "\tcompile 'com.google.firebase:firebase-crash:11.0.8'", - "\tcompile 'com.google.firebase:firebase-core:11.0.8'", - "com.firebase:firebase-client-android:2.5.2", - "com.google.firebase.crashlytics", - "com.google.firebase.firebase-perf", - "com.google.firebase:firebase-ads", - "com.google.firebase:firebase-analytics", - "com.google.firebase:firebase-appindexing", - "com.google.firebase:firebase-auth", - "com.google.firebase:firebase-config", - "com.google.firebase:firebase-core", - "com.google.firebase:firebase-crash", - "com.google.firebase:firebase-crashlytics", - "com.google.firebase:firebase-database", - "com.google.firebase:firebase-dynamic-links", - "com.google.firebase:firebase-firestore", - "com.google.firebase:firebase-inappmessaging", - "com.google.firebase:firebase-inappmessaging-display", - "com.google.firebase:firebase-messaging", - "com.google.firebase:firebase-ml-natural-language", - "com.google.firebase:firebase-ml-natural-language-smart-reply-model", - "com.google.firebase:firebase-ml-vision", - "com.google.firebase:firebase-perf", - "com.google.firebase:firebase-plugins", - "com.google.firebase:firebase-storage" - ], - "license": "NonFree", - "name": "Firebase" - }, - "google-maps": { - "anti_features": [ - "NonFreeDep", - "NonFreeNet" - ], - "api_key_ids": [ - "com\\.google\\.android\\.geo\\.API_KEY", - "com\\.google\\.android\\.maps\\.v2\\.API_KEY" - ], - "documentation": [ - "https://developers.google.com/maps/documentation/android-sdk/overview" - ], - "license": "NonFree", - "name": "Google Maps" - }, - "io.fabric.sdk.android": { - "anti_features": [ - "NonFreeComp", - "Tracking" - ], - "code_signatures": [ - "io/fabric/sdk/android" - ], - "description": "Framework to integrate services. Provides e.g. crash reports and analytics. Aquired by Google in 2017.", - "license": "NonFree" - }, - "io.github.sinaweibosdk": { - "code_signatures": [ - "com/sina" - ], - "documentation": [ - "https://github.com/sinaweibosdk/weibo_android_sdk" - ], - "gradle_signatures": [ - "io.github.sinaweibosdk" - ], - "license": "NonFree", - "name": "SinaWeiboSDK" - }, - "io.intercom": { - "anti_features": [ - "NonFreeComp", - "NonFreeNet" - ], - "code_signatures": [ - "io/intercom" - ], - "description": "engage customers with email, push, and in\u2011app messages and support them with an integrated knowledge base and help desk.", - "license": "NonFree" - }, - "io.objectbox": { - "code_signatures": [ - "io/objectbox" - ], - "documentation": [ - "https://objectbox.io/faq/#license-pricing" - ], - "gradle_signatures": [ - "io.objectbox:objectbox-gradle-plugin" - ], - "license": "NonFree", - "name": "ObjectBox Database" - }, - "me.pushy": { - "code_signatures": [ - "me/pushy" - ], - "documentation": [ - "https://pushy.me/" - ], - "gradle_signatures": [ - "me.pushy" - ], - "license": "NonFree", - "name": "Pushy" - }, - "org.gradle.toolchains.foojay-resolver-convention": { - "documentation": [ - "https://github.com/gradle/foojay-toolchains" - ], - "gradle_signatures": [ - "org.gradle.toolchains.foojay-resolver" - ], - "license": "Apache-2.0", - "name": "Foojay Toolchains Plugin" - }, - "org.mariuszgromada.math": { - "code_signatures": [ - "org/mariuszgromada/math/mxparser/parsertokens/SyntaxStringBuilder", - "org/mariuszgromada/math/mxparser/CalcStepRecord", - "org/mariuszgromada/math/mxparser/CalcStepsRegister", - "org/mariuszgromada/math/mxparser/License", - "org/mariuszgromada/math/mxparser/CloneCache", - "org/mariuszgromada/math/mxparser/ElementAtTheEnd", - "org/mariuszgromada/math/mxparser/CompilationDetails", - "org/mariuszgromada/math/mxparser/CompiledElement" - ], - "documentation": [ - "https://mathparser.org", - "https://mathparser.org/mxparser-license/" - ], - "gradle_signatures": [ - "org.mariuszgromada.math:MathParser.org-mXparser:[5-9]" - ], - "license": "NonFree", - "name": "mXparser" - }, - "tornaco.android.sec": { - "anti_features": [ - "NonFreeComp" - ], - "code_signatures": [ - "tornaco/android/sec" - ], - "description": "proprietary part of the Thanox application", - "license": "NonFree" - } - }, - "timestamp": 1747829076.702502, - "version": 1, - "last_updated": 1750710966.431471 -}''' diff --git a/fdroidserver/signatures.py b/fdroidserver/signatures.py index 00c9d264..78f4bcd3 100644 --- a/fdroidserver/signatures.py +++ b/fdroidserver/signatures.py @@ -15,17 +15,21 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import logging -import os -import re -import sys from argparse import ArgumentParser -from . import _, common +import re +import os +import sys +import logging + +from . import _ +from . import common +from . import net from .exception import FDroidException def extract_signature(apkpath): + if not os.path.exists(apkpath): raise FDroidException("file APK does not exists '{}'".format(apkpath)) if not common.verify_apk_signature(apkpath): @@ -42,6 +46,7 @@ def extract_signature(apkpath): def extract(options): + # Create tmp dir if missing… tmp_dir = 'tmp' if not os.path.exists(tmp_dir): @@ -57,40 +62,26 @@ def extract(options): try: if os.path.isfile(apk): sigdir = extract_signature(apk) - logging.info( - _("Fetched signatures for '{apkfilename}' -> '{sigdir}'").format( - apkfilename=apk, sigdir=sigdir - ) - ) + logging.info(_("Fetched signatures for '{apkfilename}' -> '{sigdir}'") + .format(apkfilename=apk, sigdir=sigdir)) elif httpre.match(apk): if apk.startswith('https') or options.no_check_https: try: - from . import net - tmp_apk = os.path.join(tmp_dir, 'signed.apk') net.download_file(apk, tmp_apk) sigdir = extract_signature(tmp_apk) - logging.info( - _( - "Fetched signatures for '{apkfilename}' -> '{sigdir}'" - ).format(apkfilename=apk, sigdir=sigdir) - ) + logging.info(_("Fetched signatures for '{apkfilename}' -> '{sigdir}'") + .format(apkfilename=apk, sigdir=sigdir)) finally: if tmp_apk and os.path.exists(tmp_apk): os.remove(tmp_apk) else: - logging.warning( - _( - 'refuse downloading via insecure HTTP connection ' - '(use HTTPS or specify --no-https-check): {apkfilename}' - ).format(apkfilename=apk) - ) + logging.warning(_('refuse downloading via insecure HTTP connection ' + '(use HTTPS or specify --no-https-check): {apkfilename}') + .format(apkfilename=apk)) except FDroidException as e: - logging.warning( - _("Failed fetching signatures for '{apkfilename}': {error}").format( - apkfilename=apk, error=e - ) - ) + logging.warning(_("Failed fetching signatures for '{apkfilename}': {error}") + .format(apkfilename=apk, error=e)) if e.detail: logging.debug(e.detail) @@ -98,12 +89,12 @@ def extract(options): def main(): parser = ArgumentParser() common.setup_global_opts(parser) - parser.add_argument( - "APK", nargs='*', help=_("signed APK, either a file-path or HTTPS URL.") - ) + parser.add_argument("APK", nargs='*', + help=_("signed APK, either a file-path or HTTPS URL.")) parser.add_argument("--no-check-https", action="store_true", default=False) - options = common.parse_args(parser) - common.set_console_logging(options.verbose, options.color) - common.read_config() + options = parser.parse_args() + + # Read config.py... + common.read_config(options) extract(options) diff --git a/fdroidserver/signindex.py b/fdroidserver/signindex.py index 47cd5ec2..2c5859d5 100644 --- a/fdroidserver/signindex.py +++ b/fdroidserver/signindex.py @@ -16,154 +16,70 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import json -import logging import os import time import zipfile from argparse import ArgumentParser +import logging -from . import _, common, metadata +from . import _ +from . import common from .exception import FDroidException config = None +options = None start_timestamp = time.gmtime() -def sign_jar(jar, use_old_algs=False): - """Sign a JAR file with the best available algorithm. - - The current signing method uses apksigner to sign the JAR so that - it will automatically select algorithms that are compatible with - Android SDK 23, which added the most recent algorithms: - https://developer.android.com/reference/java/security/Signature - - This signing method uses then inherits the default signing - algothim settings, since Java and Android both maintain those. - That helps avoid a repeat of being stuck on an old signing - algorithm. That means specifically that this call to apksigner - does not specify any of the algorithms. - - The old indexes must be signed by SHA1withRSA otherwise they will - no longer be compatible with old Androids. +def sign_jar(jar): + """ + Sign a JAR file with Java's jarsigner. This method requires a properly initialized config object. + This does use old hashing algorithms, i.e. SHA1, but that's not + broken yet for file verification. This could be set to SHA256, + but then Android < 4.3 would not be able to verify it. + https://code.google.com/p/android/issues/detail?id=38321 """ - if use_old_algs: - # This does use old hashing algorithms, i.e. SHA1, but that's not - # broken yet for file verification. This could be set to SHA256, - # but then Android < 4.3 would not be able to verify it. - # https://code.google.com/p/android/issues/detail?id=38321 - args = [ - config['jarsigner'], - '-keystore', - config['keystore'], - '-storepass:env', - 'FDROID_KEY_STORE_PASS', - '-digestalg', - 'SHA1', - '-sigalg', - 'SHA1withRSA', - jar, - config['repo_keyalias'], - ] - if config['keystore'] == 'NONE': - args += config['smartcardoptions'] - else: # smardcards never use -keypass - args += ['-keypass:env', 'FDROID_KEY_PASS'] - else: - # https://developer.android.com/studio/command-line/apksigner - args = [ - config['apksigner'], - 'sign', - '--min-sdk-version', - '23', # enable all current algorithms - '--max-sdk-version', - '24', # avoid future incompatible algorithms - # disable all APK signature types, only use JAR sigs aka v1 - '--v1-signing-enabled', - 'true', - '--v2-signing-enabled', - 'false', - '--v3-signing-enabled', - 'false', - '--v4-signing-enabled', - 'false', - '--ks', - config['keystore'], - '--ks-pass', - 'env:FDROID_KEY_STORE_PASS', - '--ks-key-alias', - config['repo_keyalias'], - ] - if config['keystore'] == 'NONE': - args += common.get_apksigner_smartcardoptions(config['smartcardoptions']) - else: # smardcards never use --key-pass - args += ['--key-pass', 'env:FDROID_KEY_PASS'] - args += [jar] + args = [config['jarsigner'], '-keystore', config['keystore'], + '-storepass:env', 'FDROID_KEY_STORE_PASS', + '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA', + jar, config['repo_keyalias']] + if config['keystore'] == 'NONE': + args += config['smartcardoptions'] + else: # smardcards never use -keypass + args += ['-keypass:env', 'FDROID_KEY_PASS'] env_vars = { 'FDROID_KEY_STORE_PASS': config['keystorepass'], 'FDROID_KEY_PASS': config.get('keypass', ""), } p = common.FDroidPopen(args, envs=env_vars) - if not use_old_algs and p.returncode != 0: - # workaround for apksigner v30 on f-droid.org publish server - v4 = args.index("--v4-signing-enabled") - del args[v4 + 1] - del args[v4] - p = common.FDroidPopen(args, envs=env_vars) - if p.returncode != 0: - raise FDroidException("Failed to sign %s: %s" % (jar, p.output)) + if p.returncode != 0: + raise FDroidException("Failed to sign %s!" % jar) -def sign_index(repodir, json_name): - """Sign data file like entry.json to make a signed JAR like entry.jar. - - The data file like index-v1.json means that there is unsigned - data. That file is then stuck into a jar and signed by the - signing process. This is a bit different than sign_jar, which is - used for index.jar: that creates index.xml then puts that in a - index_unsigned.jar, then that file is signed. - - This also checks to make sure that the JSON files are intact - before signing them. Broken JSON files should never be signed, so - taking some extra time and failing hard is the preferred - option. This signing process can happen on an entirely separate - machine and file tree, so this ensures that nothing got broken - during transfer. - +def sign_index_v1(repodir, json_name): """ - json_file = os.path.join(repodir, json_name) - with open(json_file, encoding="utf-8") as fp: - data = json.load(fp) - if json_name == 'entry.json': - index_file = os.path.join(repodir, data['index']['name'].lstrip('/')) - sha256 = common.sha256sum(index_file) - if sha256 != data['index']['sha256']: - raise FDroidException( - _('%s has bad SHA-256: %s') % (index_file, sha256) - ) - with open(index_file) as fp: - index = json.load(fp) - if not isinstance(index, dict): - raise FDroidException(_('%s did not produce a dict!') % index_file) - elif json_name == 'index-v1.json': - [metadata.App(app) for app in data["apps"]] + Sign index-v1.json to make index-v1.jar + This is a bit different than index.jar: instead of their being index.xml + and index_unsigned.jar, the presence of index-v1.json means that there is + unsigned data. That file is then stuck into a jar and signed by the + signing process. index-v1.json is never published to the repo. It is + included in the binary transparency log, if that is enabled. + """ name, ext = common.get_extension(json_name) + index_file = os.path.join(repodir, json_name) jar_file = os.path.join(repodir, name + '.jar') with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar: - jar.write(json_file, json_name) - - if json_name in ('index.xml', 'index-v1.json'): - sign_jar(jar_file, use_old_algs=True) - else: - sign_jar(jar_file) + jar.write(index_file, json_name) + sign_jar(jar_file) def status_update_json(signed): - """Output a JSON file with metadata about this run.""" + """Output a JSON file with metadata about this run""" + logging.debug(_('Outputting JSON')) output = common.setup_status_output(start_timestamp) if signed: @@ -172,20 +88,18 @@ def status_update_json(signed): def main(): - global config + + global config, options parser = ArgumentParser() common.setup_global_opts(parser) - common.parse_args(parser) + options = parser.parse_args() - config = common.read_config() + config = common.read_config(options) if 'jarsigner' not in config: raise FDroidException( - _( - 'Java jarsigner not found! Install in standard location or set java_paths!' - ) - ) + _('Java jarsigner not found! Install in standard location or set java_paths!')) repodirs = ['repo'] if config['archive_older'] != 0: @@ -207,14 +121,8 @@ def main(): json_name = 'index-v1.json' index_file = os.path.join(output_dir, json_name) if os.path.exists(index_file): - sign_index(output_dir, json_name) - logging.info('Signed ' + index_file) - signed.append(index_file) - - json_name = 'entry.json' - index_file = os.path.join(output_dir, json_name) - if os.path.exists(index_file): - sign_index(output_dir, json_name) + sign_index_v1(output_dir, json_name) + os.remove(index_file) logging.info('Signed ' + index_file) signed.append(index_file) diff --git a/fdroidserver/stats.py b/fdroidserver/stats.py new file mode 100644 index 00000000..9d6cbfdc --- /dev/null +++ b/fdroidserver/stats.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +# +# stats.py - part of the FDroid server tools +# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import sys +import os +import re +import time +import traceback +import glob +import json +from argparse import ArgumentParser +import paramiko +import socket +import logging +import subprocess +from collections import Counter + +from . import _ +from . import common +from . import metadata + + +def carbon_send(key, value): + s = socket.socket() + s.connect((config['carbon_host'], config['carbon_port'])) + msg = '%s %d %d\n' % (key, value, int(time.time())) + s.sendall(msg) + s.close() + + +options = None +config = None + + +def most_common_stable(counts): + pairs = [] + for s in counts: + pairs.append((s, counts[s])) + return sorted(pairs, key=lambda t: (-t[1], t[0])) + + +def main(): + + global options, config + + # Parse command line... + parser = ArgumentParser() + common.setup_global_opts(parser) + parser.add_argument("-d", "--download", action="store_true", default=False, + help=_("Download logs we don't have")) + parser.add_argument("--recalc", action="store_true", default=False, + help=_("Recalculate aggregate stats - use when changes " + "have been made that would invalidate old cached data.")) + parser.add_argument("--nologs", action="store_true", default=False, + help=_("Don't do anything logs-related")) + metadata.add_metadata_arguments(parser) + options = parser.parse_args() + metadata.warnings_action = options.W + + config = common.read_config(options) + + if not config['update_stats']: + logging.info("Stats are disabled - set \"update_stats = True\" in your config.py") + sys.exit(1) + + # Get all metadata-defined apps... + allmetaapps = [app for app in metadata.read_metadata().values()] + metaapps = [app for app in allmetaapps if not app.Disabled] + + statsdir = 'stats' + logsdir = os.path.join(statsdir, 'logs') + datadir = os.path.join(statsdir, 'data') + if not os.path.exists(statsdir): + os.mkdir(statsdir) + if not os.path.exists(logsdir): + os.mkdir(logsdir) + if not os.path.exists(datadir): + os.mkdir(datadir) + + if options.download: + # Get any access logs we don't have... + ssh = None + ftp = None + try: + logging.info('Retrieving logs') + ssh = paramiko.SSHClient() + ssh.load_system_host_keys() + ssh.connect(config['stats_server'], username=config['stats_user'], + timeout=10, key_filename=config['webserver_keyfile']) + ftp = ssh.open_sftp() + ftp.get_channel().settimeout(60) + logging.info("...connected") + + ftp.chdir('logs') + files = ftp.listdir() + for f in files: + if f.startswith('access-') and f.endswith('.log.gz'): + + destpath = os.path.join(logsdir, f) + destsize = ftp.stat(f).st_size + if not os.path.exists(destpath) \ + or os.path.getsize(destpath) != destsize: + logging.debug("...retrieving " + f) + ftp.get(f, destpath) + except Exception: + traceback.print_exc() + sys.exit(1) + finally: + # Disconnect + if ftp is not None: + ftp.close() + if ssh is not None: + ssh.close() + + knownapks = common.KnownApks() + unknownapks = [] + + if not options.nologs: + # Process logs + logging.info('Processing logs...') + appscount = Counter() + appsvercount = Counter() + logexpr = r'(?P[.:0-9a-fA-F]+) - - \[(?P