From aa9cc1499109d057100a563610ec5309db599b99 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 12 Mar 2025 17:53:05 +0100 Subject: [PATCH 1/6] gitlab-ci: refactor to rules: and ditch old only: syntax Adding workflow: is required, otherwise there would be duplicate pipelines for all users in the @fdroid group. There would be "branch pipelines" and "merge request pipelines". Confusingly, only jobs with rules: get duplicated. --- .gitlab-ci.yml | 96 ++++++++++++++++++++++++++------------------------ 1 file changed, 49 insertions(+), 47 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f1c04aea..c1c05989 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,16 @@ --- +# Use merge request pipelines when a merge request is open for the branch. +# Use branch pipelines when a merge request is not open for the branch. +# https://docs.gitlab.com/ci/yaml/workflow/#switch-between-branch-pipelines-and-merge-request-pipelines +workflow: + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS + when: never + - if: $CI_COMMIT_BRANCH + + variables: pip: pip3 --timeout 100 --retries 10 # speed up git checkout phase @@ -94,8 +105,8 @@ metadata_v0: debian_testing: image: debian:testing <<: *apt-template - only: - - master@fdroid/fdroidserver + rules: + - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver" script: - apt-get install aapt @@ -123,8 +134,8 @@ debian_testing: ubuntu_lts_ppa: image: ubuntu:latest <<: *apt-template - only: - - master@fdroid/fdroidserver + rules: + - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver" script: - export ANDROID_HOME=/usr/lib/android-sdk - apt-get install gnupg @@ -188,11 +199,11 @@ ubuntu_jammy_pip: gradlew-fdroid: image: debian:bookworm-slim <<: *apt-template - only: - changes: - - .gitlab-ci.yml - - gradlew-fdroid - - tests/test_gradlew-fdroid + rules: + - changes: + - .gitlab-ci.yml + - gradlew-fdroid + - tests/test_gradlew-fdroid script: - apt-get install ca-certificates curl default-jdk-headless shellcheck unzip - shellcheck --severity=error --color gradlew-fdroid tests/test_gradlew-fdroid @@ -255,10 +266,7 @@ lint_format_bandit_checks: safety: image: debian:bookworm-slim rules: - # once only:/changes: are ported to rules:, this could be removed: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: never - - if: $CI_PIPELINE_SOURCE == "push" && $SAFETY_API_KEY + - if: $SAFETY_API_KEY changes: - .gitlab-ci.yml - .safety-policy.yml @@ -283,11 +291,7 @@ safety: yamllint: image: debian:bookworm-slim rules: - # once only:/changes: are ported to rules:, this could be removed: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: never - - if: $CI_PIPELINE_SOURCE == "push" - changes: + - changes: - .gitlab-ci.yml - .safety-policy.yml - .yamllint @@ -308,7 +312,6 @@ yamllint: tests/*/*/.*.yml -# Run all the various linters and static analysis tools. locales: image: debian:bookworm-slim variables: @@ -380,8 +383,8 @@ fedora_latest: macOS: tags: - saas-macos-medium-m1 - only: - - master@fdroid/fdroidserver + rules: + - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver" script: - export HOMEBREW_CURL_RETRIES=10 - brew update > /dev/null @@ -446,16 +449,16 @@ gradle: # Run an actual build in a simple, faked version of the buildserver guest VM. fdroid build: image: registry.gitlab.com/fdroid/fdroidserver:buildserver - only: - changes: - - .gitlab-ci.yml - - fdroidserver/build.py - - fdroidserver/common.py - - fdroidserver/exception.py - - fdroidserver/metadata.py - - fdroidserver/net.py - - fdroidserver/scanner.py - - fdroidserver/vmtools.py + rules: + - changes: + - .gitlab-ci.yml + - fdroidserver/build.py + - fdroidserver/common.py + - fdroidserver/exception.py + - fdroidserver/metadata.py + - fdroidserver/net.py + - fdroidserver/scanner.py + - fdroidserver/vmtools.py cache: key: "$CI_JOB_NAME" paths: @@ -516,11 +519,11 @@ fdroid build: plugin_fetchsrclibs: image: debian:bookworm-slim <<: *apt-template - only: - changes: - - .gitlab-ci.yml - - examples/fdroid_fetchsrclibs.py - - fdroidserver/__main__.py + rules: + - changes: + - .gitlab-ci.yml + - examples/fdroid_fetchsrclibs.py + - fdroidserver/__main__.py script: - apt-get install curl @@ -560,8 +563,8 @@ plugin_fetchsrclibs: servergitmirrors: image: debian:bookworm-slim <<: *apt-template - only: - - master@fdroid/fdroidserver + rules: + - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver" script: - apt-get install default-jdk-headless @@ -622,8 +625,8 @@ Build documentation: Windows: tags: - windows - only: - - windows + rules: + - if: $CI_COMMIT_BRANCH == "windows" script: - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" - choco install --no-progress -y git --force --params "/GitAndUnixToolsOnPath" @@ -686,13 +689,12 @@ pages: docker: dependencies: - fdroid build - only: - changes: - - .gitlab-ci.yml - - makebuildserver - - buildserver/* - variables: - - $CI_COMMIT_BRANCH == "master" || $CI_PROJECT_NAMESPACE != "fdroid" + rules: + - if: $CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "fdroid/fdroidserver" + changes: + - .gitlab-ci.yml + - makebuildserver + - buildserver/* image: docker:dind services: - docker:dind From 3b9d39ac614a31f554b3d7231957740fc5472bec Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 12 Mar 2025 18:21:09 +0100 Subject: [PATCH 2/6] gitlab-ci: move shellcheck to own job fenced by changes: --- .gitlab-ci.yml | 29 ++++++++++++++++++++++++----- hooks/install-hooks.sh | 2 +- hooks/pre-commit | 4 ++-- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c1c05989..0bca13ef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -205,8 +205,7 @@ gradlew-fdroid: - gradlew-fdroid - tests/test_gradlew-fdroid script: - - apt-get install ca-certificates curl default-jdk-headless shellcheck unzip - - shellcheck --severity=error --color gradlew-fdroid tests/test_gradlew-fdroid + - apt-get install ca-certificates curl default-jdk-headless unzip - ./tests/test_gradlew-fdroid @@ -232,7 +231,6 @@ lint_format_bandit_checks: python3-nose python3-pip python3-yaml - shellcheck - $pip install --break-system-packages bandit pylint-gitlab - export EXITVALUE=0 - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; } @@ -250,8 +248,6 @@ lint_format_bandit_checks: tests/*.py > pylint-report.json || set_error - - shellcheck --exclude SC2046,SC2090 --severity=warning --color tests/run-tests - || set_error - exit $EXITVALUE artifacts: reports: @@ -259,6 +255,29 @@ lint_format_bandit_checks: when: always +shellcheck: + image: debian:bookworm-slim + rules: + - changes: + - .gitlab-ci.yml + - gradlew-fdroid + - hooks/install-hooks.sh + - hooks/pre-commit + - tests/run-tests + - tests/test_gradlew-fdroid + <<: *apt-template + script: + - apt-get install shellcheck + # TODO GitLab Code Quality report https://github.com/koalaman/shellcheck/issues/3155 + - shellcheck --exclude SC2046,SC2090 --severity=warning --color + hooks/install-hooks.sh + hooks/pre-commit + tests/run-tests + # TODO make the gradlew things pass the standard above + - shellcheck --severity=error --color + gradlew-fdroid + tests/test_gradlew-fdroid + # Check all the dependencies in Debian to mirror production. CVEs are # generally fixed in the latest versions in pip/pypi.org, so it isn't # so important to scan that kind of install in CI. diff --git a/hooks/install-hooks.sh b/hooks/install-hooks.sh index 69b314d4..e266301b 100755 --- a/hooks/install-hooks.sh +++ b/hooks/install-hooks.sh @@ -2,7 +2,7 @@ # # Install all the client hooks -BASE_DIR="$(cd $(dirname $0); pwd -P)" +BASE_DIR="$(cd $(dirname $0) || exit; pwd -P)" HOOK_NAMES="applypatch-msg pre-applypatch post-applypatch pre-commit prepare-commit-msg commit-msg post-commit pre-rebase post-checkout post-merge pre-receive update post-receive post-update pre-auto-gc" HOOK_DIR="$(git rev-parse --show-toplevel)/.git/hooks" diff --git a/hooks/pre-commit b/hooks/pre-commit index c0859570..039c1b65 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -36,7 +36,7 @@ else *.rb) RB_FILES+=" $f" ;; - *.yml|.*.yml|.yamllint) + *.yml|*.yaml|.yamllint) YML_FILES+=" $f" ;; *) @@ -66,7 +66,7 @@ cmd_exists() { } find_command() { - for name in $@; do + for name in "$@"; do for suff in "3" "-3" "-python3" ""; do cmd=${name}${suff} if cmd_exists $cmd; then From 3e67bee0378a60a260645bc4018f88aa45422d73 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 26 Mar 2025 22:19:15 +0100 Subject: [PATCH 3/6] gitlab-ci: move pylint to its own job --- .gitlab-ci.yml | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0bca13ef..c058e9a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -97,6 +97,17 @@ metadata_v0: - rm /etc/apt/apt.conf.d/99nocacertificates - apt-get dist-upgrade +# For jobs that only need to run when there are changes to Python files. +.python-rules-changes: &python-rules-changes + rules: + - changes: + - .gitlab-ci.yml + - fdroid + - makebuildserver + - setup.py + - fdroidserver/*.py + - tests/*.py + # Since F-Droid uses Debian as its default platform, from production # servers to CI to contributor machines, it is important to know when @@ -225,13 +236,12 @@ lint_format_bandit_checks: make pycodestyle pyflakes3 - pylint python3-dev python3-git python3-nose python3-pip python3-yaml - - $pip install --break-system-packages bandit pylint-gitlab + - $pip install --break-system-packages bandit - export EXITVALUE=0 - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; } - ./hooks/pre-commit || set_error @@ -240,6 +250,16 @@ lint_format_bandit_checks: -ii --ini .bandit || set_error + - exit $EXITVALUE + + +pylint: + image: debian:bookworm-slim + <<: *python-rules-changes + <<: *apt-template + script: + - apt-get install pylint python3-pip + - $pip install --break-system-packages pylint-gitlab - pylint --output-format=pylint_gitlab.GitlabCodeClimateReporter fdroid makebuildserver @@ -247,8 +267,6 @@ lint_format_bandit_checks: fdroidserver/*.py tests/*.py > pylint-report.json - || set_error - - exit $EXITVALUE artifacts: reports: codequality: pylint-report.json From 6ad79e3c06911a01545db489c231fcb46207b54a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 12 Mar 2025 18:37:42 +0100 Subject: [PATCH 4/6] gitlab-ci: move bandit to its own job --- .gitlab-ci.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c058e9a1..e16e2823 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -221,7 +221,7 @@ gradlew-fdroid: # Run all the various linters and static analysis tools. -lint_format_bandit_checks: +hooks/pre-commit: image: debian:bookworm-slim variables: LANG: C.UTF-8 @@ -241,17 +241,16 @@ lint_format_bandit_checks: python3-nose python3-pip python3-yaml - - $pip install --break-system-packages bandit - - export EXITVALUE=0 - - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; } - - ./hooks/pre-commit || set_error - - bandit - -r - -ii - --ini .bandit - || set_error - - exit $EXITVALUE + - ./hooks/pre-commit +bandit: + image: debian:bookworm-slim + <<: *python-rules-changes + <<: *apt-template + script: + - apt-get install python3-pip + - $pip install --break-system-packages bandit + - bandit -r -ii --ini .bandit pylint: image: debian:bookworm-slim From 7f67a882e74c152221bc7ecf29c3af1a7312f466 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 27 Mar 2025 01:45:22 +0100 Subject: [PATCH 5/6] gitlab-ci: only run Python-only jobs when .py files change --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e16e2823..d648aa6a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -642,6 +642,7 @@ servergitmirrors: Build documentation: image: debian:bookworm-slim + <<: *python-rules-changes <<: *apt-template script: - apt-get install make python3-sphinx python3-numpydoc python3-pydata-sphinx-theme pydocstyle fdroidserver @@ -758,6 +759,7 @@ docker: # PUBLISH is the signing server. It has a very minimal manual setup. PUBLISH: image: debian:bullseye-backports + <<: *python-rules-changes script: - apt-get update - apt-get -qy upgrade From 9bcd13bfddea8ff88c0eb54737adc7f5762cec8f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 27 Mar 2025 02:33:40 +0100 Subject: [PATCH 6/6] gitlab-ci: add "lint" pre-stage before default "test" stage --- .gitlab-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d648aa6a..0f5cf3f7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,12 @@ workflow: - if: $CI_COMMIT_BRANCH +stages: + - lint + - test # default for jobs that do not specify stage: + - deploy + + variables: pip: pip3 --timeout 100 --retries 10 # speed up git checkout phase @@ -222,6 +228,7 @@ gradlew-fdroid: # Run all the various linters and static analysis tools. hooks/pre-commit: + stage: lint image: debian:bookworm-slim variables: LANG: C.UTF-8 @@ -253,6 +260,7 @@ bandit: - bandit -r -ii --ini .bandit pylint: + stage: lint image: debian:bookworm-slim <<: *python-rules-changes <<: *apt-template @@ -273,6 +281,7 @@ pylint: shellcheck: + stage: lint image: debian:bookworm-slim rules: - changes: @@ -325,6 +334,7 @@ safety: # TODO tests/*/*/*.yaml are not covered yamllint: + stage: lint image: debian:bookworm-slim rules: - changes: @@ -349,6 +359,7 @@ yamllint: locales: + stage: lint image: debian:bookworm-slim variables: LANG: C.UTF-8 @@ -367,6 +378,7 @@ locales: black: + stage: lint image: debian:bookworm-slim <<: *apt-template script: