From a2978a5526411d18872e3efab06d409008be4b78 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 4 Dec 2017 21:22:16 +0100 Subject: [PATCH 1/9] common: aapt 24.0.0 (v0.2-2964546) is now required Without a recent aapt, the tag will not be found. --- fdroidserver/common.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 6e022f47..6511181f 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -388,8 +388,14 @@ def test_aapt_version(aapt): minor = m.group(2) bugfix = m.group(3) # the Debian package has the version string like "v0.2-23.0.2" - if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'): - logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-23.0.0 or newer!") + too_old = False + if '.' in bugfix: + if LooseVersion(bugfix) < LooseVersion('24.0.0'): + too_old = True + elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2964546'): + too_old = True + if too_old: + logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-24.0.0 or newer!") .format(aapt=aapt)) else: logging.warning(_('Unknown version of aapt, might cause problems: ') + output) From 67e6cbe7937793644c158a240e9cf304f639175c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 4 Dec 2017 12:10:46 +0100 Subject: [PATCH 2/9] hooks/pre-commit: make ruby and dash tests optional These are only used for checking syntax in buildserver/Vagrantfile. Not requiring ruby makes doing CI tests on lots of distros easier and faster. dash is an 'essential' package on Debian derivs, so those tests will always be run somewhere. --- hooks/pre-commit | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hooks/pre-commit b/hooks/pre-commit index 00d777f9..2bbfe39c 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -85,8 +85,10 @@ find_command() { echo : } +DASH=$(find_command dash) PYFLAKES=$(find_command pyflakes) PEP8=$(find_command pycodestyle pep8) +RUBY=$(find_command ruby) if [ "$PY_FILES $PY_TEST_FILES" != " " ]; then if ! $PYFLAKES $PY_FILES $PY_TEST_FILES; then @@ -110,7 +112,7 @@ if [ "$PY_TEST_FILES" != "" ]; then fi for f in $SH_FILES; do - if ! dash -n $f; then + if ! $DASH -n $f; then err "dash tests failed!" fi done @@ -122,7 +124,7 @@ for f in $BASH_FILES; do done for f in $RB_FILES; do - if ! ruby -c $f 1>/dev/null; then + if ! $RUBY -c $f 1>/dev/null; then err "ruby tests failed!" fi done From bfe2c00834ffa2dca1ac737da412235bb23b19e4 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Sun, 3 Dec 2017 15:01:48 +0100 Subject: [PATCH 3/9] common.testCase: fix find_sdk_tools when aapt is installed in /usr/bin The testlogic was broken when having both aapt in /usr/bin and also as part of the android sdk. --- tests/common.TestCase | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/common.TestCase b/tests/common.TestCase index 0ed9ced9..9f4b3ada 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -86,11 +86,8 @@ class CommonTest(unittest.TestCase): sdk_path = os.getenv('ANDROID_HOME') if os.path.exists(sdk_path): fdroidserver.common.config['sdk_path'] = sdk_path - if os.path.exists('/usr/bin/aapt'): - # this test only works when /usr/bin/aapt is installed - self._find_all() build_tools = os.path.join(sdk_path, 'build-tools') - if self._set_build_tools(): + if self._set_build_tools() or os.path.exists('/usr/bin/aapt'): self._find_all() else: print('no build-tools found: ' + build_tools) From 1b1475c982d25b2d35464781ed3b32e06531f877 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 4 Dec 2017 22:43:41 +0100 Subject: [PATCH 4/9] gitlab-ci: add test runs on Ubuntu, Debian/testing, Fedora --- .gitlab-ci.yml | 67 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0b0a4d3e..9ed32ba3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ -image: registry.gitlab.com/fdroid/ci-images-server:latest test: + image: registry.gitlab.com/fdroid/ci-images-server:latest script: - pip3 install -e . - cd tests @@ -10,6 +10,7 @@ test: # released version. Ensure that the official tags are included when # running these tests on forks as well. metadata_v0: + image: registry.gitlab.com/fdroid/ci-images-server:latest script: - git fetch https://gitlab.com/fdroid/fdroidserver 0.8 - cd tests @@ -25,3 +26,67 @@ metadata_v0: - cd fdroiddata - ../tests/dump_internal_metadata_format.py - diff -uw metadata/dump_* + +debian_testing: + image: debian:testing + only: + - master@fdroid/fdroidserver + script: + - apt update -y + - apt dist-upgrade -y + - apt-get install -y --no-install-recommends + aapt adb android-platform-tools-base android-sdk-common fdroidserver + git gnupg python3-setuptools zipalign + - export ANDROID_HOME=/usr/lib/android-sdk + - export LANG=C.UTF-8 + - cd tests + - rm -f install.TestCase # fails frequently and is unimportant + - ./run-tests + +ubuntu_lts: + image: ubuntu:latest + only: + - master@fdroid/fdroidserver + script: + - apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key 9AAC253193B65D4DF1D0A13EEC4632C79C5E0151 + - export RELEASE=`sed -n 's,^deb [^ ][^ ]* \([a-z]*\).*,\1,p' /etc/apt/sources.list | head -1` + - echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list + - apt update -y + - apt dist-upgrade -y + - apt-get install -y --no-install-recommends + aapt adb android-platform-tools-base android-sdk-common fdroidserver + git gnupg python3-setuptools unzip wget zipalign + - export ANDROID_HOME=/usr/lib/android-sdk + # xenial's aapt is too old + - wget --no-verbose https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip + - unzip -q build-tools_r27.0.1-linux.zip + - rm build-tools_r27.0.1-linux.zip + - mv android-8.1.0 $ANDROID_HOME/build-tools/27.0.1 + - export LANG=C.UTF-8 + - cd tests + - ./run-tests + +fedora_latest: + image: fedora:latest + only: + - master@fdroid/fdroidserver + script: + - dnf -y update + - dnf -y install git gnupg java-1.8.0-openjdk-devel python3 python3-pip rsync unzip wget + - pip3 install -e . + - wget --no-verbose -O tools.zip https://dl.google.com/android/repository/tools_r25.2.4-linux.zip + - unzip -q tools.zip + - rm tools.zip + - export ANDROID_HOME=~/android-sdk + - mkdir $ANDROID_HOME + - mv tools $ANDROID_HOME/ + - mkdir -p $ANDROID_HOME/licenses/ + - printf "\n8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e" > $ANDROID_HOME/licenses/android-sdk-license + - printf "\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license + - printf "\n79120722343a6f314e0719f863036c702b0e6b2a\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license-old + - mkdir ~/.android + - touch ~/.android/repositories.cfg + - echo y | $ANDROID_HOME/tools/bin/sdkmanager "platform-tools" + - echo y | $ANDROID_HOME/tools/bin/sdkmanager "build-tools;25.0.2" + - cd tests + - ./run-tests From dda9c8b77422a7c2c9eb3af75222d30912831358 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 6 Dec 2017 10:13:45 +0100 Subject: [PATCH 5/9] gitlab-ci: add `pip install` test on Arch Linux --- .gitlab-ci.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9ed32ba3..3460f092 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -66,6 +66,25 @@ ubuntu_lts: - cd tests - ./run-tests +pip_install: + image: archlinux/base + only: + - master@fdroid/fdroidserver + script: + - pacman --sync --sysupgrade --refresh --noconfirm grep python-pip python-virtualenv tar + # setup venv to act as release build machine + - python -m venv sdist-env + - . sdist-env/bin/activate + - ./setup.py compile_catalog sdist + - deactivate + - tar tzf dist/fdroidserver-*.tar.gz | grep locale/de/LC_MESSAGES/fdroidserver.mo + # back to bare machine to act as user's install machine + - pip install dist/fdroidserver-*.tar.gz + - test -e /usr/share/locale/de/LC_MESSAGES/fdroidserver.mo + - fdroid + - fdroid readmeta + - fdroid update --help + fedora_latest: image: fedora:latest only: From a0a68c7a1371ec3f9fb7b6b5f101b611a70913c8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 5 Dec 2017 16:11:10 +0100 Subject: [PATCH 6/9] setup requires Babel aka python3-babel to compile translations https://forum.f-droid.org/t/f-droid-server-building-error/1670 --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index a382b254..b7ab51e7 100755 --- a/setup.py +++ b/setup.py @@ -78,6 +78,9 @@ setup(name='fdroidserver', data_files=get_data_files(), python_requires='>=3.4', cmdclass={'versioncheck': VersionCheckCommand}, + setup_requires=[ + 'babel', + ], install_requires=[ 'clint', 'GitPython', From 964ef996a01ceac8d6735668f2c9fd9119512346 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 6 Dec 2017 09:47:08 +0100 Subject: [PATCH 7/9] makebuildserver: make copy_caches_from_host do rsync like `fdroid build` This rsync hung because of an SSH unknown key prompt. Since this is just the vm host sshing to the vm guest, it is not essential to check the host keys. --- fdroidserver/build.py | 1 + makebuildserver | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index bbb5a2b4..d83342d5 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -100,6 +100,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): # Helper to copy the contents of a directory to the server... def send_dir(path): logging.debug("rsyncing " + path + " to " + ftp.getcwd()) + # TODO this should move to `vagrant rsync` from >= v1.5 try: subprocess.check_output(['rsync', '--recursive', '--perms', '--links', '--quiet', '--rsh=' + 'ssh -o StrictHostKeyChecking=no' + diff --git a/makebuildserver b/makebuildserver index 7ce2e62d..b453d813 100755 --- a/makebuildserver +++ b/makebuildserver @@ -553,10 +553,16 @@ def main(): for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'): fullpath = os.path.join(os.getenv('HOME'), d) if os.path.isdir(fullpath): - # TODO newer versions of vagrant provide `vagrant rsync` + ssh_command = ' '.join('ssh -i {0} -p {1}'.format(key, port), + '-o StrictHostKeyChecking=no', + '-o UserKnownHostsFile=/dev/null', + '-o LogLevel=FATAL', + '-o IdentitiesOnly=yes', + '-o PasswordAuthentication=no') + # TODO vagrant 1.5+ provides `vagrant rsync` run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d]) subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e', - 'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port), + ssh_command, fullpath + '/', user + '@' + hostname + ':~/' + d + '/']) From 1a77c6af3863c76192e1b472183a8c6ae9679f71 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 6 Dec 2017 13:41:03 +0100 Subject: [PATCH 8/9] init: fix test for aapt when no aapt has been found Just give a proper error message rather than this stack trace: Traceback (most recent call last): File "/home/hans/code/fdroid/server/fdroid", line 156, in main() File "/home/hans/code/fdroid/server/fdroid", line 132, in main mod.main() File "/export/share/code/fdroid/server/fdroidserver/init.py", line 148, in main if os.path.isfile(aapt): File "/usr/lib/python3.5/genericpath.py", line 30, in isfile st = os.stat(path) --- fdroidserver/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 9d03e0b9..7b59bdb7 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -144,7 +144,7 @@ def main(): if os.path.isfile(os.path.join(d, 'aapt')): aapt = os.path.join(d, 'aapt') break - if os.path.isfile(aapt): + if aapt and os.path.isfile(aapt): dirname = os.path.basename(os.path.dirname(aapt)) if dirname == 'build-tools': # this is the old layout, before versioned build-tools From 05abbfbabdd3bf089207cfa8b4f7bfdf70fa1095 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 6 Dec 2017 17:51:12 +0100 Subject: [PATCH 9/9] gitlab-ci: move sdist test run to new fedora job A full run of the test suite takes quite a bit of time. This removes one of the 3 runs from the main 'tests' job, and puts it into the Fedora job. That test run is mostly to make sure the setup.py and source tarball are correctly, so that doesn't affect merge requests very often. This also tests `pip install --user`, which was not really being tested before. --- .gitlab-ci.yml | 14 ++++++++++---- tests/complete-ci-tests | 22 ---------------------- 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3460f092..2b337572 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -91,12 +91,16 @@ fedora_latest: - master@fdroid/fdroidserver script: - dnf -y update - - dnf -y install git gnupg java-1.8.0-openjdk-devel python3 python3-pip rsync unzip wget - - pip3 install -e . + - dnf -y install git gnupg java-1.8.0-openjdk-devel python3 python3-babel + python3-pip rsync unzip wget + - ./setup.py compile_catalog sdist + - useradd -m -c "test account" --password "fakepassword" testuser + - su testuser --login --command "cd `pwd`; pip3 install --user dist/fdroidserver-*.tar.gz" + - test -e ~testuser/.local/share/locale/de/LC_MESSAGES/fdroidserver.mo - wget --no-verbose -O tools.zip https://dl.google.com/android/repository/tools_r25.2.4-linux.zip - unzip -q tools.zip - rm tools.zip - - export ANDROID_HOME=~/android-sdk + - export ANDROID_HOME=`pwd`/android-sdk - mkdir $ANDROID_HOME - mv tools $ANDROID_HOME/ - mkdir -p $ANDROID_HOME/licenses/ @@ -107,5 +111,7 @@ fedora_latest: - touch ~/.android/repositories.cfg - echo y | $ANDROID_HOME/tools/bin/sdkmanager "platform-tools" - echo y | $ANDROID_HOME/tools/bin/sdkmanager "build-tools;25.0.2" + - chown -R testuser . - cd tests - - ./run-tests + - su testuser --login --command + "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests" diff --git a/tests/complete-ci-tests b/tests/complete-ci-tests index 37e59489..4573fcd5 100755 --- a/tests/complete-ci-tests +++ b/tests/complete-ci-tests @@ -74,28 +74,6 @@ for locale in *; do done -#------------------------------------------------------------------------------# -# test building the source tarball, then installing it -cd $WORKSPACE -python3 setup.py compile_catalog sdist - -# make sure translation files got compiled and included -tar tzf dist/fdroidserver-*.tar.gz | grep locale/de/LC_MESSAGES/fdroidserver.mo - -rm -rf $WORKSPACE/env -$pyvenv $WORKSPACE/env -. $WORKSPACE/env/bin/activate -# workaround https://github.com/pypa/setuptools/issues/937 -pip3 install --quiet setuptools==33.1.1 -pip3 install --quiet dist/fdroidserver-*.tar.gz - -# make sure translation files were installed -test -e $WORKSPACE/env/share/locale/de/LC_MESSAGES/fdroidserver.mo - -# run tests in new pip+pyvenv install -fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource - - #------------------------------------------------------------------------------# # test install using install direct from git repo cd $WORKSPACE