From a8a1333b8032662907b0f0e4a67aa308ec595729 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 26 Oct 2016 20:46:34 +0200 Subject: [PATCH 1/6] update buildserver scripts in sdist tarball manifest This was overlooked in previous work on ./makebuildserver --- MANIFEST.in | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index ca8eea6e..ecefc9ed 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,13 +5,15 @@ include fdroid include jenkins-build include makebuildserver include buildserver/config.buildserver.py -include buildserver/fixpaths.sh -include buildserver/cookbooks/android-ndk/recipes/default.rb -include buildserver/cookbooks/android-sdk/recipes/default.rb -include buildserver/cookbooks/fdroidbuild-general/recipes/default.rb -include buildserver/cookbooks/gradle/recipes/default.rb -include buildserver/cookbooks/gradle/recipes/gradle -include buildserver/cookbooks/kivy/recipes/default.rb +include buildserver/provision-android-ndk +include buildserver/provision-android-sdk +include buildserver/provision-apt-get-install +include buildserver/provision-apt-proxy +include buildserver/provision-gradle +include buildserver/provision-pip +include buildserver/provision-qt-sdk +include buildserver/provision-ubuntu-trusty-paramiko +include buildserver/Vagrantfile include completion/bash-completion include docs/fdl.texi include docs/fdroid.texi From ab8d51012d0b36b9b927dbd613b5087eeba3452f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 26 Oct 2016 20:41:37 +0200 Subject: [PATCH 2/6] use versionName unmodified as specified The versionName is defined as a string or string resource that can be any arbitrary data. fdroid should not second guess the developer here, and should just use the versionName unmodified. For anything that needs to compare different versions of apps, versionCode should always be used since that's what Android uses. https://developer.android.com/guide/topics/manifest/manifest-element.html#vname --- fdroidserver/build.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index daf109dd..37febba9 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -860,14 +860,6 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, if foundid != app.id: raise BuildException("Wrong package ID - build " + foundid + " but expected " + app.id) - # Some apps (e.g. Timeriffic) have had the bonkers idea of - # including the entire changelog in the version number. Remove - # it so we can compare. (TODO: might be better to remove it - # before we compile, in fact) - index = version.find(" //") - if index != -1: - version = version[:index] - if (version != build.version or vercode != build.vercode): raise BuildException(("Unexpected version/version code in output;" From 8ecff5bd616f6c492a9b8779846cedc4b9c5b4c4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 31 Oct 2016 16:51:34 +0100 Subject: [PATCH 3/6] get_release_filename() to handle any file type, not just APKs In order to support non-APK files that are built by `fdroid build`, this function that names the file releases needs to be generic. --- fdroidserver/build.py | 12 ++++++------ fdroidserver/common.py | 7 +++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 37febba9..021c5da7 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -412,7 +412,7 @@ def build_server(app, build, vcs, build_dir, output_dir, force): ftp.chdir(homedir + '/tmp') else: ftp.chdir(homedir + '/unsigned') - apkfile = common.getapkname(app, build) + apkfile = common.get_release_filename(app, build) tarball = common.getsrcname(app, build) try: ftp.get(apkfile, os.path.join(output_dir, apkfile)) @@ -884,7 +884,7 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, # Copy the unsigned apk to our destination directory for further # processing (by publish.py)... - dest = os.path.join(output_dir, common.getapkname(app, build)) + dest = os.path.join(output_dir, common.get_release_filename(app, build)) shutil.copyfile(src, dest) # Move the source tarball into the output directory... @@ -912,17 +912,17 @@ def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extl :returns: True if the build was done, False if it wasn't necessary. """ - dest_apk = common.getapkname(app, build) + dest_file = common.get_release_filename(app, build) - dest = os.path.join(output_dir, dest_apk) - dest_repo = os.path.join(repo_dir, dest_apk) + dest = os.path.join(output_dir, dest_file) + dest_repo = os.path.join(repo_dir, dest_file) if not test: if os.path.exists(dest) or os.path.exists(dest_repo): return False if also_check_dir: - dest_also = os.path.join(also_check_dir, dest_apk) + dest_also = os.path.join(also_check_dir, dest_file) if os.path.exists(dest_also): return False diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 1710c405..2ef58460 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -459,8 +459,11 @@ def apknameinfo(filename): return result -def getapkname(app, build): - return "%s_%s.apk" % (app.id, build.vercode) +def get_release_filename(app, build): + if build.output: + return "%s_%s.%s" % (app.id, build.vercode, get_file_extension(build.output)) + else: + return "%s_%s.apk" % (app.id, build.vercode) def getsrcname(app, build): From 84e09cd2a2c9ba4f963e4c56cd1df5cf4ae2e59f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 31 Oct 2016 19:53:55 +0100 Subject: [PATCH 4/6] allow arbitrary build products, not only APKs This makes it so that the final build product can be specified in output= and it'll work no matter if its an APK or not. This was developed around the case of building the OTA update.zip for the Privileged Extension. It should work for any build process in theory but it has not yet been tested. https://gitlab.com/fdroid/privileged-extension/issues/9 --- fdroidserver/build.py | 107 ++++++++++++++++++++++------------------- fdroidserver/common.py | 7 ++- fdroidserver/update.py | 2 +- tests/common.TestCase | 4 +- 4 files changed, 66 insertions(+), 54 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 021c5da7..cbbed3cd 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -454,6 +454,54 @@ def capitalize_intact(string): return string[0].upper() + string[1:] +def get_metadata_from_apk(app, build, apkfile): + """get the required metadata from the built APK""" + + p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False) + + vercode = None + version = None + foundid = None + nativecode = None + for line in p.output.splitlines(): + if line.startswith("package:"): + pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*") + m = pat.match(line) + if m: + foundid = m.group(1) + pat = re.compile(".*versionCode='([0-9]*)'.*") + m = pat.match(line) + if m: + vercode = m.group(1) + pat = re.compile(".*versionName='([^']*)'.*") + m = pat.match(line) + if m: + version = m.group(1) + elif line.startswith("native-code:"): + nativecode = line[12:] + + # Ignore empty strings or any kind of space/newline chars that we don't + # care about + if nativecode is not None: + nativecode = nativecode.strip() + nativecode = None if not nativecode else nativecode + + if build.buildjni and build.buildjni != ['no']: + if nativecode is None: + raise BuildException("Native code should have been built but none was packaged") + if build.novcheck: + vercode = build.vercode + version = build.version + if not version or not vercode: + raise BuildException("Could not find version information in build in output") + if not foundid: + raise BuildException("Could not find package ID in output") + if foundid != app.id: + raise BuildException("Wrong package ID - build " + foundid + " but expected " + app.id) + + return vercode, version + + def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh): """Do a build locally.""" @@ -809,7 +857,7 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, src = os.path.normpath(apks[0]) # Make sure it's not debuggable... - if common.isApkDebuggable(src, config): + if common.isApkAndDebuggable(src, config): raise BuildException("APK is debuggable") # By way of a sanity check, make sure the version and version @@ -818,56 +866,17 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, if not os.path.exists(src): raise BuildException("Unsigned apk is not at expected location of " + src) - p = SdkToolsPopen(['aapt', 'dump', 'badging', src], output=False) - - vercode = None - version = None - foundid = None - nativecode = None - for line in p.output.splitlines(): - if line.startswith("package:"): - pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*") - m = pat.match(line) - if m: - foundid = m.group(1) - pat = re.compile(".*versionCode='([0-9]*)'.*") - m = pat.match(line) - if m: - vercode = m.group(1) - pat = re.compile(".*versionName='([^']*)'.*") - m = pat.match(line) - if m: - version = m.group(1) - elif line.startswith("native-code:"): - nativecode = line[12:] - - # Ignore empty strings or any kind of space/newline chars that we don't - # care about - if nativecode is not None: - nativecode = nativecode.strip() - nativecode = None if not nativecode else nativecode - - if build.buildjni and build.buildjni != ['no']: - if nativecode is None: - raise BuildException("Native code should have been built but none was packaged") - if build.novcheck: + if common.get_file_extension(src) == 'apk': + vercode, version = get_metadata_from_apk(app, build, src) + if (version != build.version or vercode != build.vercode): + raise BuildException(("Unexpected version/version code in output;" + " APK: '%s' / '%s', " + " Expected: '%s' / '%s'") + % (version, str(vercode), build.version, + str(build.vercode))) + else: vercode = build.vercode version = build.version - if not version or not vercode: - raise BuildException("Could not find version information in build in output") - if not foundid: - raise BuildException("Could not find package ID in output") - if foundid != app.id: - raise BuildException("Wrong package ID - build " + foundid + " but expected " + app.id) - - if (version != build.version or - vercode != build.vercode): - raise BuildException(("Unexpected version/version code in output;" - " APK: '%s' / '%s', " - " Expected: '%s' / '%s'") - % (version, str(vercode), build.version, - str(build.vercode)) - ) # Add information for 'fdroid verify' to be able to reproduce the build # environment. diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 2ef58460..b653d5a8 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1618,11 +1618,14 @@ def get_file_extension(filename): return os.path.splitext(filename)[1].lower()[1:] -def isApkDebuggable(apkfile, config): - """Returns True if the given apk file is debuggable +def isApkAndDebuggable(apkfile, config): + """Returns True if the given file is an APK and is debuggable :param apkfile: full path to the apk to check""" + if get_file_extension(apkfile) != 'apk': + return False + p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'], output=False) if p.returncode != 0: diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 30390f82..cebd5a92 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -739,7 +739,7 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False): apk['minSdkVersion'] = 1 # Check for debuggable apks... - if common.isApkDebuggable(apkfile, config): + if common.isApkAndDebuggable(apkfile, config): logging.warning('{0} is set to android:debuggable="true"'.format(apkfile)) # Get the signature (or md5 of, to be precise)... diff --git a/tests/common.TestCase b/tests/common.TestCase index d7dca14e..d01568a2 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -74,7 +74,7 @@ class CommonTest(unittest.TestCase): testfiles.append(os.path.join(os.path.dirname(__file__), 'urzip-badsig.apk')) testfiles.append(os.path.join(os.path.dirname(__file__), 'urzip-badcert.apk')) for apkfile in testfiles: - debuggable = fdroidserver.common.isApkDebuggable(apkfile, config) + debuggable = fdroidserver.common.isApkAndDebuggable(apkfile, config) self.assertTrue(debuggable, "debuggable APK state was not properly parsed!") # these are set NOT debuggable @@ -82,7 +82,7 @@ class CommonTest(unittest.TestCase): testfiles.append(os.path.join(os.path.dirname(__file__), 'urzip-release.apk')) testfiles.append(os.path.join(os.path.dirname(__file__), 'urzip-release-unsigned.apk')) for apkfile in testfiles: - debuggable = fdroidserver.common.isApkDebuggable(apkfile, config) + debuggable = fdroidserver.common.isApkAndDebuggable(apkfile, config) self.assertFalse(debuggable, "debuggable APK state was not properly parsed!") From 56d51fcd6be992c7bbc38431db06817816c1e08e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 3 Nov 2016 10:26:38 +0100 Subject: [PATCH 5/6] gpg-sign all valid files in the repo, including source tarballs This makes sure there is a GPG signature on any file that is included in the repo, including APKs, OBB, source tarballs, media files, OTA update ZIPs, etc. Having a GPG signature is more important on non-APK files since they mostly do not have any signature mechanism of their own. This also adds basic tests of adding non-APK/OBB files to a repo with `fdroid update`. closes #232 --- examples/config.py | 2 +- fdroidserver/common.py | 11 +++++++++++ fdroidserver/gpgsign.py | 15 +++++++++------ fdroidserver/update.py | 4 +--- tests/gnupghome/pubring.gpg | Bin 0 -> 724 bytes tests/gnupghome/random_seed | Bin 0 -> 600 bytes tests/gnupghome/secring.gpg | Bin 0 -> 1388 bytes tests/gnupghome/trustdb.gpg | Bin 0 -> 1280 bytes tests/repo/fake.ota.update_1234.zip | Bin 0 -> 233 bytes .../repo/obb.main.twoversions_1101617_src.tar.gz | Bin 0 -> 150 bytes tests/run-tests | 14 +++++++++++++- 11 files changed, 35 insertions(+), 11 deletions(-) create mode 100644 tests/gnupghome/pubring.gpg create mode 100644 tests/gnupghome/random_seed create mode 100644 tests/gnupghome/secring.gpg create mode 100644 tests/gnupghome/trustdb.gpg create mode 100644 tests/repo/fake.ota.update_1234.zip create mode 100644 tests/repo/obb.main.twoversions_1101617_src.tar.gz diff --git a/examples/config.py b/examples/config.py index 63edc718..3b1ab95c 100644 --- a/examples/config.py +++ b/examples/config.py @@ -86,7 +86,7 @@ The repository of older versions of applications from the main demo repository. # current_version_name_source = 'id' # Optionally, override home directory for gpg -# gpghome = /home/fdroid/somewhere/else/.gnupg +# gpghome = '/home/fdroid/somewhere/else/.gnupg' # The ID of a GPG key for making detached signatures for apks. Optional. # gpgkey = '1DBA2E89' diff --git a/fdroidserver/common.py b/fdroidserver/common.py index b653d5a8..08708f31 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2084,3 +2084,14 @@ def get_per_app_repos(): repos.append(d) break return repos + + +def is_repo_file(filename): + '''Whether the file in a repo is a build product to be delivered to users''' + return os.path.isfile(filename) \ + and os.path.basename(filename) not in [ + 'index.jar', + 'index.xml', + 'index.html', + 'categories.txt', + ] diff --git a/fdroidserver/gpgsign.py b/fdroidserver/gpgsign.py index 41b5a43f..4c9cf6bb 100644 --- a/fdroidserver/gpgsign.py +++ b/fdroidserver/gpgsign.py @@ -50,10 +50,13 @@ def main(): sys.exit(1) # Process any apks that are waiting to be signed... - for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))): - - apkfilename = os.path.basename(apkfile) - sigfilename = apkfilename + ".asc" + for f in sorted(glob.glob(os.path.join(output_dir, '*.*'))): + if common.get_file_extension(f) == 'asc': + continue + if not common.is_repo_file(f): + continue + filename = os.path.basename(f) + sigfilename = filename + ".asc" sigpath = os.path.join(output_dir, sigfilename) if not os.path.exists(sigpath): @@ -64,13 +67,13 @@ def main(): gpgargs.extend(['--homedir', config['gpghome']]) if 'gpgkey' in config: gpgargs.extend(['--local-user', config['gpgkey']]) - gpgargs.append(os.path.join(output_dir, apkfilename)) + gpgargs.append(os.path.join(output_dir, filename)) p = FDroidPopen(gpgargs) if p.returncode != 0: logging.error("Signing failed.") sys.exit(1) - logging.info('Signed ' + apkfilename) + logging.info('Signed ' + filename) if __name__ == "__main__": diff --git a/fdroidserver/update.py b/fdroidserver/update.py index cebd5a92..110de3ef 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -517,13 +517,11 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): cachechanged = False repo_files = [] for name in os.listdir(repodir): - if name in ['index.jar', 'index.xml', 'index.html', 'categories.txt', ]: - continue file_extension = common.get_file_extension(name) if file_extension == 'apk' or file_extension == 'obb': continue filename = os.path.join(repodir, name) - if not os.path.isfile(filename): + if not common.is_repo_file(name): continue stat = os.stat(filename) if stat.st_size == 0: diff --git a/tests/gnupghome/pubring.gpg b/tests/gnupghome/pubring.gpg new file mode 100644 index 0000000000000000000000000000000000000000..fc60c42a12c6b85fc4af9da6199c1f4013eef803 GIT binary patch literal 724 zcmbQy%Mu~=kDrl+VXtfG<-5BsEZ=f!j�qN!X^hO2W*irfq9;VVqDTNk(R|LRx;2LSkMe zkPTFjR9d1?0@P8gsbG@^G{yl$*mdk+5oTm!P-0~Qd6Y?-nT?y1gPoa)O_YO)lZ%^0 zjER|%Nsf_8yn%s>Q$Xz6k;~@_zyG!lXJME+Rq*YFQ+pD6>ekKcIp!(haYahc_?6ZI zyX%d8`zMNBTV{U#z5MKJbDnUCxhY*%v2`$Lco^TF@n-T*o+4%mIqv`3529UXsC8{! zR9}b-7jBu;Jxefx>c`j(V+* zHJ2DH<2EocFz)g?p*h_1q^9hiqSckE zTguS>JpMwS{*%C&KP&z%Q)w)n+G+SQ$ofZAnP0KwrgW}g)=?x-o9Df0I0x&51utvUgmVCh#_9 xZxUw=7vCu(yy+^u-&Pam73;=z?Gz0(u literal 0 HcmV?d00001 diff --git a/tests/gnupghome/random_seed b/tests/gnupghome/random_seed new file mode 100644 index 0000000000000000000000000000000000000000..cb41f6e0107d690b8d2aec094ed6e7b10648c86d GIT binary patch literal 600 zcmV-e0;m1-T`r;8DQxp34*eCDD#z{;y@dF;VHekN60OR4My2@deo&N*Omm?%0ldsV z9=*FgRYYJv49GmT(8c&wU`%`)R)ya9Z@Yaqn<=~6J{b>i7ZFrU*ACe`%I@pX#^ zKbko|9)MpiW$Fjt!EB813PK>ap!JCL3lr@0!4}!4F4JrZyy|0CIGqmCM>qQ_{{aNJr@51lFjRO@}mVhI zcWjC~FO~#W8(tZ3Dk93K&|}2n&4}u!hp+Vxd&-|r>x zAA;fag&I>Q*WO@tA8=^xj#7ePTAUPcKe4H)G5nm z15?r=X7*gm9gmGzDnaEr?mi7{H)CII`y_YZOWxww+Xy*q&_9I;4KOtd6S>maCi8fn mWPbmrC!LTJ{9oF=FDJ1W(n7{Hbof>h8d3JB+@GwcYCa1GUM(L0 literal 0 HcmV?d00001 diff --git a/tests/gnupghome/secring.gpg b/tests/gnupghome/secring.gpg new file mode 100644 index 0000000000000000000000000000000000000000..20b160863a87ac3b36fa315feeba66ebdf3b87bc GIT binary patch literal 1388 zcmV-y1(W)f0oVjs8vYLf1OUB7Rny(N(5JM@oEvW=bXKzMA`=73mbQjO1D`Dj-*33c z!CC%a$tjPXNCP;f(4hjI<3V*v%4qpVBL~`Y@E}cMaODN9;3cmOn65A`Vx3Mr1HpWB zZ}P0sKQXcxa#a8k0RRC21OHEu z9e2JCL6`7aO4gGYa!SIf*FcQx={r#_}<49*%~M!lQ}dn zQh}2u3ALIuoqjepF3Jk2#K~tIlj8ya*!CHsxk%Dq&72mgZ>yrmE-6^$M{FTGx?;9C zn_w5h7W^bgGHf;JXzy6ElgmbTWZ0&Ytk8zd>PW*;*R|Ei+ zl@aaG%DiBVey*O3#z`1R)EX``>MEc<*MW||krvjbH_z`Mo7SA<3Km8p(8+I@hq>8%iPcN~a+Du}Go06AW;0ssJ=0oVjs8vYLf1OUjSZ=mN_RhI8o>VJ&| z2wTX1XH9K@88Q-h!ki198kD<~DvD(-bpVIwU(jwZbm&fpwLMG3in}_*!;O za~ZN{3R4Bbg8uVeHtlQLKSajAl+K@*2M6Wphe0VDX%y|v&cCz7ZiVfPqx?gRHS+v2 z5sBUm3bNtg%u@gn0RRC21O7V)31T#TAdG>CH~-^oZX1?CPsGW$7F44FJL9FdtLp>gFHp%J=4H$*pMqOW{vwjliA zjS}^Qo9y=6n1#z$XgleIy z0yjqosAcoG$$@oS%sE;$iUhVSle_Gh)Y)oN3*Yez1hEByYO4PuZy!aQDv`lSi^MqPL27u~zJYf7%Os7v&M KRH!MVsV*{UyDX3m!qN;f3~7nkse1V(iF&04DTyVidIdR&$*CFn zIVq_{p&^_M%zQtkB0#vbf}4SnFj8=K4N)*M yFf`)|@MdI^W5#8t1kh#%21X#>(g(iQ+MLKT8W9aa$`U z_-}nB&*!e~`pfVBi8%IX+2W~@dzPQ~jQ^~t^4$6D5*gi@!SU> config.py echo "install_list = 'org.adaway'" >> config.py echo "uninstall_list = {'com.android.vending', 'com.facebook.orca',}" >> config.py +echo "gpghome = '$GNUPGHOME'" >> config.py +echo "gpgkey = 'CE71F7FB'" >> config.py $fdroid update --verbose test -e repo/index.xml test -e repo/index.jar grep -F 'source tarball '; $out.=$this->human_readable_size(filesize($this->site_path.'/repo/'.$apk['srcname'])); + if(file_exists($this->site_path.'/repo/'.$apk['srcname'].'.asc')) { + $out.=' GPG Signature '; + } } if(isset($apk['permissions'])) {