Merge branch 'per-app-repos' into 'master'

config option to enable per-app repos for nightly builds

For Guardian Project, we've been running an fdroid repo for the nightly builds for each of our apps: https://dev.guardianproject.info/debug  This is built using a big, hacked up shell script: [update-debug-fdroid-repo](https://github.com/guardianproject/fdroid-repo-tools/blob/master/update-debug-fdroid-repo).  It has proven very useful to us to be able to subscribe to the nightly build for a single app, so this the first step of porting that horrid shell script to `fdroidserver`.

This also helps make the fdroidserver tool suite the single set of tools for all types of builds and releases. That will hopefully drive more free software developers to make f-droid.org the core channel for official releases.

See merge request !66

buildserver/cookbooks/kivy/recipes/default.rb

@@ -24,7 +24,7 @@ script "install-p4a" do
   cwd "/home/vagrant"
   interpreter "bash"
   code "
-    git clone git://github.com/kivy/python-for-android
+    git clone https://github.com/kivy/python-for-android
     chown -R vagrant:vagrant python-for-android
     cd python-for-android
     git checkout ca369d774e2

examples/config.py

@@ -56,6 +56,12 @@ archive_description = """
 The repository of older versions of applications from the main demo repository.
 """
 
+# Normally, all apps are collected into a single app repository, like on
+# https://f-droid.org.  For certain situations, it is better to make a repo
+# that is made up of APKs only from a single app.  For example, an automated
+# build server that publishes nightly builds.
+# per_app_repos = True
+
 # `fdroid update` will create a link to the current version of a given app.
 # This provides a static path to the current APK.  To disable the creation of
 # this link, uncomment this:

fdroidserver/common.py

@@ -58,6 +58,7 @@ default_config = {
     'mvn3': "mvn",
     'gradle': 'gradle',
     'sync_from_local_copy_dir': False,
+    'per_app_repos': False,
     'make_current_version_link': True,
     'current_version_name_source': 'Name',
     'update_stats': False,
@@ -2135,3 +2136,26 @@ def download_file(url, local_filename=None, dldir='tmp'):
                 f.write(chunk)
                 f.flush()
     return local_filename
+
+
+def get_per_app_repos():
+    '''per-app repos are dirs named with the packageName of a single app'''
+
+    # Android packageNames are Java packages, they may contain uppercase or
+    # lowercase letters ('A' through 'Z'), numbers, and underscores
+    # ('_'). However, individual package name parts may only start with
+    # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
+    p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
+
+    repos = []
+    for root, dirs, files in os.walk(os.getcwd()):
+        for d in dirs:
+            print 'checking', root, 'for', d
+            if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
+                # standard parts of an fdroid repo, so never packageNames
+                continue
+            elif p.match(d) \
+                    and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
+                repos.append(d)
+        break
+    return repos

fdroidserver/server.py

@@ -104,10 +104,10 @@ def update_awsbucket(repo_section):
                     extra['content_type'] = 'application/pgp-signature'
                 logging.info(' uploading ' + os.path.relpath(file_to_upload)
                              + ' to s3://' + awsbucket + '/' + object_name)
-                obj = driver.upload_object(file_path=file_to_upload,
+                with open(file_to_upload, 'rb') as iterator:
+                    obj = driver.upload_object_via_stream(iterator=iterator,
                                                           container=container,
                                                           object_name=object_name,
-                                           verify_hash=False,
                                                           extra=extra)
     # delete the remnants in the bucket, they do not exist locally
     while objs:
@@ -285,6 +285,8 @@ def main():
         repo_sections.append('archive')
         if not os.path.exists('archive'):
             os.mkdir('archive')
+    if config['per_app_repos']:
+        repo_sections += common.get_per_app_repos()
 
     if args[0] == 'init':
         ssh = paramiko.SSHClient()

fdroidserver/update.py

@@ -1016,6 +1016,28 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi
                 apks.remove(apk)
 
 
+def add_apks_to_per_app_repos(repodir, apks):
+    apks_per_app = dict()
+    for apk in apks:
+        apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
+        apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
+        apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
+        apks_per_app[apk['id']] = apk
+
+        if not os.path.exists(apk['per_app_icons']):
+            logging.info('Adding new repo for only ' + apk['id'])
+            os.makedirs(apk['per_app_icons'])
+
+        apkpath = os.path.join(repodir, apk['apkname'])
+        shutil.copy(apkpath, apk['per_app_repo'])
+        apksigpath = apkpath + '.sig'
+        if os.path.exists(apksigpath):
+            shutil.copy(apksigpath, apk['per_app_repo'])
+        apkascpath = apkpath + '.asc'
+        if os.path.exists(apkascpath):
+            shutil.copy(apkascpath, apk['per_app_repo'])
+
+
 config = None
 options = None
 
@@ -1119,14 +1141,11 @@ def main():
             apkcache = pickle.load(cf)
     else:
         apkcache = {}
-    cachechanged = False
 
     delete_disabled_builds(apps, apkcache, repodirs)
 
     # Scan all apks in the main repo
-    apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
+    apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
-    if cc:
-        cachechanged = True
 
     # Generate warnings for apk's with no metadata (or create skeleton
     # metadata files, if requested on the command line)
@@ -1218,6 +1237,20 @@ def main():
     # name comes from there!)
     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
 
+    # APKs are placed into multiple repos based on the app package, providing
+    # per-app subscription feeds for nightly builds and things like it
+    if config['per_app_repos']:
+        add_apks_to_per_app_repos(repodirs[0], apks)
+        for appid, app in apps.iteritems():
+            repodir = os.path.join(appid, 'fdroid', 'repo')
+            appdict = dict()
+            appdict[appid] = app
+            if os.path.isdir(repodir):
+                make_index(appdict, [appid], apks, repodir, False, categories)
+            else:
+                logging.info('Skipping index generation for ' + appid)
+        return
+
     if len(repodirs) > 1:
         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])