mirrors/fdroidserver
1
0
Fork 0
mirror of https://github.com/f-droid/fdroidserver.git synced 2025-09-16 16:02:33 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Continue using FDroidPopen and SilentPopen more, support shell=

Browse source
This commit is contained in:
Daniel Martí 2014-02-17 13:25:55 +01:00
parent cc0943777e
commit 8533b8c44e
2 changed files with 85 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

15
fdroidserver/build.py
View file

@ -32,7 +32,7 @@ from optparse import OptionParser, OptionError
import logging import logging
import common, metadata import common, metadata
from common import BuildException, VCSException, FDroidPopen from common import BuildException, VCSException, FDroidPopen, SilentPopen
def get_builder_vm_id(): def get_builder_vm_id():
vd = os.path.join('builder', '.vagrant') vd = os.path.join('builder', '.vagrant')
@ -364,7 +364,7 @@ def adapt_gradle(build_dir):
path = os.path.join(root, 'build.gradle') path = os.path.join(root, 'build.gradle')
logging.info("Adapting build.gradle at %s" % path) logging.info("Adapting build.gradle at %s" % path)
subprocess.call(['sed', '-i', FDroidPopen(['sed', '-i',
r's@buildToolsVersion\([ =]*\)["\'][0-9\.]*["\']@buildToolsVersion\1"' r's@buildToolsVersion\([ =]*\)["\'][0-9\.]*["\']@buildToolsVersion\1"'
+ config['build_tools'] + '"@g', path]) + config['build_tools'] + '"@g', path])
@ -495,11 +495,11 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
'-Dandroid.release=true', 'package'] '-Dandroid.release=true', 'package']
if 'target' in thisbuild: if 'target' in thisbuild:
target = thisbuild["target"].split('-')[1] target = thisbuild["target"].split('-')[1]
subprocess.call(['sed', '-i', FDroidPopen(['sed', '-i',
's@<platform>[0-9]*</platform>@<platform>'+target+'</platform>@g', 's@<platform>[0-9]*</platform>@<platform>'+target+'</platform>@g',
'pom.xml'], cwd=root_dir) 'pom.xml'], cwd=root_dir)
if '@' in thisbuild['maven']: if '@' in thisbuild['maven']:
subprocess.call(['sed', '-i', FDroidPopen(['sed', '-i',
's@<platform>[0-9]*</platform>@<platform>'+target+'</platform>@g', 's@<platform>[0-9]*</platform>@<platform>'+target+'</platform>@g',
'pom.xml'], cwd=maven_dir) 'pom.xml'], cwd=maven_dir)
@ -537,7 +537,8 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
cmd += ' ./distribute.sh' cmd += ' ./distribute.sh'
cmd += ' -m ' + "'" + ' '.join(modules) + "'" cmd += ' -m ' + "'" + ' '.join(modules) + "'"
cmd += ' -d fdroid' cmd += ' -d fdroid'
if subprocess.call(cmd, cwd='python-for-android', shell=True) != 0: p = FDroidPopen(cmd, cwd='python-for-android', shell=True)
if p.returncode != 0:
raise BuildException("Distribute build failed") raise BuildException("Distribute build failed")
cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name') cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
@ -652,9 +653,9 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
if not os.path.exists(src): if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " + src) raise BuildException("Unsigned apk is not at expected location of " + src)
p = FDroidPopen([os.path.join(config['sdk_path'], p = SilentPopen([os.path.join(config['sdk_path'],
'build-tools', config['build_tools'], 'aapt'), 'build-tools', config['build_tools'], 'aapt'),
'dump', 'badging', src], output=False) 'dump', 'badging', src])
vercode = None vercode = None
version = None version = None

141
fdroidserver/common.py
View file

@ -302,33 +302,38 @@ class vcs_git(vcs):
def gotorevisionx(self, rev): def gotorevisionx(self, rev):
if not os.path.exists(self.local): if not os.path.exists(self.local):
# Brand new checkout # Brand new checkout
if subprocess.call(['git', 'clone', self.remote, self.local]) != 0: p = SilentPopen(['git', 'clone', self.remote, self.local])
if p.returncode != 0:
raise VCSException("Git clone failed") raise VCSException("Git clone failed")
self.checkrepo() self.checkrepo()
else: else:
self.checkrepo() self.checkrepo()
# Discard any working tree changes # Discard any working tree changes
if subprocess.call(['git', 'reset', '--hard'], cwd=self.local) != 0: p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
if p.returncode != 0:
raise VCSException("Git reset failed") raise VCSException("Git reset failed")
# Remove untracked files now, in case they're tracked in the target # Remove untracked files now, in case they're tracked in the target
# revision (it happens!) # revision (it happens!)
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0: p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
if p.returncode != 0:
raise VCSException("Git clean failed") raise VCSException("Git clean failed")
if not self.refreshed: if not self.refreshed:
# Get latest commits and tags from remote # Get latest commits and tags from remote
if subprocess.call(['git', 'fetch', 'origin'], p = SilentPopen(['git', 'fetch', 'origin'], cwd=self.local)
cwd=self.local) != 0: if p.returncode != 0:
raise VCSException("Git fetch failed") raise VCSException("Git fetch failed")
if subprocess.call(['git', 'fetch', '--prune', '--tags', 'origin'], p = SilentPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local)
cwd=self.local) != 0: if p.returncode != 0:
raise VCSException("Git fetch failed") raise VCSException("Git fetch failed")
self.refreshed = True self.refreshed = True
# Check out the appropriate revision # Check out the appropriate revision
rev = str(rev if rev else 'origin/master') rev = str(rev if rev else 'origin/master')
if subprocess.call(['git', 'checkout', '-f', rev], cwd=self.local) != 0: p = SilentPopen(['git', 'checkout', '-f', rev], cwd=self.local)
if p.returncode != 0:
raise VCSException("Git checkout failed") raise VCSException("Git checkout failed")
# Get rid of any uncontrolled files left behind # Get rid of any uncontrolled files left behind
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0: p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
if p.returncode != 0:
raise VCSException("Git clean failed") raise VCSException("Git clean failed")
def initsubmodules(self): def initsubmodules(self):
@ -340,9 +345,8 @@ class vcs_git(vcs):
p = SilentPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local) p = SilentPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local)
if p.returncode != 0: if p.returncode != 0:
raise VCSException("Git submodule reset failed") raise VCSException("Git submodule reset failed")
if subprocess.call(['git', 'submodule', 'update', p = SilentPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
'--init', '--force', '--recursive'], if p.returncode != 0:
cwd=self.local) != 0:
raise VCSException("Git submodule update failed") raise VCSException("Git submodule update failed")
def gettags(self): def gettags(self):
@ -386,27 +390,29 @@ class vcs_gitsvn(vcs):
gitsvn_cmd += ' -t %s' % i[5:] gitsvn_cmd += ' -t %s' % i[5:]
elif i.startswith('branches='): elif i.startswith('branches='):
gitsvn_cmd += ' -b %s' % i[9:] gitsvn_cmd += ' -b %s' % i[9:]
if subprocess.call([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], p = SilentPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True)
shell=True) != 0: if p.returncode != 0:
raise VCSException("Git clone failed") raise VCSException("Git clone failed")
else: else:
if subprocess.call([gitsvn_cmd + " %s %s" % (self.remote, self.local)], p = SilentPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True)
shell=True) != 0: if p.returncode != 0:
raise VCSException("Git clone failed") raise VCSException("Git clone failed")
self.checkrepo() self.checkrepo()
else: else:
self.checkrepo() self.checkrepo()
# Discard any working tree changes # Discard any working tree changes
if subprocess.call(['git', 'reset', '--hard'], cwd=self.local) != 0: p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
if p.returncode != 0:
raise VCSException("Git reset failed") raise VCSException("Git reset failed")
# Remove untracked files now, in case they're tracked in the target # Remove untracked files now, in case they're tracked in the target
# revision (it happens!) # revision (it happens!)
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0: p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
if p.returncode != 0:
raise VCSException("Git clean failed") raise VCSException("Git clean failed")
if not self.refreshed: if not self.refreshed:
# Get new commits and tags from repo # Get new commits and tags from repo
if subprocess.call(['%sgit svn rebase %s' % self.userargs()], p = SilentPopen(['%sgit svn rebase %s' % self.userargs()], cwd=self.local, shell=True)
cwd=self.local, shell=True) != 0: if p.returncode != 0:
raise VCSException("Git svn rebase failed") raise VCSException("Git svn rebase failed")
self.refreshed = True self.refreshed = True
@ -444,7 +450,8 @@ class vcs_gitsvn(vcs):
raise VCSException("Git svn checkout failed") raise VCSException("Git svn checkout failed")
# Get rid of any uncontrolled files left behind # Get rid of any uncontrolled files left behind
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0: p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
if p.returncode != 0:
raise VCSException("Git clean failed") raise VCSException("Git clean failed")
def gettags(self): def gettags(self):
@ -472,24 +479,25 @@ class vcs_svn(vcs):
def gotorevisionx(self, rev): def gotorevisionx(self, rev):
if not os.path.exists(self.local): if not os.path.exists(self.local):
if subprocess.call(['svn', 'checkout', self.remote, self.local] + p = SilentPopen(['svn', 'checkout', self.remote, self.local] + self.userargs())
self.userargs()) != 0: if p.returncode != 0:
raise VCSException("Svn checkout failed") raise VCSException("Svn checkout failed")
else: else:
for svncommand in ( for svncommand in (
'svn revert -R .', 'svn revert -R .',
r"svn status | awk '/\?/ {print $2}' | xargs rm -rf"): r"svn status | awk '/\?/ {print $2}' | xargs rm -rf"):
if subprocess.call(svncommand, cwd=self.local, shell=True) != 0: p = SilentPopen(svncommand, cwd=self.local, shell=True)
if p.returncode != 0:
raise VCSException("Svn reset ({0}) failed in {1}".format(svncommand, self.local)) raise VCSException("Svn reset ({0}) failed in {1}".format(svncommand, self.local))
if not self.refreshed: if not self.refreshed:
if subprocess.call(['svn', 'update'] + p = SilentPopen(['svn', 'update'] + self.userargs(), cwd=self.local)
self.userargs(), cwd=self.local) != 0: if p.returncode != 0:
raise VCSException("Svn update failed") raise VCSException("Svn update failed")
self.refreshed = True self.refreshed = True
revargs = list(['-r', rev] if rev else []) revargs = list(['-r', rev] if rev else [])
if subprocess.call(['svn', 'update', '--force'] + revargs + p = SilentPopen(['svn', 'update', '--force'] + revargs + self.userargs(), cwd=self.local)
self.userargs(), cwd=self.local) != 0: if p.returncode != 0:
raise VCSException("Svn update failed") raise VCSException("Svn update failed")
def getref(self): def getref(self):
@ -506,30 +514,32 @@ class vcs_hg(vcs):
def gotorevisionx(self, rev): def gotorevisionx(self, rev):
if not os.path.exists(self.local): if not os.path.exists(self.local):
if subprocess.call(['hg', 'clone', self.remote, self.local]) !=0: p = SilentPopen(['hg', 'clone', self.remote, self.local])
if p.returncode != 0:
raise VCSException("Hg clone failed") raise VCSException("Hg clone failed")
else: else:
if subprocess.call('hg status -uS | xargs rm -rf', p = SilentPopen('hg status -uS | xargs rm -rf', cwd=self.local, shell=True)
cwd=self.local, shell=True) != 0: if p.returncode != 0:
raise VCSException("Hg clean failed") raise VCSException("Hg clean failed")
if not self.refreshed: if not self.refreshed:
if subprocess.call(['hg', 'pull'], p = SilentPopen(['hg', 'pull'], cwd=self.local)
cwd=self.local) != 0: if p.returncode != 0:
raise VCSException("Hg pull failed") raise VCSException("Hg pull failed")
self.refreshed = True self.refreshed = True
rev = str(rev if rev else 'default') rev = str(rev if rev else 'default')
if not rev: if not rev:
return return
if subprocess.call(['hg', 'update', '-C', rev], p = SilentPopen(['hg', 'update', '-C', rev], cwd=self.local)
cwd=self.local) != 0: if p.returncode != 0:
raise VCSException("Hg checkout failed") raise VCSException("Hg checkout failed")
p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local) p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
# Also delete untracked files, we have to enable purge extension for that: # Also delete untracked files, we have to enable purge extension for that:
if "'purge' is provided by the following extension" in p.stdout: if "'purge' is provided by the following extension" in p.stdout:
with open(self.local+"/.hg/hgrc", "a") as myfile: with open(self.local+"/.hg/hgrc", "a") as myfile:
myfile.write("\n[extensions]\nhgext.purge=\n") myfile.write("\n[extensions]\nhgext.purge=\n")
if subprocess.call(['hg', 'purge', '--all'], cwd=self.local) != 0: p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
if p.returncode != 0:
raise VCSException("HG purge failed") raise VCSException("HG purge failed")
elif p.returncode != 0: elif p.returncode != 0:
raise VCSException("HG purge failed") raise VCSException("HG purge failed")
@ -546,21 +556,22 @@ class vcs_bzr(vcs):
def gotorevisionx(self, rev): def gotorevisionx(self, rev):
if not os.path.exists(self.local): if not os.path.exists(self.local):
if subprocess.call(['bzr', 'branch', self.remote, self.local]) != 0: p = SilentPopen(['bzr', 'branch', self.remote, self.local])
if p.returncode != 0:
raise VCSException("Bzr branch failed") raise VCSException("Bzr branch failed")
else: else:
if subprocess.call(['bzr', 'clean-tree', '--force', p = SilentPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local)
'--unknown', '--ignored'], cwd=self.local) != 0: if p.returncode != 0:
raise VCSException("Bzr revert failed") raise VCSException("Bzr revert failed")
if not self.refreshed: if not self.refreshed:
if subprocess.call(['bzr', 'pull'], p = SilentPopen(['bzr', 'pull'], cwd=self.local)
cwd=self.local) != 0: if p.returncode != 0:
raise VCSException("Bzr update failed") raise VCSException("Bzr update failed")
self.refreshed = True self.refreshed = True
revargs = list(['-r', rev] if rev else []) revargs = list(['-r', rev] if rev else [])
if subprocess.call(['bzr', 'revert'] + revargs, p = SilentPopen(['bzr', 'revert'] + revargs, cwd=self.local)
cwd=self.local) != 0: if p.returncode != 0:
raise VCSException("Bzr revert failed") raise VCSException("Bzr revert failed")
def gettags(self): def gettags(self):
@ -893,8 +904,8 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
patch = patch.strip() patch = patch.strip()
logging.info("Applying " + patch) logging.info("Applying " + patch)
patch_path = os.path.join('metadata', app['id'], patch) patch_path = os.path.join('metadata', app['id'], patch)
if subprocess.call(['patch', '-p1', p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
'-i', os.path.abspath(patch_path)], cwd=build_dir) != 0: if p.returncode != 0:
raise BuildException("Failed to apply patch %s" % patch_path) raise BuildException("Failed to apply patch %s" % patch_path)
# Get required source libraries # Get required source libraries
@ -953,13 +964,13 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
if 'target' in build: if 'target' in build:
n = build["target"].split('-')[1] n = build["target"].split('-')[1]
subprocess.call(['sed', '-i', FDroidPopen(['sed', '-i',
's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+n+'@g', 's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+n+'@g',
'build.gradle'], cwd=root_dir) 'build.gradle'], cwd=root_dir)
if '@' in build['gradle']: if '@' in build['gradle']:
gradle_dir = os.path.join(root_dir, build['gradle'].split('@',1)[1]) gradle_dir = os.path.join(root_dir, build['gradle'].split('@',1)[1])
gradle_dir = os.path.normpath(gradle_dir) gradle_dir = os.path.normpath(gradle_dir)
subprocess.call(['sed', '-i', FDroidPopen(['sed', '-i',
's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+n+'@g', 's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+n+'@g',
'build.gradle'], cwd=gradle_dir) 'build.gradle'], cwd=gradle_dir)
@ -973,14 +984,16 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
if not os.path.isfile(path): if not os.path.isfile(path):
continue continue
if has_extension(path, 'xml'): if has_extension(path, 'xml'):
if subprocess.call(['sed','-i', p = SilentPopen(['sed', '-i',
's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g', 's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g',
path]) != 0: path])
if p.returncode != 0:
raise BuildException("Failed to amend manifest") raise BuildException("Failed to amend manifest")
elif has_extension(path, 'gradle'): elif has_extension(path, 'gradle'):
if subprocess.call(['sed','-i', p = SilentPopen(['sed', '-i',
's/versionName[ ]*=[ ]*"[^"]*"/versionName = "' + build['version'] + '"/g', 's/versionName[ ]*=[ ]*"[^"]*"/versionName = "' + build['version'] + '"/g',
path]) != 0: path])
if p.returncode != 0:
raise BuildException("Failed to amend build.gradle") raise BuildException("Failed to amend build.gradle")
if build['forcevercode']: if build['forcevercode']:
logging.info("Changing the version code") logging.info("Changing the version code")
@ -988,14 +1001,16 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
if not os.path.isfile(path): if not os.path.isfile(path):
continue continue
if has_extension(path, 'xml'): if has_extension(path, 'xml'):
if subprocess.call(['sed','-i', p = SilentPopen(['sed', '-i',
's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g', 's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g',
path]) != 0: path])
if p.returncode != 0:
raise BuildException("Failed to amend manifest") raise BuildException("Failed to amend manifest")
elif has_extension(path, 'gradle'): elif has_extension(path, 'gradle'):
if subprocess.call(['sed','-i', p = SilentPopen(['sed', '-i',
's/versionCode[ ]*=[ ]*[0-9]*/versionCode = ' + build['vercode'] + '/g', 's/versionCode[ ]*=[ ]*[0-9]*/versionCode = ' + build['vercode'] + '/g',
path]) != 0: path])
if p.returncode != 0:
raise BuildException("Failed to amend build.gradle") raise BuildException("Failed to amend build.gradle")
# Delete unwanted files # Delete unwanted files
@ -1011,9 +1026,9 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
raise BuildException("rm removes whole build directory") raise BuildException("rm removes whole build directory")
if os.path.lexists(rdest): if os.path.lexists(rdest):
if os.path.islink(rdest): if os.path.islink(rdest):
subprocess.call('unlink ' + rdest, shell=True) SilentPopen('unlink ' + rdest, shell=True)
else: else:
subprocess.call('rm -rf ' + rdest, shell=True) SilentPopen('rm -rf ' + rdest, shell=True)
else: else:
logging.info("...but it didn't exist") logging.info("...but it didn't exist")
@ -1317,7 +1332,7 @@ class PopenResult:
returncode = None returncode = None
stdout = '' stdout = ''
def SilentPopen(commands, cwd=None): def SilentPopen(commands, cwd=None, shell=False):
""" """
Run a command silently and capture the output. Run a command silently and capture the output.
@ -1328,20 +1343,18 @@ def SilentPopen(commands, cwd=None):
if cwd: if cwd:
cwd = os.path.normpath(cwd) cwd = os.path.normpath(cwd)
if cwd:
logging.debug("Directory: %s" % cwd) logging.debug("Directory: %s" % cwd)
logging.debug("> %s" % ' '.join(commands)) logging.debug("> %s" % ' '.join(commands))
result = PopenResult() result = PopenResult()
p = subprocess.Popen(commands, cwd=cwd, p = subprocess.Popen(commands, cwd=cwd, shell=shell,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
result.stdout = p.communicate()[0] result.stdout = p.communicate()[0]
result.returncode = p.returncode result.returncode = p.returncode
return result return result
def FDroidPopen(commands, cwd=None): def FDroidPopen(commands, cwd=None, shell=False):
""" """
Run a command and capture the possibly huge output. Run a command and capture the possibly huge output.
@ -1352,13 +1365,11 @@ def FDroidPopen(commands, cwd=None):
if cwd: if cwd:
cwd = os.path.normpath(cwd) cwd = os.path.normpath(cwd)
if cwd:
logging.info("Directory: %s" % cwd) logging.info("Directory: %s" % cwd)
logging.info("> %s" % ' '.join(commands)) logging.info("> %s" % ' '.join(commands))
result = PopenResult() result = PopenResult()
p = subprocess.Popen(commands, cwd=cwd, p = subprocess.Popen(commands, cwd=cwd, shell=shell,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout_queue = Queue.Queue() stdout_queue = Queue.Queue()