mirror of
https://github.com/f-droid/fdroidserver.git
synced 2025-11-06 15:30:28 +03:00
Buildserver snapshot refinements
This commit is contained in:
Ciaran Gultnieks
parent
448fa61d0c
commit
cdfa020ba7
2 changed files with 129 additions and 120 deletions
|
|
@ -49,7 +49,12 @@ def build_server(app, thisbuild, build_dir, output_dir):
|
||||||
output = p.communicate()[0]
|
output = p.communicate()[0]
|
||||||
if output.find('fdroidclean') != -1:
|
if output.find('fdroidclean') != -1:
|
||||||
print "...snapshot exists - resetting build server to clean state"
|
print "...snapshot exists - resetting build server to clean state"
|
||||||
subprocess.call(['vagrant', 'suspend'], cwd='builder')
|
p = subprocess.Popen(['vagrant', 'status'],
|
||||||
|
cwd='builder', stdout=subprocess.PIPE)
|
||||||
|
output = p.communicate()[0]
|
||||||
|
if output.find('running') != -1:
|
||||||
|
print "...suspending"
|
||||||
|
subprocess.call(['vagrant', 'suspend'], cwd='builder')
|
||||||
if subprocess.call(['vagrant', 'snap', 'go', 'fdroidclean'],
|
if subprocess.call(['vagrant', 'snap', 'go', 'fdroidclean'],
|
||||||
cwd='builder') == 0:
|
cwd='builder') == 0:
|
||||||
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
|
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
|
||||||
|
|
@ -89,122 +94,129 @@ def build_server(app, thisbuild, build_dir, output_dir):
|
||||||
if output.find('fdroidclean') == -1:
|
if output.find('fdroidclean') == -1:
|
||||||
raise BuildException("Failed to take snapshot.")
|
raise BuildException("Failed to take snapshot.")
|
||||||
|
|
||||||
# Get SSH configuration settings for us to connect...
|
|
||||||
subprocess.call('vagrant ssh-config >sshconfig',
|
|
||||||
cwd='builder', shell=True)
|
|
||||||
vagranthost = 'default' # Host in ssh config file
|
|
||||||
|
|
||||||
# Load and parse the SSH config...
|
|
||||||
sshconfig = ssh.SSHConfig()
|
|
||||||
sshf = open('builder/sshconfig', 'r')
|
|
||||||
sshconfig.parse(sshf)
|
|
||||||
sshf.close()
|
|
||||||
sshconfig = sshconfig.lookup(vagranthost)
|
|
||||||
|
|
||||||
# Open SSH connection...
|
|
||||||
sshs = ssh.SSHClient()
|
|
||||||
sshs.set_missing_host_key_policy(ssh.AutoAddPolicy())
|
|
||||||
sshs.connect(sshconfig['hostname'], username=sshconfig['user'],
|
|
||||||
port=int(sshconfig['port']), timeout=10, look_for_keys=False,
|
|
||||||
key_filename=sshconfig['identityfile'])
|
|
||||||
|
|
||||||
# Get an SFTP connection...
|
|
||||||
ftp = sshs.open_sftp()
|
|
||||||
ftp.get_channel().settimeout(15)
|
|
||||||
|
|
||||||
# Put all the necessary files in place...
|
|
||||||
ftp.chdir('/home/vagrant')
|
|
||||||
|
|
||||||
# Helper to copy the contents of a directory to the server...
|
|
||||||
def send_dir(path):
|
|
||||||
root = os.path.dirname(path)
|
|
||||||
main = os.path.basename(path)
|
|
||||||
ftp.mkdir(main)
|
|
||||||
for r, d, f in os.walk(path):
|
|
||||||
rr = os.path.relpath(r, root)
|
|
||||||
ftp.chdir(rr)
|
|
||||||
for dd in d:
|
|
||||||
ftp.mkdir(dd)
|
|
||||||
for ff in f:
|
|
||||||
ftp.put(os.path.join(root, rr, ff), ff)
|
|
||||||
for i in range(len(rr.split('/'))):
|
|
||||||
ftp.chdir('..')
|
|
||||||
ftp.chdir('..')
|
|
||||||
|
|
||||||
print "Preparing server for build..."
|
|
||||||
serverpath = os.path.abspath(os.path.dirname(__file__))
|
|
||||||
ftp.put(os.path.join(serverpath, 'build.py'), 'build.py')
|
|
||||||
ftp.put(os.path.join(serverpath, 'common.py'), 'common.py')
|
|
||||||
ftp.put(os.path.join(serverpath, '..', 'config.buildserver.py'), 'config.py')
|
|
||||||
|
|
||||||
# Copy the metadata - just the file for this app...
|
|
||||||
ftp.mkdir('metadata')
|
|
||||||
ftp.chdir('metadata')
|
|
||||||
ftp.put(os.path.join('metadata', app['id'] + '.txt'),
|
|
||||||
app['id'] + '.txt')
|
|
||||||
ftp.chdir('..')
|
|
||||||
# Create the build directory...
|
|
||||||
ftp.mkdir('build')
|
|
||||||
ftp.chdir('build')
|
|
||||||
ftp.mkdir('extlib')
|
|
||||||
# Copy the main app source code
|
|
||||||
if os.path.exists(build_dir):
|
|
||||||
send_dir(build_dir)
|
|
||||||
# Copy any extlibs that are required...
|
|
||||||
if thisbuild.has_key('extlibs'):
|
|
||||||
ftp.chdir('build')
|
|
||||||
ftp.chdir('extlib')
|
|
||||||
for lib in thisbuild['extlibs'].split(';'):
|
|
||||||
lp = lib.split('/')
|
|
||||||
for d in lp[:-1]:
|
|
||||||
ftp.mkdir(d)
|
|
||||||
ftp.chdir(d)
|
|
||||||
ftp.put(os.path.join('build/extlib', lib), lp[-1])
|
|
||||||
for _ in lp[:-1]:
|
|
||||||
ftp.chdir('..')
|
|
||||||
ftp.chdir('..')
|
|
||||||
ftp.chdir('..')
|
|
||||||
# Copy any srclibs that are required...
|
|
||||||
if thisbuild.has_key('srclibs'):
|
|
||||||
ftp.chdir('build')
|
|
||||||
ftp.chdir('extlib')
|
|
||||||
for lib in thisbuild['srclibs'].split(';'):
|
|
||||||
lp = lib.split('@').split('/')
|
|
||||||
for d in lp[:-1]:
|
|
||||||
ftp.mkdir(d)
|
|
||||||
ftp.chdir(d)
|
|
||||||
ftp.put(os.path.join('build/extlib', lib), lp[-1])
|
|
||||||
for _ in lp[:-1]:
|
|
||||||
ftp.chdir('..')
|
|
||||||
ftp.chdir('..')
|
|
||||||
ftp.chdir('..')
|
|
||||||
|
|
||||||
# Execute the build script...
|
|
||||||
print "Starting build..."
|
|
||||||
chan = sshs.get_transport().open_session()
|
|
||||||
stdoutf = chan.makefile('rb')
|
|
||||||
stderrf = chan.makefile_stderr('rb')
|
|
||||||
chan.exec_command('python build.py --on-server -p ' +
|
|
||||||
app['id'] + ' --vercode ' + thisbuild['vercode'])
|
|
||||||
returncode = chan.recv_exit_status()
|
|
||||||
output = stdoutf.read()
|
|
||||||
error = stderrf.read()
|
|
||||||
if returncode != 0:
|
|
||||||
raise BuildException("Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
|
|
||||||
|
|
||||||
# Retrieve the built files...
|
|
||||||
ftp.chdir('/home/vagrant/unsigned')
|
|
||||||
apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
|
|
||||||
tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
|
|
||||||
try:
|
try:
|
||||||
ftp.get(apkfile, os.path.join(output_dir, apkfile))
|
|
||||||
ftp.get(tarball, os.path.join(output_dir, tarball))
|
|
||||||
except:
|
|
||||||
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
|
|
||||||
ftp.close()
|
|
||||||
|
|
||||||
# Suspend the buildserver...
|
# Get SSH configuration settings for us to connect...
|
||||||
subprocess.call(['vagrant', 'suspend'], cwd='builder')
|
print "Getting ssh configuration..."
|
||||||
|
subprocess.call('vagrant ssh-config >sshconfig',
|
||||||
|
cwd='builder', shell=True)
|
||||||
|
vagranthost = 'default' # Host in ssh config file
|
||||||
|
|
||||||
|
# Load and parse the SSH config...
|
||||||
|
sshconfig = ssh.SSHConfig()
|
||||||
|
sshf = open('builder/sshconfig', 'r')
|
||||||
|
sshconfig.parse(sshf)
|
||||||
|
sshf.close()
|
||||||
|
sshconfig = sshconfig.lookup(vagranthost)
|
||||||
|
|
||||||
|
# Open SSH connection...
|
||||||
|
print "Connecting to virtual machine..."
|
||||||
|
sshs = ssh.SSHClient()
|
||||||
|
sshs.set_missing_host_key_policy(ssh.AutoAddPolicy())
|
||||||
|
sshs.connect(sshconfig['hostname'], username=sshconfig['user'],
|
||||||
|
port=int(sshconfig['port']), timeout=10, look_for_keys=False,
|
||||||
|
key_filename=sshconfig['identityfile'])
|
||||||
|
|
||||||
|
# Get an SFTP connection...
|
||||||
|
ftp = sshs.open_sftp()
|
||||||
|
ftp.get_channel().settimeout(15)
|
||||||
|
|
||||||
|
# Put all the necessary files in place...
|
||||||
|
ftp.chdir('/home/vagrant')
|
||||||
|
|
||||||
|
# Helper to copy the contents of a directory to the server...
|
||||||
|
def send_dir(path):
|
||||||
|
root = os.path.dirname(path)
|
||||||
|
main = os.path.basename(path)
|
||||||
|
ftp.mkdir(main)
|
||||||
|
for r, d, f in os.walk(path):
|
||||||
|
rr = os.path.relpath(r, root)
|
||||||
|
ftp.chdir(rr)
|
||||||
|
for dd in d:
|
||||||
|
ftp.mkdir(dd)
|
||||||
|
for ff in f:
|
||||||
|
if not os.path.islink(os.path.join(root, rr, ff)):
|
||||||
|
ftp.put(os.path.join(root, rr, ff), ff)
|
||||||
|
for i in range(len(rr.split('/'))):
|
||||||
|
ftp.chdir('..')
|
||||||
|
ftp.chdir('..')
|
||||||
|
|
||||||
|
print "Preparing server for build..."
|
||||||
|
serverpath = os.path.abspath(os.path.dirname(__file__))
|
||||||
|
ftp.put(os.path.join(serverpath, 'build.py'), 'build.py')
|
||||||
|
ftp.put(os.path.join(serverpath, 'common.py'), 'common.py')
|
||||||
|
ftp.put(os.path.join(serverpath, '..', 'config.buildserver.py'), 'config.py')
|
||||||
|
|
||||||
|
# Copy the metadata - just the file for this app...
|
||||||
|
ftp.mkdir('metadata')
|
||||||
|
ftp.chdir('metadata')
|
||||||
|
ftp.put(os.path.join('metadata', app['id'] + '.txt'),
|
||||||
|
app['id'] + '.txt')
|
||||||
|
ftp.chdir('..')
|
||||||
|
# Create the build directory...
|
||||||
|
ftp.mkdir('build')
|
||||||
|
ftp.chdir('build')
|
||||||
|
ftp.mkdir('extlib')
|
||||||
|
# Copy the main app source code
|
||||||
|
if os.path.exists(build_dir):
|
||||||
|
send_dir(build_dir)
|
||||||
|
# Copy any extlibs that are required...
|
||||||
|
if thisbuild.has_key('extlibs'):
|
||||||
|
ftp.chdir('build')
|
||||||
|
ftp.chdir('extlib')
|
||||||
|
for lib in thisbuild['extlibs'].split(';'):
|
||||||
|
lp = lib.split('/')
|
||||||
|
for d in lp[:-1]:
|
||||||
|
ftp.mkdir(d)
|
||||||
|
ftp.chdir(d)
|
||||||
|
ftp.put(os.path.join('build/extlib', lib), lp[-1])
|
||||||
|
for _ in lp[:-1]:
|
||||||
|
ftp.chdir('..')
|
||||||
|
ftp.chdir('..')
|
||||||
|
ftp.chdir('..')
|
||||||
|
# Copy any srclibs that are required...
|
||||||
|
if thisbuild.has_key('srclibs'):
|
||||||
|
ftp.chdir('build')
|
||||||
|
ftp.chdir('extlib')
|
||||||
|
for lib in thisbuild['srclibs'].split(';'):
|
||||||
|
lp = lib.split('@').split('/')
|
||||||
|
for d in lp[:-1]:
|
||||||
|
ftp.mkdir(d)
|
||||||
|
ftp.chdir(d)
|
||||||
|
ftp.put(os.path.join('build/extlib', lib), lp[-1])
|
||||||
|
for _ in lp[:-1]:
|
||||||
|
ftp.chdir('..')
|
||||||
|
ftp.chdir('..')
|
||||||
|
ftp.chdir('..')
|
||||||
|
|
||||||
|
# Execute the build script...
|
||||||
|
print "Starting build..."
|
||||||
|
chan = sshs.get_transport().open_session()
|
||||||
|
stdoutf = chan.makefile('rb')
|
||||||
|
stderrf = chan.makefile_stderr('rb')
|
||||||
|
chan.exec_command('python build.py --on-server -p ' +
|
||||||
|
app['id'] + ' --vercode ' + thisbuild['vercode'])
|
||||||
|
returncode = chan.recv_exit_status()
|
||||||
|
output = stdoutf.read()
|
||||||
|
error = stderrf.read()
|
||||||
|
if returncode != 0:
|
||||||
|
raise BuildException("Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
|
||||||
|
|
||||||
|
# Retrieve the built files...
|
||||||
|
ftp.chdir('/home/vagrant/unsigned')
|
||||||
|
apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
|
||||||
|
tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
|
||||||
|
try:
|
||||||
|
ftp.get(apkfile, os.path.join(output_dir, apkfile))
|
||||||
|
ftp.get(tarball, os.path.join(output_dir, tarball))
|
||||||
|
except:
|
||||||
|
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
|
||||||
|
ftp.close()
|
||||||
|
|
||||||
|
finally:
|
||||||
|
|
||||||
|
# Suspend the build server.
|
||||||
|
subprocess.call(['vagrant', 'suspend'], cwd='builder')
|
||||||
|
|
||||||
|
|
||||||
def build_local(app, thisbuild, vcs, build_dir, output_dir, extlib_dir, tmp_dir, install, force, verbose=False):
|
def build_local(app, thisbuild, vcs, build_dir, output_dir, extlib_dir, tmp_dir, install, force, verbose=False):
|
||||||
|
|
|
||||||
|
|
@ -2,12 +2,9 @@
|
||||||
set -e
|
set -e
|
||||||
rm -f buildserver.box
|
rm -f buildserver.box
|
||||||
cd buildserver
|
cd buildserver
|
||||||
|
vagrant halt
|
||||||
vagrant up
|
vagrant up
|
||||||
sleep 5
|
vagrant halt
|
||||||
vagrant ssh -c "sudo shutdown -h now"
|
|
||||||
cd ..
|
|
||||||
# Just to wait until it's shut down!
|
|
||||||
sleep 20
|
|
||||||
vagrant package --base `VBoxManage list vms | grep buildserver | sed 's/"\(.*\)".*/\1/'` --output buildserver.box
|
vagrant package --base `VBoxManage list vms | grep buildserver | sed 's/"\(.*\)".*/\1/'` --output buildserver.box
|
||||||
vagrant box add buildserver buildserver.box -f && rm buildserver.box
|
vagrant box add buildserver buildserver.box -f && rm buildserver.box
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue