#!/usr/bin/env python2 # -*- coding: utf-8 -*- # # import.py - part of the FDroid server tools # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . import sys import os import shutil import urllib from optparse import OptionParser import common, metadata # Get the repo type and address from the given web page. The page is scanned # in a rather naive manner for 'git clone xxxx', 'hg clone xxxx', etc, and # when one of these is found it's assumed that's the information we want. # Returns repotype, address, or None, reason def getrepofrompage(url): req = urllib.urlopen(url) if req.getcode() != 200: return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode())) page = req.read() # Works for Google Code and BitBucket... index = page.find('hg clone') if index != -1: repotype = 'hg' repo = page[index + 9:] index = repo.find('<') if index == -1: return (None, "Error while getting repo address") repo = repo[:index] repo = repo.split('"')[0] return (repotype, repo) # Works for Google Code and BitBucket... index=page.find('git clone') if index != -1: repotype = 'git' repo = page[index + 10:] index = repo.find('<') if index == -1: return (None, "Error while getting repo address") repo = repo[:index] repo = repo.split('"')[0] return (repotype, repo) # Google Code only... index=page.find('svn checkout') if index != -1: repotype = 'git-svn' repo = page[index + 13:] prefix = 'http' if not repo.startswith(prefix): return (None, "Unexpected checkout instructions format") repo = 'http' + repo[len(prefix):] index = repo.find('<') if index == -1: return (None, "Error while getting repo address - no end tag? '" + repo + "'") sys.exit(1) repo = repo[:index] index = repo.find(' ') if index == -1: return (None, "Error while getting repo address - no space? '" + repo + "'") repo = repo[:index] repo = repo.split('"')[0] return (repotype, repo) return (None, "No information found." + page) config = None options = None def main(): global config, options # Parse command line... parser = OptionParser() parser.add_option("-u", "--url", default=None, help="Project URL to import from.") parser.add_option("-s", "--subdir", default=None, help="Path to main android project subdirectory, if not in root.") parser.add_option("-r", "--repo", default=None, help="Allows a different repo to be specified for a multi-repo google code project") parser.add_option("--rev", default=None, help="Allows a different revision (or git branch) to be specified for the initial import") (options, args) = parser.parse_args() config = common.read_config(options) if not options.url: print "Specify project url." sys.exit(1) url = options.url tmp_dir = 'tmp' if not os.path.isdir(tmp_dir): print "Creating temporary directory" os.makedirs(tmp_dir) # Get all apps... apps = metadata.read_metadata() # Figure out what kind of project it is... projecttype = None issuetracker = None license = None website = url #by default, we might override it if url.startswith('git://'): projecttype = 'git' repo = url repotype = 'git' sourcecode = "" website = "" elif url.startswith('https://github.com'): if url.endswith('/'): url = url[:-1] if url.endswith('.git'): print "A github URL should point to the project, not the git repo" sys.exit(1) projecttype = 'github' repo = url + '.git' repotype = 'git' sourcecode = url issuetracker = url + '/issues' elif url.startswith('https://gitorious.org/'): projecttype = 'gitorious' repo = 'https://git.gitorious.org/' + url[22:] + '.git' repotype = 'git' sourcecode = url elif url.startswith('https://bitbucket.org/'): if url.endswith('/'): url = url[:-1] projecttype = 'bitbucket' sourcecode = url + '/src' issuetracker = url + '/issues' # Figure out the repo type and adddress... repotype, repo = getrepofrompage(sourcecode) if not repotype: print "Unable to determine vcs type. " + repo sys.exit(1) elif url.startswith('http://code.google.com/p/'): if not url.endswith('/'): url += '/'; projecttype = 'googlecode' sourcecode = url + 'source/checkout' if options.repo: sourcecode += "?repo=" + options.repo issuetracker = url + 'issues/list' # Figure out the repo type and adddress... repotype, repo = getrepofrompage(sourcecode) if not repotype: print "Unable to determine vcs type. " + repo sys.exit(1) # Figure out the license... req = urllib.urlopen(url) if req.getcode() != 200: print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode()) sys.exit(1) page = req.read() index = page.find('Code license') if index == -1: print "Couldn't find license data" sys.exit(1) ltext = page[index:] lprefix = 'rel="nofollow">' index = ltext.find(lprefix) if index == -1: print "Couldn't find license text" sys.exit(1) ltext = ltext[index + len(lprefix):] index = ltext.find('<') if index == -1: print "License text not formatted as expected" sys.exit(1) ltext = ltext[:index] if ltext == 'GNU GPL v3': license = 'GPLv3' elif ltext == 'GNU GPL v2': license = 'GPLv2' elif ltext == 'Apache License 2.0': license = 'Apache2' elif ltext == 'MIT License': license = 'MIT' elif ltext == 'GNU Lesser GPL': license = 'LGPL' elif ltext == 'Mozilla Public License 1.1': license = 'MPL' elif ltext == 'New BSD License': license = 'NewBSD' else: print "License " + ltext + " is not recognised" sys.exit(1) if not projecttype: print "Unable to determine the project type." print "The URL you supplied was not in one of the supported formats. Please consult" print "the manual for a list of supported formats, and supply one of those." sys.exit(1) # Get a copy of the source so we can extract some info... print 'Getting source from ' + repotype + ' repo at ' + repo src_dir = os.path.join(tmp_dir, 'importer') if os.path.exists(src_dir): shutil.rmtree(src_dir) vcs = common.getvcs(repotype, repo, src_dir) vcs.gotorevision(options.rev) if options.subdir: root_dir = os.path.join(src_dir, options.subdir) else: root_dir = src_dir # Extract some information... paths = common.manifest_paths(root_dir, None) if not paths: print "No android project could be found. Specify --subdir?" sys.exit(1) version, vercode, package = common.parse_androidmanifests(paths) if not package: print "Couldn't find package ID" sys.exit(1) if not version: print "WARNING: Couldn't find latest version name" if not vercode: print "WARNING: Couldn't find latest version code" # Make sure it's actually new... for app in apps: if app['id'] == package: print "Package " + package + " already exists" sys.exit(1) # Construct the metadata... app = metadata.parse_metadata(None) app['id'] = package app['Web Site'] = website app['Source Code'] = sourcecode if issuetracker: app['Issue Tracker'] = issuetracker if license: app['License'] = license app['Repo Type'] = repotype app['Repo'] = repo app['Update Check Mode'] = "Tags" # Create a build line... build = {} build['version'] = version if version else '?' build['vercode'] = vercode if vercode else '?' build['commit'] = '?' build['disable'] = 'Generated by import.py - check/set version fields and commit id' if options.subdir: build['subdir'] = options.subdir if os.path.exists(os.path.join(root_dir, 'jni')): build['buildjni'] = 'yes' app['builds'].append(build) # Keep the repo directory to save bandwidth... if not os.path.exists('build'): os.mkdir('build') shutil.move(src_dir, os.path.join('build', package)) with open('build/.fdroidvcs-' + package, 'w') as f: f.write(repotype + ' ' + repo) metafile = os.path.join('metadata', package + '.txt') metadata.write_metadata(metafile, app) print "Wrote " + metafile if __name__ == "__main__": main()