#!/usr/bin/env python # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. """ Setup mozbase packages for development. Packages may be specified as command line arguments. If no arguments are given, install all packages. See https://wiki.mozilla.org/Auto-tools/Projects/Mozbase """ import os import subprocess import sys from optparse import OptionParser from subprocess import PIPE try: from subprocess import check_call as call except ImportError: from subprocess import call # directory containing this file here = os.path.dirname(os.path.abspath(__file__)) # all python packages mozbase_packages = [i for i in os.listdir(here) if os.path.exists(os.path.join(here, i, 'setup.py'))] test_packages = [ "mock" # testing: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Tests ] extra_packages = [ "sphinx" # documentation: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Documentation ] def cycle_check(order, dependencies): """ensure no cyclic dependencies""" order_dict = dict([(j, i) for i, j in enumerate(order)]) for package, deps in dependencies.items(): index = order_dict[package] for d in deps: assert index > order_dict[d], "Cyclic dependencies detected" def info(directory): "get the package setup.py information" assert os.path.exists(os.path.join(directory, 'setup.py')) # setup the egg info try: call([sys.executable, 'setup.py', 'egg_info'], cwd=directory, stdout=PIPE) except subprocess.CalledProcessError: print "Error running setup.py in %s" % directory raise # get the .egg-info directory egg_info = [entry for entry in os.listdir(directory) if entry.endswith('.egg-info')] assert len(egg_info) == 1, 'Expected one .egg-info directory in %s, got: %s' % (directory, egg_info) egg_info = os.path.join(directory, egg_info[0]) assert os.path.isdir(egg_info), "%s is not a directory" % egg_info # read the package information pkg_info = os.path.join(egg_info, 'PKG-INFO') info_dict = {} for line in file(pkg_info).readlines(): if not line or line[0].isspace(): continue # XXX neglects description assert ':' in line key, value = [i.strip() for i in line.split(':', 1)] info_dict[key] = value return info_dict def get_dependencies(directory): "returns the package name and dependencies given a package directory" # get the package metadata info_dict = info(directory) # get the .egg-info directory egg_info = [entry for entry in os.listdir(directory) if entry.endswith('.egg-info')][0] # read the dependencies requires = os.path.join(directory, egg_info, 'requires.txt') dependencies = [] if os.path.exists(requires): for line in file(requires): line = line.strip() # in requires.txt file, a dependency is a non empty line # Also lines like [device] are sections to mark optional # dependencies, we don't want those sections. if line and not (line.startswith('[') and line.endswith(']')): dependencies.append(line) # return the information return info_dict['Name'], dependencies def dependency_info(dep): "return dictionary of dependency information from a dependency string" retval = dict(Name=None, Type=None, Version=None) for joiner in ('==', '<=', '>='): if joiner in dep: retval['Type'] = joiner name, version = [i.strip() for i in dep.split(joiner, 1)] retval['Name'] = name retval['Version'] = version break else: retval['Name'] = dep.strip() return retval def unroll_dependencies(dependencies): """ unroll a set of dependencies to a flat list dependencies = {'packageA': set(['packageB', 'packageC', 'packageF']), 'packageB': set(['packageC', 'packageD', 'packageE', 'packageG']), 'packageC': set(['packageE']), 'packageE': set(['packageF', 'packageG']), 'packageF': set(['packageG']), 'packageX': set(['packageA', 'packageG'])} """ order = [] # flatten all packages = set(dependencies.keys()) for deps in dependencies.values(): packages.update(deps) while len(order) != len(packages): for package in packages.difference(order): if set(dependencies.get(package, set())).issubset(order): order.append(package) break else: raise AssertionError("Cyclic dependencies detected") cycle_check(order, dependencies) # sanity check return order def main(args=sys.argv[1:]): # parse command line options usage = '%prog [options] [package] [package] [...]' parser = OptionParser(usage=usage, description=__doc__) parser.add_option('-d', '--dependencies', dest='list_dependencies', action='store_true', default=False, help="list dependencies for the packages") parser.add_option('--list', action='store_true', default=False, help="list what will be installed") parser.add_option('--extra', '--install-extra-packages', action='store_true', default=False, help="installs extra supporting packages as well as core mozbase ones") options, packages = parser.parse_args(args) if not packages: # install all packages packages = sorted(mozbase_packages) # ensure specified packages are in the list assert set(packages).issubset(mozbase_packages), "Packages should be in %s (You gave: %s)" % (mozbase_packages, packages) if options.list_dependencies: # list the package dependencies for package in packages: print '%s: %s' % get_dependencies(os.path.join(here, package)) parser.exit() # gather dependencies # TODO: version conflict checking deps = {} alldeps = {} mapping = {} # mapping from subdir name to package name # core dependencies for package in packages: key, value = get_dependencies(os.path.join(here, package)) deps[key] = [dependency_info(dep)['Name'] for dep in value] mapping[package] = key # keep track of all dependencies for non-mozbase packages for dep in value: alldeps[dependency_info(dep)['Name']] = ''.join(dep.split()) # indirect dependencies flag = True while flag: flag = False for value in deps.values(): for dep in value: if dep in mozbase_packages and dep not in deps: key, value = get_dependencies(os.path.join(here, dep)) deps[key] = [dep for dep in value] for dep in value: alldeps[dep] = ''.join(dep.split()) mapping[package] = key flag = True break if flag: break # get the remaining names for the mapping for package in mozbase_packages: if package in mapping: continue key, value = get_dependencies(os.path.join(here, package)) mapping[package] = key # unroll dependencies unrolled = unroll_dependencies(deps) # make a reverse mapping: package name -> subdirectory reverse_mapping = dict([(j,i) for i, j in mapping.items()]) # we only care about dependencies in mozbase unrolled = [package for package in unrolled if package in reverse_mapping] if options.list: # list what will be installed for package in unrolled: print package parser.exit() # set up the packages for development for package in unrolled: call([sys.executable, 'setup.py', 'develop', '--no-deps'], cwd=os.path.join(here, reverse_mapping[package])) # add the directory of sys.executable to path to aid the correct # `easy_install` getting called # https://bugzilla.mozilla.org/show_bug.cgi?id=893878 os.environ['PATH'] = '%s%s%s' % (os.path.dirname(os.path.abspath(sys.executable)), os.path.pathsep, os.environ.get('PATH', '').strip(os.path.pathsep)) # install non-mozbase dependencies # these need to be installed separately and the --no-deps flag # subsequently used due to a bug in setuptools; see # https://bugzilla.mozilla.org/show_bug.cgi?id=759836 pypi_deps = dict([(i, j) for i,j in alldeps.items() if i not in unrolled]) for package, version in pypi_deps.items(): # easy_install should be available since we rely on setuptools call(['easy_install', version]) # install packages required for unit testing for package in test_packages: call(['easy_install', package]) # install extra non-mozbase packages if desired if options.extra: for package in extra_packages: call(['easy_install', package]) if __name__ == '__main__': main()