#!/usr/bin/python3 ##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===## # # The LLVM Compiler Infrastructure # # This file is distributed under the University of Illinois Open Source # License. See LICENSE.TXT for details. # ##===----------------------------------------------------------------------===## # # This script builds many different flavors of the LLVM ecosystem. It # will build LLVM, Clang, llvm-gcc, and dragonegg as well as run tests # on them. This script is convenient to use to check builds and tests # before committing changes to the upstream repository # # A typical source setup uses three trees and looks like this: # # official # dragonegg # trunk # gcc # trunk # llvm # trunk # tools # clang # tags # RELEASE_28 # tools # clang # llvm-gcc # trunk # tags # RELEASE_28 # staging # dragonegg # trunk # gcc # trunk # llvm # trunk # tools # clang # tags # RELEASE_28 # tools # clang # llvm-gcc # trunk # tags # RELEASE_28 # commit # dragonegg # trunk # gcc # trunk # llvm # trunk # tools # clang # tags # RELEASE_28 # tools # clang # llvm-gcc # trunk # tags # RELEASE_28 # # "gcc" above is the upstream FSF gcc and "gcc/trunk" refers to the # 4.5 branch as discussed in the dragonegg build guide. # # In a typical workflow, the "official" tree always contains unchanged # sources from the main LLVM project repositories. The "staging" tree # is where local work is done. A set of changes resides there waiting # to be moved upstream. The "commit" tree is where changes from # "staging" make their way upstream. Individual incremental changes # from "staging" are applied to "commit" and committed upstream after # a successful build and test run. A successful build is one in which # testing results in no more failures than seen in the testing of the # "official" tree. # # A build may be invoked as such: # # llvmbuild --src=~/llvm/commit --src=~/llvm/staging # --src=~/llvm/official --branch=trunk --branch=tags/RELEASE_28 # --build=debug --build=release --build=paranoid # --prefix=/home/greened/install --builddir=/home/greened/build # # This will build the LLVM ecosystem, including LLVM, Clang, llvm-gcc, # gcc 4.5 and dragonegg, putting build results in ~/build and # installing tools in ~/install. llvmbuild creates separate build and # install directories for each source/branch/build flavor. In the # above example, llvmbuild will build debug, release and paranoid # (debug+checks) flavors of the trunk and RELEASE_28 branches from # each source tree (official, staging and commit) for a total of # eighteen builds. All builds will be run in parallel. # # The user may control parallelism via the --jobs and --threads # switches. --jobs tells llvmbuild the maximum total number of builds # to activate in parallel. The user may think of it as equivalent to # the GNU make -j switch. --threads tells llvmbuild how many worker # threads to use to accomplish those builds. If --threads is less # than --jobs, --threads workers will be launched and each one will # pick a source/branch/flavor combination to build. Then llvmbuild # will invoke GNU make with -j (--jobs / --threads) to use up the # remaining job capacity. Once a worker is finished with a build, it # will pick another combination off the list and start building it. # ##===----------------------------------------------------------------------===## import optparse import os import sys import threading import queue import logging import traceback import subprocess import re # TODO: Use shutil.which when it is available (3.2 or later) def find_executable(executable, path=None): """Try to find 'executable' in the directories listed in 'path' (a string listing directories separated by 'os.pathsep'; defaults to os.environ['PATH']). Returns the complete filename or None if not found """ if path is None: path = os.environ['PATH'] paths = path.split(os.pathsep) extlist = [''] if os.name == 'os2': (base, ext) = os.path.splitext(executable) # executable files on OS/2 can have an arbitrary extension, but # .exe is automatically appended if no dot is present in the name if not ext: executable = executable + ".exe" elif sys.platform == 'win32': pathext = os.environ['PATHEXT'].lower().split(os.pathsep) (base, ext) = os.path.splitext(executable) if ext.lower() not in pathext: extlist = pathext for ext in extlist: execname = executable + ext if os.path.isfile(execname): return execname else: for p in paths: f = os.path.join(p, execname) if os.path.isfile(f): return f else: return None def is_executable(fpath): return os.path.exists(fpath) and os.access(fpath, os.X_OK) def add_options(parser): parser.add_option("-v", "--verbose", action="store_true", default=False, help=("Output informational messages" " [default: %default]")) parser.add_option("--src", action="append", help=("Top-level source directory [default: %default]")) parser.add_option("--build", action="append", default=["debug"], help=("Build types to run [default: %default]")) parser.add_option("--branch", action="append", help=("Source branch to build [default: %default]")) parser.add_option("--cc", default=find_executable("cc"), help=("The C compiler to use [default: %default]")) parser.add_option("--cxx", default=find_executable("c++"), help=("The C++ compiler to use [default: %default]")) parser.add_option("--threads", default=4, type="int", help=("The number of worker threads to use " "[default: %default]")) parser.add_option("--jobs", "-j", default=8, type="int", help=("The number of simultaneous build jobs " "[default: %default]")) parser.add_option("--prefix", help=("Root install directory [default: %default]")) parser.add_option("--builddir", help=("Root build directory [default: %default]")) return def check_options(parser, options, valid_builds): # See if we're building valid flavors. for build in options.build: if (build not in valid_builds): parser.error("'" + build + "' is not a valid build flavor " + str(valid_builds)) # See if we can find source directories. for src in options.src: for component in ["llvm", "llvm-gcc", "gcc", "dragonegg"]: compsrc = src + "/" + component if (not os.path.isdir(compsrc)): parser.error("'" + compsrc + "' does not exist") if (options.branch is not None): for branch in options.branch: if (not os.path.isdir(os.path.join(compsrc, branch))): parser.error("'" + os.path.join(compsrc, branch) + "' does not exist") # See if we can find the compilers options.cc = find_executable(options.cc) options.cxx = find_executable(options.cxx) return # Find a unique short name for the given set of paths. This searches # back through path components until it finds unique component names # among all given paths. def get_path_abbrevs(paths): # Find the number of common starting characters in the last component # of the paths. unique_paths = list(paths) class NotFoundException(Exception): pass # Find a unique component of each path. unique_bases = unique_paths[:] found = 0 while len(unique_paths) > 0: bases = [os.path.basename(src) for src in unique_paths] components = { c for c in bases } # Account for single entry in paths. if len(components) > 1 or len(components) == len(bases): # We found something unique. for c in components: if bases.count(c) == 1: index = bases.index(c) unique_bases[index] = c # Remove the corresponding path from the set under # consideration. unique_paths[index] = None unique_paths = [ p for p in unique_paths if p is not None ] unique_paths = [os.path.dirname(src) for src in unique_paths] if len(unique_paths) > 0: raise NotFoundException() abbrevs = dict(zip(paths, [base for base in unique_bases])) return abbrevs # Given a set of unique names, find a short character sequence that # uniquely identifies them. def get_short_abbrevs(unique_bases): # Find a unique start character for each path base. my_unique_bases = unique_bases[:] unique_char_starts = unique_bases[:] while len(my_unique_bases) > 0: for start, char_tuple in enumerate(zip(*[base for base in my_unique_bases])): chars = { c for c in char_tuple } # Account for single path. if len(chars) > 1 or len(chars) == len(char_tuple): # We found something unique. for c in chars: if char_tuple.count(c) == 1: index = char_tuple.index(c) unique_char_starts[index] = start # Remove the corresponding path from the set under # consideration. my_unique_bases[index] = None my_unique_bases = [ b for b in my_unique_bases if b is not None ] break if len(my_unique_bases) > 0: raise NotFoundException() abbrevs = [abbrev[start_index:start_index+3] for abbrev, start_index in zip([base for base in unique_bases], [index for index in unique_char_starts])] abbrevs = dict(zip(unique_bases, abbrevs)) return abbrevs class Builder(threading.Thread): class ExecutableNotFound(Exception): pass class FileNotExecutable(Exception): pass def __init__(self, work_queue, jobs, cc, cxx, build_abbrev, source_abbrev, branch_abbrev, build_prefix, install_prefix): super().__init__() self.work_queue = work_queue self.jobs = jobs self.cc = cc self.cxx = cxx self.build_abbrev = build_abbrev self.source_abbrev = source_abbrev self.branch_abbrev = branch_abbrev self.build_prefix = build_prefix self.install_prefix = install_prefix self.component_abbrev = dict( llvm="llvm", llvm_gcc="lgcc", llvm2="llv2", gcc="ugcc", dagonegg="degg") def run(self): while True: try: source, branch, build = self.work_queue.get() self.dobuild(source, branch, build) except: traceback.print_exc() finally: self.work_queue.task_done() def execute(self, command, execdir, env, component): prefix = self.component_abbrev[component.replace("-", "_")] pwd = os.getcwd() if not os.path.exists(execdir): os.makedirs(execdir) execenv = os.environ.copy() for key, value in env.items(): execenv[key] = value self.logger.debug("[" + prefix + "] " + "env " + str(env) + " " + " ".join(command)); try: proc = subprocess.Popen(command, cwd=execdir, env=execenv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) line = proc.stdout.readline() while line: self.logger.info("[" + prefix + "] " + str(line, "utf-8").rstrip()) line = proc.stdout.readline() except: traceback.print_exc() # Get a list of C++ include directories to pass to clang. def get_includes(self): # Assume we're building with g++ for now. command = [self.cxx] command += ["-v", "-x", "c++", "/dev/null", "-fsyntax-only"] includes = [] self.logger.debug(command) try: proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) gather = False line = proc.stdout.readline() while line: self.logger.debug(line) if re.search("End of search list", str(line)) is not None: self.logger.debug("Stop Gather") gather = False if gather: includes.append(str(line, "utf-8").strip()) if re.search("#include <...> search starts", str(line)) is not None: self.logger.debug("Start Gather") gather = True line = proc.stdout.readline() except: traceback.print_exc() self.logger.debug(includes) return includes def dobuild(self, source, branch, build): build_suffix = "" ssabbrev = get_short_abbrevs([ab for ab in self.source_abbrev.values()]) if branch is not None: sbabbrev = get_short_abbrevs([ab for ab in self.branch_abbrev.values()]) prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + sbabbrev[self.branch_abbrev[branch]] + "-" + self.build_abbrev[build] + "]" self.install_prefix += "/" + self.source_abbrev[source] + "/" + branch + "/" + build build_suffix += self.source_abbrev[source] + "/" + branch + "/" + build else: prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + self.build_abbrev[build] + "]" self.install_prefix += "/" + self.source_abbrev[source] + "/" + build build_suffix += "/" + self.source_abbrev[source] + "/" + build self.logger = logging.getLogger(prefix) self.logger.debug(self.install_prefix) # Assume we're building with gcc for now. cxxincludes = self.get_includes() cxxroot = cxxincludes[0] cxxarch = os.path.basename(cxxincludes[1]) configure_flags = dict( llvm=dict(debug=["--prefix=" + self.install_prefix, "--with-cxx-include-root=" + cxxroot, "--with-cxx-include-arch=" + cxxarch], release=["--prefix=" + self.install_prefix, "--enable-optimized", "--with-cxx-include-root=" + cxxroot, "--with-cxx-include-arch=" + cxxarch], paranoid=["--prefix=" + self.install_prefix, "--enable-expensive-checks", "--with-cxx-include-root=" + cxxroot, "--with-cxx-include-arch=" + cxxarch]), llvm_gcc=dict(debug=["--prefix=" + self.install_prefix, "--enable-checking", "--program-prefix=llvm-", "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix, "--enable-languages=c,c++,fortran"], release=["--prefix=" + self.install_prefix, "--program-prefix=llvm-", "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix, "--enable-languages=c,c++,fortran"], paranoid=["--prefix=" + self.install_prefix, "--enable-checking", "--program-prefix=llvm-", "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix, "--enable-languages=c,c++,fortran"]), llvm2=dict(debug=["--prefix=" + self.install_prefix, "--with-llvmgccdir=" + self.install_prefix + "/bin", "--with-cxx-include-root=" + cxxroot, "--with-cxx-include-arch=" + cxxarch], release=["--prefix=" + self.install_prefix, "--enable-optimized", "--with-llvmgccdir=" + self.install_prefix + "/bin", "--with-cxx-include-root=" + cxxroot, "--with-cxx-include-arch=" + cxxarch], paranoid=["--prefix=" + self.install_prefix, "--enable-expensive-checks", "--with-llvmgccdir=" + self.install_prefix + "/bin", "--with-cxx-include-root=" + cxxroot, "--with-cxx-include-arch=" + cxxarch]), gcc=dict(debug=["--prefix=" + self.install_prefix, "--enable-checking"], release=["--prefix=" + self.install_prefix], paranoid=["--prefix=" + self.install_prefix, "--enable-checking"]), dragonegg=dict(debug=[], release=[], paranoid=[])) configure_env = dict( llvm=dict(debug=dict(CC=self.cc, CXX=self.cxx), release=dict(CC=self.cc, CXX=self.cxx), paranoid=dict(CC=self.cc, CXX=self.cxx)), llvm_gcc=dict(debug=dict(CC=self.cc, CXX=self.cxx), release=dict(CC=self.cc, CXX=self.cxx), paranoid=dict(CC=self.cc, CXX=self.cxx)), llvm2=dict(debug=dict(CC=self.cc, CXX=self.cxx), release=dict(CC=self.cc, CXX=self.cxx), paranoid=dict(CC=self.cc, CXX=self.cxx)), gcc=dict(debug=dict(CC=self.cc, CXX=self.cxx), release=dict(CC=self.cc, CXX=self.cxx), paranoid=dict(CC=self.cc, CXX=self.cxx)), dragonegg=dict(debug=dict(CC=self.cc, CXX=self.cxx), release=dict(CC=self.cc, CXX=self.cxx), paranoid=dict(CC=self.cc, CXX=self.cxx))) make_flags = dict( llvm=dict(debug=["-j" + str(self.jobs)], release=["-j" + str(self.jobs)], paranoid=["-j" + str(self.jobs)]), llvm_gcc=dict(debug=["-j" + str(self.jobs), "bootstrap"], release=["-j" + str(self.jobs), "bootstrap"], paranoid=["-j" + str(self.jobs), "bootstrap"]), llvm2=dict(debug=["-j" + str(self.jobs)], release=["-j" + str(self.jobs)], paranoid=["-j" + str(self.jobs)]), gcc=dict(debug=["-j" + str(self.jobs), "bootstrap"], release=["-j" + str(self.jobs), "bootstrap"], paranoid=["-j" + str(self.jobs), "bootstrap"]), dragonegg=dict(debug=["-j" + str(self.jobs)], release=["-j" + str(self.jobs)], paranoid=["-j" + str(self.jobs)])) make_env = dict( llvm=dict(debug=dict(), release=dict(), paranoid=dict()), llvm_gcc=dict(debug=dict(), release=dict(), paranoid=dict()), llvm2=dict(debug=dict(), release=dict(), paranoid=dict()), gcc=dict(debug=dict(), release=dict(), paranoid=dict()), dragonegg=dict(debug=dict(GCC=self.install_prefix + "/bin/gcc", LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"), release=dict(GCC=self.install_prefix + "/bin/gcc", LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"), paranoid=dict(GCC=self.install_prefix + "/bin/gcc", LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"))) make_install_flags = dict( llvm=dict(debug=["install"], release=["install"], paranoid=["install"]), llvm_gcc=dict(debug=["install"], release=["install"], paranoid=["install"]), llvm2=dict(debug=["install"], release=["install"], paranoid=["install"]), gcc=dict(debug=["install"], release=["install"], paranoid=["install"]), dragonegg=dict(debug=["install"], release=["install"], paranoid=["install"])) make_install_env = dict( llvm=dict(debug=dict(), release=dict(), paranoid=dict()), llvm_gcc=dict(debug=dict(), release=dict(), paranoid=dict()), llvm2=dict(debug=dict(), release=dict(), paranoid=dict()), gcc=dict(debug=dict(), release=dict(), paranoid=dict()), dragonegg=dict(debug=dict(), release=dict(), paranoid=dict())) make_check_flags = dict( llvm=dict(debug=["check"], release=["check"], paranoid=["check"]), llvm_gcc=dict(debug=["check"], release=["check"], paranoid=["check"]), llvm2=dict(debug=["check"], release=["check"], paranoid=["check"]), gcc=dict(debug=["check"], release=["check"], paranoid=["check"]), dragonegg=dict(debug=["check"], release=["check"], paranoid=["check"])) make_check_env = dict( llvm=dict(debug=dict(), release=dict(), paranoid=dict()), llvm_gcc=dict(debug=dict(), release=dict(), paranoid=dict()), llvm2=dict(debug=dict(), release=dict(), paranoid=dict()), gcc=dict(debug=dict(), release=dict(), paranoid=dict()), dragonegg=dict(debug=dict(), release=dict(), paranoid=dict())) for component in ["llvm", "llvm-gcc", "llvm2", "gcc", "dragonegg"]: comp = component[:] srcdir = source + "/" + comp.rstrip("2") builddir = self.build_prefix + "/" + comp + "/" + build_suffix installdir = self.install_prefix if (branch is not None): srcdir += "/" + branch self.logger.info("Configuring " + component + " in " + builddir) self.configure(component, srcdir, builddir, configure_flags[comp.replace("-", "_")][build], configure_env[comp.replace("-", "_")][build]) self.logger.info("Building " + component + " in " + builddir) self.make(component, srcdir, builddir, make_flags[comp.replace("-", "_")][build], make_env[comp.replace("-", "_")][build]) self.logger.info("Installing " + component + " in " + installdir) self.make(component, srcdir, builddir, make_install_flags[comp.replace("-", "_")][build], make_install_env[comp.replace("-", "_")][build]) self.logger.info("Testing " + component + " in " + builddir) self.make(component, srcdir, builddir, make_check_flags[comp.replace("-", "_")][build], make_check_env[comp.replace("-", "_")][build]) def configure(self, component, srcdir, builddir, flags, env): configure_files = dict( llvm=[(srcdir + "/configure", builddir + "/Makefile")], llvm_gcc=[(srcdir + "/configure", builddir + "/Makefile"), (srcdir + "/gcc/configure", builddir + "/gcc/Makefile")], llvm2=[(srcdir + "/configure", builddir + "/Makefile")], gcc=[(srcdir + "/configure", builddir + "/Makefile"), (srcdir + "/gcc/configure", builddir + "/gcc/Makefile")], dragonegg=[()]) doconfig = False for conf, mf in configure_files[component.replace("-", "_")]: if os.path.exists(conf) and os.path.exists(mf): confstat = os.stat(conf) makestat = os.stat(mf) if confstat.st_mtime > makestat.st_mtime: doconfig = True break else: doconfig = True break if not doconfig: return program = srcdir + "/configure" if not is_executable(program): return args = [program] args += ["--verbose"] args += flags self.execute(args, builddir, env, component) def make(self, component, srcdir, builddir, flags, env): program = find_executable("make") if program is None: raise ExecutableNotFound if not is_executable(program): raise FileNotExecutable args = [program] args += flags self.execute(args, builddir, env, component) # Global constants build_abbrev = dict(debug="dbg", release="opt", paranoid="par") # Parse options parser = optparse.OptionParser(version="%prog 1.0") add_options(parser) (options, args) = parser.parse_args() check_options(parser, options, build_abbrev.keys()); if options.verbose: logging.basicConfig(level=logging.DEBUG, format='%(name)-13s: %(message)s') else: logging.basicConfig(level=logging.INFO, format='%(name)-13s: %(message)s') source_abbrev = get_path_abbrevs(set(options.src)) branch_abbrev = get_path_abbrevs(set(options.branch)) work_queue = queue.Queue() for t in range(options.threads): jobs = options.jobs // options.threads builder = Builder(work_queue, jobs, options.cc.strip(), options.cxx.strip(), build_abbrev, source_abbrev, branch_abbrev, options.builddir.strip(), options.prefix.strip()) builder.daemon = True builder.start() for build in set(options.build): for source in set(options.src): if options.branch is not None: for branch in set(options.branch): work_queue.put((source, branch, build)) else: work_queue.put((source, None, build)) work_queue.join()