#!/usr/bin/python2.7 # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os import os.path import shutil import subprocess import platform import json import argparse import tempfile import glob import errno from contextlib import contextmanager import sys DEBUG = os.getenv("DEBUG") def check_run(args): global DEBUG if DEBUG: print >> sys.stderr, ' '.join(args) r = subprocess.call(args) assert r == 0 def run_in(path, args): d = os.getcwd() global DEBUG if DEBUG: print >> sys.stderr, 'cd "%s"' % path os.chdir(path) check_run(args) if DEBUG: print >> sys.stderr, 'cd "%s"' % d os.chdir(d) def patch(patch, srcdir): patch = os.path.realpath(patch) check_run(['patch', '-d', srcdir, '-p1', '-i', patch, '--fuzz=0', '-s']) def build_package(package_build_dir, run_cmake, cmake_args): if not os.path.exists(package_build_dir): os.mkdir(package_build_dir) if run_cmake: run_in(package_build_dir, ["cmake"] + cmake_args) run_in(package_build_dir, ["ninja", "install"]) @contextmanager def updated_env(env): old_env = os.environ.copy() os.environ.update(env) yield os.environ.clear() os.environ.update(old_env) def build_tar_package(tar, name, base, directory): name = os.path.realpath(name) run_in(base, [tar, "-c", "-%s" % ("J" if ".xz" in name else "j"), "-f", name, directory]) def copy_dir_contents(src, dest): for f in glob.glob("%s/*" % src): try: destname = "%s/%s" % (dest, os.path.basename(f)) shutil.copytree(f, destname) except OSError as e: if e.errno == errno.ENOTDIR: shutil.copy2(f, destname) elif e.errno == errno.EEXIST: if os.path.isdir(f): copy_dir_contents(f, destname) else: os.remove(destname) shutil.copy2(f, destname) else: raise Exception('Directory not copied. Error: %s' % e) def mkdir_p(path): try: os.makedirs(path) except OSError as e: if e.errno != errno.EEXIST or not os.path.isdir(path): raise def build_and_use_libgcc(env, clang_dir): with updated_env(env): tempdir = tempfile.mkdtemp() gcc_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "build-gcc") run_in(gcc_dir, ["./build-gcc.sh", tempdir, "libgcc"]) run_in(tempdir, ["tar", "-xf", "gcc.tar.xz"]) libgcc_dir = glob.glob(os.path.join(tempdir, "gcc", "lib", "gcc", "x86_64-unknown-linux-gnu", "[0-9]*"))[0] clang_lib_dir = os.path.join(clang_dir, "lib", "gcc", "x86_64-unknown-linux-gnu", os.path.basename(libgcc_dir)) mkdir_p(clang_lib_dir) copy_dir_contents(libgcc_dir, clang_lib_dir) libgcc_dir = os.path.join(tempdir, "gcc", "lib64") clang_lib_dir = os.path.join(clang_dir, "lib") copy_dir_contents(libgcc_dir, clang_lib_dir) include_dir = os.path.join(tempdir, "gcc", "include") clang_include_dir = os.path.join(clang_dir, "include") copy_dir_contents(include_dir, clang_include_dir) shutil.rmtree(tempdir) def svn_co(url, directory, revision): check_run(["svn", "co", "-r", revision, url, directory]) def svn_update(directory, revision): run_in(directory, ["svn", "update", "-r", revision]) def build_one_stage(env, src_dir, stage_dir, build_libcxx, build_type, assertions, python_path): with updated_env(env): build_one_stage_aux(src_dir, stage_dir, build_libcxx, build_type, assertions, python_path) def get_platform(): p = platform.system() if p == "Darwin": return "macosx64" elif p == "Linux": if platform.processor() == "x86_64": return "linux64" else: return "linux32" else: raise NotImplementedError("Not supported platform") def is_darwin(): return platform.system() == "Darwin" def build_one_stage_aux(src_dir, stage_dir, build_libcxx, build_type, assertions, python_path): if not os.path.exists(stage_dir): os.mkdir(stage_dir) build_dir = stage_dir + "/build" inst_dir = stage_dir + "/clang" run_cmake = True if os.path.exists(build_dir): run_cmake = False cmake_args = ["-GNinja", "-DCMAKE_BUILD_TYPE=%s" % build_type, "-DLLVM_TARGETS_TO_BUILD=X86;ARM", "-DLLVM_ENABLE_ASSERTIONS=%s" % ("ON" if assertions else "OFF"), "-DPYTHON_EXECUTABLE=%s" % python_path, "-DCMAKE_INSTALL_PREFIX=%s" % inst_dir, "-DLLVM_TOOL_LIBCXX_BUILD=%s" % ("ON" if build_libcxx else "OFF"), "-DLIBCXX_LIBCPPABI_VERSION=\"\"", src_dir]; build_package(build_dir, run_cmake, cmake_args) if __name__ == "__main__": # The directories end up in the debug info, so the easy way of getting # a reproducible build is to run it in a know absolute directory. # We use a directory in /builds/slave because the mozilla infrastructure # cleans it up automatically. base_dir = "/builds/slave/moz-toolchain" source_dir = base_dir + "/src" build_dir = base_dir + "/build" llvm_source_dir = source_dir + "/llvm" clang_source_dir = source_dir + "/clang" compiler_rt_source_dir = source_dir + "/compiler-rt" libcxx_source_dir = source_dir + "/libcxx" if is_darwin(): os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.7' parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', required=True, type=argparse.FileType('r'), help="Clang configuration file") parser.add_argument('--clean', required=False, action='store_true', help="Clean the build directory") args = parser.parse_args() config = json.load(args.config) if args.clean: shutil.rmtree(build_dir) os.sys.exit(0) llvm_revision = config["llvm_revision"] llvm_repo = config["llvm_repo"] clang_repo = config["clang_repo"] compiler_repo = config["compiler_repo"] libcxx_repo = config["libcxx_repo"] stages = 3 if "stages" in config: stages = int(config["stages"]) if stages not in (1, 2, 3): raise ValueError("We only know how to build 1, 2, or 3 stages") build_type = "Release" if "build_type" in config: build_type = config["build_type"] if build_type not in ("Release", "Debug", "RelWithDebInfo", "MinSizeRel"): raise ValueError("We only know how to do Release, Debug, RelWithDebInfo or MinSizeRel builds") build_libcxx = False if "build_libcxx" in config: build_libcxx = config["build_libcxx"] if build_libcxx not in (True, False): raise ValueError("Only boolean values are accepted for build_libcxx.") assertions = False if "assertions" in config: assertions = config["assertions"] if assertions not in (True, False): raise ValueError("Only boolean values are accepted for assertions.") python_path = None if "python_path" not in config: raise ValueError("Config file needs to set python_path") python_path = config["python_path"] gcc_dir = None if "gcc_dir" in config: gcc_dir = config["gcc_dir"] if not os.path.exists(gcc_dir): raise ValueError("gcc_dir must point to an existing path") if not is_darwin() and gcc_dir is None: raise ValueError("Config file needs to set gcc_dir") cc = None if "cc" in config: cc = config["cc"] if not os.path.exists(cc): raise ValueError("cc must point to an existing path") else: raise ValueError("Config file needs to set cc") cxx = None if "cxx" in config: cxx = config["cxx"] if not os.path.exists(cxx): raise ValueError("cxx must point to an existing path") else: raise ValueError("Config file needs to set cxx") if not os.path.exists(source_dir): os.makedirs(source_dir) svn_co(llvm_repo, llvm_source_dir, llvm_revision) svn_co(clang_repo, clang_source_dir, llvm_revision) svn_co(compiler_repo, compiler_rt_source_dir, llvm_revision) svn_co(libcxx_repo, libcxx_source_dir, llvm_revision) os.symlink("../../clang", llvm_source_dir + "/tools/clang") os.symlink("../../compiler-rt", llvm_source_dir + "/projects/compiler-rt") os.symlink("../../libcxx", llvm_source_dir + "/projects/libcxx") for p in config.get("patches", {}).get(get_platform(), []): patch(p, source_dir) else: svn_update(llvm_source_dir, llvm_revision) svn_update(clang_source_dir, llvm_revision) svn_update(compiler_rt_source_dir, llvm_revision) svn_update(libcxx_source_dir, llvm_revision) if not os.path.exists(build_dir): os.makedirs(build_dir) stage1_dir = build_dir + '/stage1' stage1_inst_dir = stage1_dir + '/clang' final_stage_dir = stage1_dir if is_darwin(): extra_cflags = "" extra_cxxflags = "-stdlib=libc++" extra_cflags2 = "" extra_cxxflags2 = "-stdlib=libc++" else: extra_cflags = "-static-libgcc" extra_cxxflags = "-static-libgcc -static-libstdc++" extra_cflags2 = "-fPIC --gcc-toolchain=%s" % gcc_dir extra_cxxflags2 = "-fPIC --gcc-toolchain=%s" % gcc_dir if os.environ.has_key('LD_LIBRARY_PATH'): os.environ['LD_LIBRARY_PATH'] = '%s/lib64/:%s' % (gcc_dir, os.environ['LD_LIBRARY_PATH']); else: os.environ['LD_LIBRARY_PATH'] = '%s/lib64/' % gcc_dir build_one_stage( {"CC": cc + " %s" % extra_cflags, "CXX": cxx + " %s" % extra_cxxflags}, llvm_source_dir, stage1_dir, build_libcxx, build_type, assertions, python_path) if stages > 1: stage2_dir = build_dir + '/stage2' stage2_inst_dir = stage2_dir + '/clang' final_stage_dir = stage2_dir build_one_stage( {"CC": stage1_inst_dir + "/bin/clang %s" % extra_cflags2, "CXX": stage1_inst_dir + "/bin/clang++ %s" % extra_cxxflags2}, llvm_source_dir, stage2_dir, build_libcxx, build_type, assertions, python_path) if stages > 2: stage3_dir = build_dir + '/stage3' final_stage_dir = stage3_dir build_one_stage( {"CC": stage2_inst_dir + "/bin/clang %s" % extra_cflags2, "CXX": stage2_inst_dir + "/bin/clang++ %s" % extra_cxxflags2}, llvm_source_dir, stage3_dir, build_libcxx, build_type, assertions, python_path) if not is_darwin(): final_stage_inst_dir = final_stage_dir + '/clang' build_and_use_libgcc( {"CC": cc + " %s" % extra_cflags, "CXX": cxx + " %s" % extra_cxxflags}, final_stage_inst_dir) if is_darwin(): build_tar_package("tar", "clang.tar.bz2", final_stage_dir, "clang") else: build_tar_package("tar", "clang.tar.xz", final_stage_dir, "clang")