build-tooling-libs

#!/usr/bin/env python3
# utils/build-tooling-libs - Helper tool for building the SwiftSyntax parser
# and SwiftStaticMirror libraries -*- python -*-
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors

# This is a utility for building only the syntax parser and swift static-mirror
# libraries as fast as possible by only building the necessary dependency libraries
# and nothing extraneous. To achieve this it does a single unified CMake configuration for
# llvm/clang/swift and builds only the required CMake targets for the two libraries.
# This mechanism is fundamentally different from build-script, which builds llvm/clang
# in a separate build directory than swift.
#
# Even though this bypasses build-script, it does share some underlying helper
# utilities from the python infrastructure.
#
# This utility also provides capability to gather profile data and build the parser
# library with PGO optimization enabled.

import copy
import multiprocessing
import os
import platform
import sys

from build_swift.build_swift import argparse
from build_swift.build_swift import defaults
from build_swift.build_swift.constants import SWIFT_BUILD_ROOT
from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT
from build_swift.build_swift.wrappers import xcrun

from swift_build_support.swift_build_support import shell
from swift_build_support.swift_build_support.toolchain import host_toolchain

isDarwin = platform.system() == "Darwin"
isSwiftContainedInLLVMProject = os.path.exists(os.path.join(SWIFT_SOURCE_ROOT, "llvm"))

syntaxParserLibTarget = "tools/swift/tools/libSwiftSyntaxParser/install"
staticMirrorLibTarget = "tools/swift/tools/libStaticMirror/install"
syntaxParserTesterTarget = "swift-syntax-parser-test"

class Builder(object):
    def __init__(
        self, toolchain, args, host, arch, profile_data=None, native_build_dir=None
    ):
        self.toolchain = toolchain
        self.ninja_path = args.ninja_path
        self.build_release = args.release
        self.enable_assertions = not args.no_assertions
        self.lto_type = args.lto_type
        self.pgo_type = args.pgo_type
        self.profile_input = args.profile_input
        self.profile_data = profile_data
        self.dry_run = args.dry_run
        self.jobs = args.build_jobs
        self.verbose = args.verbose
        self.build_dir = args.build_dir
        self.install_destdir = args.install_destdir
        self.install_prefix = args.install_prefix
        self.version = args.version
        self.host = host
        self.arch = arch
        self.native_build_dir = native_build_dir
        self.swiftc_path = args.swiftc_path

    def call(self, command, env=None, without_sleeping=False):
        if without_sleeping:
            shell.call_without_sleeping(
                command, env=env, dry_run=self.dry_run, echo=self.verbose
            )
        else:
            shell.call(command, env=env, dry_run=self.dry_run, echo=self.verbose)

    def configure(self, enable_debuginfo, instrumentation=None, profile_data=None):
        environment = {}
        cmake_args = [self.toolchain.cmake, "-G", "Ninja"]
        cmake_args += ["-DCMAKE_MAKE_PROGRAM=" + self.ninja_path]

        isEmbeddedHost = isDarwin and self.host != "macosx"
        host_triple = None
        host_sdk = None
        llvm_c_flags = "-arch " + self.arch

        if self.host == "macosx":
            deployment_version = "10.12"
            host_triple = "%s-apple-macosx%s" % (self.arch, deployment_version)
            host_sdk = "OSX"
            cmake_args += [
                "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version,
                "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_OSX=" + deployment_version,
            ]
            environment["SDKROOT"] = "macosx"

        elif self.host == "linux":
            host_triple = "%s-unknown-linux" % (self.arch)
            host_sdk = "LINUX"

        elif self.host == "iphonesimulator":
            deployment_version = "10.0"
            host_triple = "%s-apple-ios%s-simulator" % (self.arch, deployment_version)
            host_sdk = "IOS_SIMULATOR"
            cmake_args += [
                "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version,
                "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_IOS=" + deployment_version,
            ]
            llvm_c_flags += " -target " + host_triple

        elif self.host == "iphoneos":
            deployment_version = "10.0"
            host_triple = "%s-apple-ios%s" % (self.arch, deployment_version)
            host_sdk = "IOS"
            cmake_args += [
                "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version,
                "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_IOS=" + deployment_version,
            ]
            llvm_c_flags += " -target " + host_triple

        elif self.host == "appletvsimulator":
            deployment_version = "10.0"
            host_triple = "%s-apple-tvos%s-simulator" % (self.arch, deployment_version)
            host_sdk = "TVOS_SIMULATOR"
            cmake_args += [
                "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version,
                "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_TVOS=" + deployment_version,
            ]
            llvm_c_flags += " -target " + host_triple

        elif self.host == "appletvos":
            deployment_version = "10.0"
            host_triple = "%s-apple-tvos%s" % (self.arch, deployment_version)
            host_sdk = "TVOS"
            cmake_args += [
                "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version,
                "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_TVOS=" + deployment_version,
            ]
            llvm_c_flags += " -target " + host_triple

        elif self.host == "watchsimulator":
            deployment_version = "3.0"
            host_triple = "%s-apple-watchos%s-simulator" % (
                self.arch,
                deployment_version,
            )
            host_sdk = "WATCHOS_SIMULATOR"
            cmake_args += [
                "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version,
                "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_WATCHOS=" + deployment_version,
            ]
            llvm_c_flags += " -target " + host_triple

        elif self.host == "watchos":
            deployment_version = "3.0"
            host_triple = "%s-apple-watchos%s" % (self.arch, deployment_version)
            host_sdk = "WATCHOS"
            cmake_args += [
                "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version,
                "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_WATCHOS=" + deployment_version,
            ]
            llvm_c_flags += " -target " + host_triple

        assert host_triple
        assert host_sdk
        cmake_args += [
            "-DLLVM_HOST_TRIPLE:STRING=" + host_triple,
            "-DLLVM_TARGET_ARCH=" + self.arch,
            "-DSWIFT_HOST_VARIANT=" + self.host,
            "-DSWIFT_HOST_VARIANT_SDK=" + host_sdk,
            "-DSWIFT_HOST_VARIANT_ARCH=" + self.arch,
            "-DCMAKE_Swift_COMPILER_TARGET=" + host_triple,
            "-DCMAKE_Swift_COMPILER=" + self.swiftc_path,
            "-DCMAKE_C_FLAGS=" + llvm_c_flags,
            "-DCMAKE_CXX_FLAGS=" + llvm_c_flags,
        ]
        if isEmbeddedHost:
            cmake_args += [
                "-DCMAKE_OSX_SYSROOT:PATH=" + xcrun.sdk_path(self.host),
                # For embedded hosts CMake runs the checks and triggers crashes because
                # the test binary was built for embedded host.
                "-DHAVE_POSIX_REGEX:BOOL=TRUE",
                "-DHAVE_STEADY_CLOCK:BOOL=TRUE",
            ]

        if isDarwin:
            if self.native_build_dir is not None:
                cmake_args += [
                    "-DLLVM_TABLEGEN="
                    + os.path.join(self.native_build_dir, "bin", "llvm-tblgen"),
                    "-DCLANG_TABLEGEN="
                    + os.path.join(self.native_build_dir, "bin", "clang-tblgen"),
                    "-DLLVM_NATIVE_BUILD=" + self.native_build_dir,
                    "-DSWIFT_NATIVE_LLVM_TOOLS_PATH:STRING="
                    + os.path.join(self.native_build_dir, "bin"),
                    "-DSWIFT_NATIVE_CLANG_TOOLS_PATH:STRING="
                    + os.path.join(self.native_build_dir, "bin"),
                    "-DSWIFT_NATIVE_SWIFT_TOOLS_PATH:STRING="
                    + os.path.join(self.native_build_dir, "bin"),
                ]
        else:
            dispatch_source_path = os.path.join(
                SWIFT_SOURCE_ROOT, "swift-corelibs-libdispatch"
            )
            cmake_args += [
                "-DSWIFT_PATH_TO_LIBDISPATCH_SOURCE:PATH=" + dispatch_source_path,
                "-DLLVM_ENABLE_LLD=ON",
            ]
        cmake_args += ["-DLLVM_TARGETS_TO_BUILD=X86"]
        if self.build_release:
            if enable_debuginfo:
                cmake_args += ["-DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo"]
            else:
                cmake_args += ["-DCMAKE_BUILD_TYPE:STRING=Release"]
        else:
            cmake_args += ["-DCMAKE_BUILD_TYPE:STRING=Debug"]
        if self.enable_assertions:
            cmake_args += ["-DLLVM_ENABLE_ASSERTIONS:BOOL=ON"]
        if instrumentation:
            cmake_args += ["-DLLVM_BUILD_INSTRUMENTED=" + instrumentation]
        if profile_data:
            cmake_args += ["-DLLVM_PROFDATA_FILE=" + profile_data]
        if self.lto_type and not instrumentation:
            cmake_args += ["-DLLVM_ENABLE_LTO=" + self.lto_type.upper()]
        if self.install_prefix:
            cmake_args += ["-DCMAKE_INSTALL_PREFIX:PATH=" + self.install_prefix]
        if self.version:
            cmake_args += ["-DSWIFT_LIBPARSER_VER:STRING=" + self.version]
        cmake_args += [
            "-DLLVM_ENABLE_PROJECTS=clang",
            "-DLLVM_EXTERNAL_PROJECTS=swift",
            '-DLLVM_EXTERNAL_SWIFT_SOURCE_DIR=' + 
            os.path.join(SWIFT_SOURCE_ROOT, 'swift'),
        ]
        cmake_args += [
            "-DSWIFT_BUILD_SYNTAXPARSERLIB=TRUE",
            "-DSWIFT_BUILD_ONLY_SYNTAXPARSERLIB=TRUE",
            "-DSWIFT_BUILD_SYNTAXPARSERTEST=TRUE",
        ]
        cmake_args += ["-DSWIFT_BUILD_PERF_TESTSUITE=NO", "-DSWIFT_INCLUDE_DOCS=NO"]
        cmake_args += [
            "-DSWIFT_BUILD_REMOTE_MIRROR=FALSE",
            "-DSWIFT_BUILD_DYNAMIC_STDLIB=FALSE",
            "-DSWIFT_BUILD_STATIC_STDLIB=FALSE",
            "-DSWIFT_BUILD_DYNAMIC_SDK_OVERLAY=FALSE",
            "-DSWIFT_BUILD_STATIC_SDK_OVERLAY=FALSE",
            "-DSWIFT_BUILD_STDLIB_EXTRA_TOOLCHAIN_CONTENT=FALSE",
        ]
        cmake_args += [
            "-DLLVM_ENABLE_LIBXML2=FALSE",
            "-DLLVM_ENABLE_LIBEDIT=FALSE",
            "-DLLVM_ENABLE_TERMINFO=FALSE",
            "-DLLVM_ENABLE_ZLIB=FALSE",
        ]
        # We are not using cmark but initialize the CMARK variables to something so
        # that configure can succeed.
        cmake_args += [
            "-DCMARK_MAIN_INCLUDE_DIR=" + os.path.join(SWIFT_SOURCE_ROOT, "cmark"),
            "-DCMARK_BUILD_INCLUDE_DIR=" + os.path.join(self.build_dir, "cmark"),
        ]
        cmake_args += [
            "-DLLVM_INCLUDE_TESTS=FALSE",
            "-DCLANG_INCLUDE_TESTS=FALSE",
            "-DSWIFT_INCLUDE_TESTS=FALSE",
        ]
        cmake_args += [
            "-DBOOTSTRAPPING_MODE=HOSTTOOLS",
            "-DEXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR=" +
            os.path.join(SWIFT_SOURCE_ROOT,
                         "swift-experimental-string-processing"),
            "-DSWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE=" +
            os.path.join(SWIFT_SOURCE_ROOT, "swift-syntax"),
        ]

        llvm_src_path = os.path.join(SWIFT_SOURCE_ROOT, "llvm") \
            if isSwiftContainedInLLVMProject \
            else os.path.join(SWIFT_SOURCE_ROOT, "llvm-project", "llvm")
        cmake_args += [llvm_src_path]
        self.call(cmake_args, env=environment)

    def build_targets(self, build_dir, targets, env=None):
        invocation = [self.toolchain.cmake, "--build", build_dir]
        invocation += ["--", "-j%d" % self.jobs]

        # libStaticMirror requires a DESTDIR for the build action
        # Ensure that if one was not provided, we still have a temporary
        # location for it.
        if env is None or (env is not None and env["DESTDIR"] is not None):
            if self.install_destdir:
                env = {"DESTDIR": self.install_destdir}
            else:
                env = {"DESTDIR": os.path.join(build_dir, "install")}

        if self.verbose:
            invocation += ["-v"]
        invocation += targets
        self.call(invocation, env=env, without_sleeping=True)

    def install(self):
        print("--- Installing ---", file=sys.stderr)
        env = None
        if self.install_destdir:
            env = {"DESTDIR": self.install_destdir}
        self.build_targets(
            self.build_dir, [syntaxParserLibTarget,
                             staticMirrorLibTarget], env=env
        )

    def get_profile_data(self, profile_dir):
        shell.makedirs(profile_dir, dry_run=self.dry_run)
        instrumentation = "IR" if self.pgo_type == "ir" else "Frontend"
        with shell.pushd(profile_dir, dry_run=self.dry_run):
            self.configure(enable_debuginfo=False, instrumentation=instrumentation)
            self.build_targets(profile_dir, [syntaxParserTesterTarget])
            # Delete existing profile data that were generated during building from
            # running tablegen.
            shell.rmtree("profiles", dry_run=self.dry_run)
            self.call(
                [
                    os.path.join("bin", syntaxParserTesterTarget),
                    self.profile_input,
                    "-time",
                ]
            )
            self.call(
                [
                    self.toolchain.llvm_profdata,
                    "merge",
                    "-output=profdata.prof",
                    "profiles",
                ]
            )

    def run(self):
        shell.makedirs(self.build_dir, dry_run=self.dry_run)

        with shell.pushd(self.build_dir, dry_run=self.dry_run):
            self.configure(enable_debuginfo=True, profile_data=self.profile_data)

        self.build_targets(self.build_dir, [syntaxParserTesterTarget, staticMirrorLibTarget])

        if self.install_destdir:
            self.install()


def extract_symbols(install_destdir, install_prefix, install_symroot, jobs):
    if not isDarwin:
        return
    extract_script = os.path.join(
        SWIFT_SOURCE_ROOT, "swift", "utils", "parser-lib", "darwin-extract-symbols"
    )
    print("--- Extracting symbols ---", file=sys.stderr)
    env = {
        "INSTALL_DIR": install_destdir,
        "INSTALL_PREFIX": install_prefix,
        "INSTALL_SYMROOT": install_symroot,
        "BUILD_JOBS": str(jobs),
    }
    shell.call([extract_script], env=env)


def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="""
Builds Swift Syntax Parser and Swift Static Mirror libraries.

Example invocations:

* Building for host (macOS, linux):

  $ utils/build-tooling-libs --release --no-assertions --build-dir /tmp/tooling-libs-build

* Building for iOS

  $ utils/build-tooling-libs --release --no-assertions --build-dir \
/tmp/tooling-libs-build-iossim --host iphonesimulator --architectures x86_64
  $ utils/build-tooling-libs --release --no-assertions --build-dir \
/tmp/tooling-libs-build-ios --host iphoneos --architectures arm64

""",
    )
    optbuilder = parser.to_builder()
    option = optbuilder.add_option
    store = optbuilder.actions.store
    store_true = optbuilder.actions.store_true
    store_int = optbuilder.actions.store_int
    store_path = optbuilder.actions.store_path

    toolchain = host_toolchain(xcrun_toolchain="default")
    default_host = "macosx" if isDarwin else "linux"
    default_architectures = platform.machine()

    default_profile_input = os.path.join(
        SWIFT_SOURCE_ROOT, "swift", "utils", "parser-lib", "profile-input.swift"
    )
    default_jobs = multiprocessing.cpu_count()
    default_build_dir = os.path.join(SWIFT_BUILD_ROOT, "tooling-libs")
    default_install_prefix = (
        defaults.DARWIN_INSTALL_PREFIX if isDarwin else defaults.UNIX_INSTALL_PREFIX
    )
    default_ninja = toolchain.ninja
    default_swiftc = toolchain.swiftc

    option("--release", store_true, help="build in release mode")
    option(
        "--lto",
        store("lto_type"),
        choices=["thin", "full"],
        const="full",
        metavar="LTO_TYPE",
        help="use lto optimization."
        "Options: thin, full. If no optional arg is provided, full is "
        "chosen by default",
    )
    option(
        "--pgo",
        store("pgo_type"),
        choices=["frontend", "ir"],
        const="ir",
        metavar="PGO_TYPE",
        help="use pgo optimization."
        "Options: frontend, ir. If no optional arg is provided, ir is "
        "chosen by default",
    )
    option(
        "--profile-input",
        store_path,
        default=default_profile_input,
        help="the source file to use for PGO profiling input (default = %s)"
        % default_profile_input,
    )
    option("--no-assertions", store_true, help="disable assertions")
    option(
        ["-v", "--verbose"],
        store_true,
        help="print the commands executed during the build",
    )
    option(
        "--dry-run",
        store_true,
        help="print the commands to execute but not actually execute them",
    )
    option(
        ["-j", "--jobs"],
        store_int("build_jobs"),
        default=default_jobs,
        help="the number of parallel build jobs to use (default = %s)" % default_jobs,
    )
    option(
        "--build-dir",
        store_path,
        default=default_build_dir,
        help="the path where the build products will be placed. (default = %s)"
        % default_build_dir,
    )
    option(
        "--host",
        store,
        choices=[
            "macosx",
            "linux",
            "iphonesimulator",
            "iphoneos",
            "appletvsimulator",
            "appletvos",
            "watchsimulator",
            "watchos",
        ],
        default=default_host,
        help="host platform to build for (default = %s)" % default_host,
    )
    option(
        "--architectures",
        store,
        default=default_architectures,
        help="space-separated list of architectures to build for. (default = %s)"
        % default_architectures,
    )
    option("--no-install", store_true, help="disable install step")
    option(
        "--install-symroot", store_path, help="the path to install debug symbols into"
    )
    option(
        "--install-destdir",
        store_path,
        help="the path to use as the filesystem root for the installation (default = "
        "'$(build_dir)/install')",
    )
    option(
        "--install-prefix",
        store,
        default=default_install_prefix,
        help="the install prefix to use (default = %s)" % default_install_prefix,
    )
    option("--version", store, help="version string to use for the parser library")
    option(
        "--ninja-path",
        store_path,
        default=default_ninja,
        help="the path to ninja (default = %s)" % default_ninja,
    )
    option(
        "--swiftc-path",
        store_path,
        default=default_swiftc,
        help="""
        Override the swiftc compiler used to build SwiftCompilerSources.
        This is needed to build the most recent state of main if it uses C++ interop 
        features that aren't in the host's toolchain yet. (default = %s)
        """ % default_swiftc
    )

    parser = optbuilder.build()
    args = parser.parse_args()

    if not args.install_destdir and not args.no_install:
        args.install_destdir = os.path.join(args.build_dir, "install")

    if not isSwiftContainedInLLVMProject:
        swift_src_path = os.path.join(SWIFT_SOURCE_ROOT, "swift")
        swift_src_in_llvm_project_path = \
            os.path.join(SWIFT_SOURCE_ROOT, "llvm-project", "swift")
        # Need to symlink 'swift' into 'llvm-project' since we will be doing
        # a unified configure with 'swift' as an external project.
        if not os.path.exists(swift_src_in_llvm_project_path):
            print("Symlinking '%s' to '%s'" %
                  (swift_src_path, swift_src_in_llvm_project_path), file=sys.stderr)
            shell.symlink(swift_src_path, swift_src_in_llvm_project_path,
                          dry_run=args.dry_run, echo=args.verbose)

    architectures = args.architectures.split(" ")
    architectures = [arch for arch in architectures if arch != ""]
    if platform.machine() in architectures:
        # Make sure the machine architecture is at the front.
        architectures.remove(platform.machine())
        architectures = [platform.machine()] + architectures

    if isDarwin:
        objroot = args.build_dir
        dstroot = args.install_destdir
        symroot = args.install_symroot
        prefix = args.install_prefix

        native_build_dir = None
        profile_data = None
        dst_dirs = []

        if args.host == "macosx" and architectures[0] == platform.machine():
            # Build for the native machine.
            arch = architectures.pop(0)
            tmpargs = copy.copy(args)
            tmpargs.build_dir = os.path.join(objroot, arch, "obj")
            if not args.no_install:
                tmpargs.install_destdir = os.path.join(objroot, arch, "dst")
                tmpargs.install_prefix = "/"

            native_build_dir = tmpargs.build_dir
            dst_dirs.append(tmpargs.install_destdir)

            if tmpargs.pgo_type:
                profile_dir = os.path.join(objroot, platform.machine() + "-profiling")
                builder = Builder(toolchain, tmpargs, tmpargs.host, arch)
                builder.get_profile_data(profile_dir)
                profile_data = os.path.join(profile_dir, "profdata.prof")

            builder = Builder(
                toolchain, tmpargs, tmpargs.host, arch, profile_data=profile_data
            )
            builder.run()

        else:
            tmpargs = copy.copy(args)
            if tmpargs.pgo_type:
                # Build for the machine and get profile data.
                native_build_dir = os.path.join(
                    objroot, platform.machine() + "-profiling"
                )
                builder = Builder(toolchain, tmpargs, "macosx", platform.machine())
                builder.get_profile_data(native_build_dir)
                profile_data = os.path.join(native_build_dir, "profdata.prof")
            else:
                # Build the tablegen binaries so we can use them for the cross-compile
                # build.
                native_build_dir = os.path.join(objroot, platform.machine() + "-tblgen")
                tmpargs.lto_type = None
                builder = Builder(toolchain, tmpargs, "macosx", platform.machine())
                shell.makedirs(native_build_dir, dry_run=tmpargs.dry_run)
                with shell.pushd(native_build_dir, dry_run=tmpargs.dry_run):
                    builder.configure(enable_debuginfo=False)
                    builder.build_targets(native_build_dir, ["llvm-tblgen", "clang-tblgen"])

        for arch in architectures:
            args.build_dir = os.path.join(objroot, arch, "obj")
            if not args.no_install:
                args.install_destdir = os.path.join(objroot, arch, "dst")
                args.install_prefix = "/"

            dst_dirs.append(args.install_destdir)

            builder = Builder(
                toolchain,
                args,
                args.host,
                arch,
                profile_data=profile_data,
                native_build_dir=native_build_dir,
            )
            builder.run()

        if not args.no_install:
            lipo = os.path.join(SWIFT_SOURCE_ROOT, "swift", "utils", "recursive-lipo")
            shell.call(
                [lipo, "-v", "--destination", os.path.join(dstroot, "./" + prefix)]
                + dst_dirs
            )

        if args.install_symroot:
            extract_symbols(dstroot, prefix, symroot, args.build_jobs)

        return 0

    assert (
        args.architectures == platform.machine()
    ), "building for non-machine architecture is not supported for non-darwin host"
    builder = Builder(toolchain, args, args.host, args.architectures)
    builder.run()
    return 0


if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        sys.exit(1)