build-script

#!/usr/bin/env python

# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors


"""
The ultimate tool for building Swift.
"""


from __future__ import absolute_import, print_function, unicode_literals

import json
import os
import platform
import sys
import time

from build_swift.build_swift import argparse
from build_swift.build_swift import defaults
from build_swift.build_swift import driver_arguments
from build_swift.build_swift import migration
from build_swift.build_swift import presets
from build_swift.build_swift.constants import BUILD_SCRIPT_IMPL_PATH
from build_swift.build_swift.constants import SWIFT_BUILD_ROOT
from build_swift.build_swift.constants import SWIFT_REPO_NAME
from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT

import six

from swift_build_support.swift_build_support import build_script_invocation
from swift_build_support.swift_build_support import shell
from swift_build_support.swift_build_support import targets
from swift_build_support.swift_build_support import workspace
from swift_build_support.swift_build_support.cmake import CMake
from swift_build_support.swift_build_support.targets import StdlibDeploymentTarget
from swift_build_support.swift_build_support.toolchain import host_toolchain
from swift_build_support.swift_build_support.utils \
    import exit_rejecting_arguments
from swift_build_support.swift_build_support.utils import fatal_error


# -----------------------------------------------------------------------------
# Constants

# These versions are community sourced. At any given time only the Xcode
# version used by Swift CI is officially supported. See ci.swift.org
_SUPPORTED_XCODE_BUILDS = [
    ("12.3", "12C33"),
    ("12.4", "12D4e"),
    ("12.5", "12E262"),
    ("13.0 beta", "13A5154h"),
    ("13.0 beta 4", "13A5201i"),
    ("13.0", "13A233"),
    ("13.1 RC 1", "13A1030d"),
    ("13.2 beta", "13C5066c")
]

# -----------------------------------------------------------------------------
# Helpers


def print_note(message, stream=sys.stdout):
    """Writes a diagnostic message to the given stream. By default this
    function outputs to stdout.
    """

    stream.write('[{}] NOTE: {}\n'.format(sys.argv[0], message))
    stream.flush()


def clean_delay():
    """Provide a short delay so accidentally invoked clean builds can be
    canceled.
    """

    sys.stdout.write('Starting clean build in  ')
    for i in range(3, 0, -1):
        sys.stdout.write('\b%d' % i)
        sys.stdout.flush()
        time.sleep(1)
    print('\b\b\b\bnow.')


def initialize_runtime_environment():
    """Change the program environment for building.
    """

    # Set an appropriate default umask.
    os.umask(0o022)

    # Unset environment variables that might affect how tools behave.
    for v in [
            'MAKEFLAGS',
            'SDKROOT',
            'MACOSX_DEPLOYMENT_TARGET',
            'IPHONEOS_DEPLOYMENT_TARGET',
            'TVOS_DEPLOYMENT_TARGET',
            'WATCHOS_DEPLOYMENT_TARGET']:
        os.environ.pop(v, None)

    # Provide a default NINJA_STATUS to format ninja output.
    if 'NINJA_STATUS' not in os.environ:
        os.environ['NINJA_STATUS'] = '[%f/%t][%p][%es] '


class JSONDumper(json.JSONEncoder):
    def __init__(self, *args, **kwargs):
        json.JSONEncoder.__init__(
            self, indent=2, separators=(',', ': '), sort_keys=True,
            *args, **kwargs)

    def default(self, o):
        if hasattr(o, '__dict__'):
            return vars(o)
        return six.text_type(o)


def print_xcodebuild_versions(file=sys.stdout):
    """
    Print the host machine's `xcodebuild` version, as well as version
    information for all available SDKs.
    """
    version = shell.capture(
        ['xcodebuild', '-version'], dry_run=False, echo=False).rstrip()
    # Allow non-zero exit codes.  Under certain obscure circumstances
    # xcodebuild can exit with a non-zero exit code even when the SDK is
    # usable.
    sdks = shell.capture(
        ['xcodebuild', '-version', '-sdk'], dry_run=False, echo=False,
        allow_non_zero_exit=True).rstrip()
    fmt = '{version}\n\n--- SDK versions ---\n{sdks}\n'

    print(fmt.format(version=version, sdks=sdks), file=file)
    file.flush()


def validate_xcode_compatibility():
    if sys.platform != 'darwin':
        return

    if os.getenv("SKIP_XCODE_VERSION_CHECK"):
        print("note: skipping Xcode version check")
        return

    version = shell.capture(
        ['xcodebuild', '-version'], dry_run=False, echo=False).strip()

    valid_build_numbers = tuple(x[1] for x in _SUPPORTED_XCODE_BUILDS)
    if not version.endswith(valid_build_numbers):
        valid_versions_string = "\n".join(
            "{} ({})".format(*x) for x in _SUPPORTED_XCODE_BUILDS)
        raise SystemExit(
            "error: using unsupported Xcode version:\n\n{}\n\n"
            "Install one of:\n{}\n\n"
            "Or set 'SKIP_XCODE_VERSION_CHECK=1' in the environment".format(
                version, valid_versions_string
            )
        )


def tar(source, destination):
    """
    Create a gzip archive of the file at 'source' at the given
    'destination' path.
    """
    # We do not use `tarfile` here because:
    #  - We wish to support LZMA2 compression while also supporting Python 2.7.
    #  - We wish to explicitly set the owner and group of the archive.
    args = ['tar', '-c', '-z', '-f', destination]

    if platform.system() != 'Darwin' and platform.system() != 'Windows':
        args += ['--owner=0', '--group=0']

    # Discard stderr output such as 'tar: Failed to open ...'. We'll detect
    # these cases using the exit code, which should cause 'shell.call' to
    # raise.
    shell.call(args + [source], stderr=shell.DEVNULL)


# -----------------------------------------------------------------------------
# Argument Validation

def validate_arguments(toolchain, args):
    if toolchain.cc is None or toolchain.cxx is None:
        fatal_error(
            "can't find clang (please install clang-3.5 or a "
            "later version)")

    if toolchain.cmake is None:
        fatal_error("can't find CMake (please install CMake)")

    if args.distcc:
        if toolchain.distcc is None:
            fatal_error(
                "can't find distcc (please install distcc)")
        if toolchain.distcc_pump is None:
            fatal_error(
                "can't find distcc-pump (please install distcc-pump)")

    if args.sccache:
        if toolchain.sccache is None:
            fatal_error(
                "can't find sccache (please install sccache)")

    if args.host_target is None or args.stdlib_deployment_targets is None:
        fatal_error("unknown operating system")

    if args.symbols_package:
        if not os.path.isabs(args.symbols_package):
            print(
                '--symbols-package must be an absolute path '
                '(was \'{}\')'.format(args.symbols_package))
            return 1
        if not args.install_symroot:
            fatal_error(
                "--install-symroot is required when specifying "
                "--symbols-package.")

    if args.android:
        if args.android_ndk is None or \
                args.android_api_level is None or \
                args.android_icu_uc is None or \
                args.android_icu_uc_include is None or \
                args.android_icu_i18n is None or \
                args.android_icu_i18n_include is None or \
                args.android_icu_data is None:
            fatal_error(
                "when building for Android, --android-ndk, "
                "--android-api-level, --android-icu-uc, "
                "--android-icu-uc-include, --android-icu-i18n, "
                "--android-icu-i18n-include, and --android-icu-data "
                "must be specified")

    targets_needing_toolchain = [
        'build_indexstoredb',
        'build_playgroundsupport',
        'build_sourcekitlsp',
        'build_toolchainbenchmarks',
        'build_swift_inspect',
        'tsan_libdispatch_test',
    ]
    has_target_needing_toolchain = \
        bool(sum(getattr(args, x) for x in targets_needing_toolchain))
    if args.legacy_impl and has_target_needing_toolchain:
        fatal_error(
            "--legacy-impl is incompatible with building packages needing "
            "a toolchain (%s)" % ", ".join(targets_needing_toolchain))


def default_stdlib_deployment_targets(args):
    """
    Return targets for the Swift stdlib, based on the build machine.
    If the build machine is not one of the recognized ones, return None.
    """

    host_target = StdlibDeploymentTarget.host_target()
    if host_target is None:
        return None

    # OS X build machines configure all Darwin platforms by default.
    # Put iOS native targets last so that we test them last
    # (it takes a long time).
    if host_target == StdlibDeploymentTarget.OSX.x86_64 or \
            host_target == StdlibDeploymentTarget.OSX.arm64:
        targets = [host_target]
        if args.build_ios and args.build_ios_simulator:
            targets.extend(StdlibDeploymentTarget.iOSSimulator.targets)
        if args.build_ios and args.build_ios_device:
            targets.extend(StdlibDeploymentTarget.iOS.targets)
        if args.build_tvos and args.build_tvos_simulator:
            targets.extend(StdlibDeploymentTarget.AppleTVSimulator.targets)
        if args.build_tvos and args.build_tvos_device:
            targets.extend(StdlibDeploymentTarget.AppleTV.targets)
        if args.build_watchos and args.build_watchos_simulator:
            targets.extend(StdlibDeploymentTarget
                           .AppleWatchSimulator.targets)
        if args.build_watchos and args.build_watchos_device:
            targets.extend(StdlibDeploymentTarget.AppleWatch.targets)
        return targets
    else:
        # All other machines only configure their host stdlib by default.
        return [host_target]


def apply_default_arguments(toolchain, args):
    # Infer if ninja is required
    ninja_required = (
        args.build_foundation or
        args.build_indexstoredb or
        args.build_sourcekitlsp or
        args.cmake_generator == 'Ninja'
    )
    if ninja_required and toolchain.ninja is None:
        args.build_ninja = True

    # Set the default stdlib-deployment-targets, if none were provided.
    if args.stdlib_deployment_targets is None:
        stdlib_targets = default_stdlib_deployment_targets(args)
        args.stdlib_deployment_targets = [
            target.name for target in stdlib_targets]

    # SwiftPM and XCTest have a dependency on Foundation.
    # On OS X, Foundation is built automatically using xcodebuild.
    # On Linux, we must ensure that it is built manually.
    if ((args.build_swiftpm or args.build_xctest) and
            platform.system() != "Darwin"):
        args.build_foundation = True

    # Foundation has a dependency on libdispatch.
    # On OS X, libdispatch is provided by the OS.
    # On Linux, we must ensure that it is built manually.
    if (args.build_foundation and
            platform.system() != "Darwin"):
        args.build_libdispatch = True

    if args.build_subdir is None:
        args.build_subdir = \
            workspace.compute_build_subdir(args)

    if args.relocate_xdg_cache_home_under_build_subdir:
        cache_under_build_subdir = os.path.join(
            SWIFT_BUILD_ROOT, args.build_subdir, '.cache')
        if args.verbose_build:
            print("Relocating XDG_CACHE_HOME to {}".format(cache_under_build_subdir))
        workspace.relocate_xdg_cache_home_under(cache_under_build_subdir)

    if args.install_destdir is None:
        args.install_destdir = os.path.join(
            SWIFT_BUILD_ROOT, args.build_subdir,
            '{}-{}'.format('toolchain', args.host_target))

    # Add optional stdlib-deployment-targets
    if args.android:
        if args.android_arch == "armv7":
            args.stdlib_deployment_targets.append(
                StdlibDeploymentTarget.Android.armv7.name)
        elif args.android_arch == "aarch64":
            args.stdlib_deployment_targets.append(
                StdlibDeploymentTarget.Android.aarch64.name)
        elif args.android_arch == "x86_64":
            args.stdlib_deployment_targets.append(
                StdlibDeploymentTarget.Android.x86_64.name)

    # Infer platform flags from manually-specified configure targets.
    # This doesn't apply to Darwin platforms, as they are
    # already configured. No building without the platform flag, though.

    android_tgts = [tgt for tgt in args.stdlib_deployment_targets
                    if StdlibDeploymentTarget.Android.contains(tgt)]
    if not args.android and len(android_tgts) > 0:
        # If building natively on an Android host, avoid the NDK
        # cross-compilation configuration.
        if not StdlibDeploymentTarget.Android.contains(StdlibDeploymentTarget
                                                       .host_target().name):
            args.android = True
        args.build_android = False

    # Include the Darwin supported architectures in the CMake options.
    if args.swift_darwin_supported_archs:
        args.extra_cmake_options.append(
            '-DSWIFT_DARWIN_SUPPORTED_ARCHS:STRING={}'.format(
                args.swift_darwin_supported_archs))

        # Remove unsupported Darwin archs from the standard library
        # deployment targets.
        supported_archs = args.swift_darwin_supported_archs.split(';')
        targets = StdlibDeploymentTarget.get_targets_by_name(
            args.stdlib_deployment_targets)

        args.stdlib_deployment_targets = [
            target.name
            for target in targets
            if (target.platform.is_darwin and
                target.arch in supported_archs)
        ]

    # Filter out any macOS stdlib deployment targets that are not supported
    # by the macOS SDK.
    targets = StdlibDeploymentTarget.get_targets_by_name(
        args.stdlib_deployment_targets)
    args.stdlib_deployment_targets = [
        target.name
        for target in targets
        if (not target.platform.is_darwin or
            target.platform.sdk_supports_architecture(
                target.arch, args.darwin_xcrun_toolchain))
    ]

    # Include the Darwin module-only architectures in the CMake options.
    if args.swift_darwin_module_archs:
        args.extra_cmake_options.append(
            '-DSWIFT_DARWIN_MODULE_ARCHS:STRING={}'.format(
                args.swift_darwin_module_archs))

# -----------------------------------------------------------------------------
# Main (preset)


def parse_preset_args():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="""Builds Swift using a preset.""")
    parser.add_argument(
        "-n", "--dry-run",
        help="print the commands that would be executed, but do not execute "
             "them",
        action="store_true",
        default=False)
    parser.add_argument(
        "--preset-file",
        help="load presets from the specified file",
        metavar="PATH",
        action="append",
        dest="preset_file_names",
        default=[])
    parser.add_argument(
        "--preset",
        help="use the specified option preset",
        metavar="NAME")
    parser.add_argument(
        "--show-presets",
        help="list all presets and exit",
        action=argparse.actions.StoreTrueAction,
        nargs=argparse.Nargs.OPTIONAL)
    parser.add_argument(
        "--distcc",
        help="use distcc",
        action=argparse.actions.StoreTrueAction,
        nargs=argparse.Nargs.OPTIONAL,
        default=os.environ.get('USE_DISTCC') == '1')
    parser.add_argument(
        "--sccache",
        help="use sccache",
        action=argparse.actions.StoreTrueAction,
        nargs=argparse.Nargs.OPTIONAL,
        default=os.environ.get('SWIFT_USE_SCCACHE') == '1')
    parser.add_argument(
        "--cmake-c-launcher",
        help="the absolute path to set CMAKE_C_COMPILER_LAUNCHER",
        metavar="PATH")
    parser.add_argument(
        "--cmake-cxx-launcher",
        help="the absolute path to set CMAKE_CXX_COMPILER_LAUNCHER",
        metavar="PATH")
    parser.add_argument(
        "-j", "--jobs",
        help="the number of parallel build jobs to use",
        type=int,
        dest="build_jobs")
    parser.add_argument(
        "preset_substitutions_raw",
        help="'name=value' pairs that are substituted in the preset",
        nargs="*",
        metavar="SUBSTITUTION")
    parser.add_argument(
        "--expand-invocation", "--expand-build-script-invocation",
        help="Print the expanded build-script invocation generated "
             "by the preset, but do not run the preset",
        action=argparse.actions.StoreTrueAction,
        nargs=argparse.Nargs.OPTIONAL)
    parser.add_argument(
        "--swiftsyntax-install-prefix",
        help="specify the directory to where SwiftSyntax should be installed")
    parser.add_argument(
        "--build-dir",
        help="specify the directory where build artifact should be stored")
    parser.add_argument(
        "--dump-config",
        help="instead of building, write JSON to stdout containing "
             "various values used to build in this configuration",
        action="store_true",
        default=False)
    parser.add_argument(
        "--reconfigure",
        help="Reconfigure all projects as we build",
        action="store_true",
        default=False)

    return parser.parse_args()


def main_preset():
    args = parse_preset_args()

    if len(args.preset_file_names) == 0:
        args.preset_file_names = [
            os.path.join(
                SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, "utils",
                "build-presets.ini")
        ]

        user_presets_file = os.path.join(os.path.expanduser("~"),
                                         '.swift-build-presets')
        if os.path.isfile(user_presets_file):
            args.preset_file_names.append(user_presets_file)

    preset_parser = presets.PresetParser()

    try:
        preset_parser.read_files(args.preset_file_names)
    except presets.PresetError as e:
        fatal_error(six.text_type(e))

    if args.show_presets:
        for name in sorted(preset_parser.preset_names,
                           key=lambda name: name.lower()):
            print(name)
        return 0

    if not args.preset:
        fatal_error("missing --preset option")

    args.preset_substitutions = {}
    for arg in args.preset_substitutions_raw:
        name, value = arg.split("=", 1)
        args.preset_substitutions[name] = value

    try:
        preset = preset_parser.get_preset(
            args.preset,
            vars=args.preset_substitutions)
    except presets.PresetError as e:
        fatal_error(six.text_type(e))

    preset_args = migration.migrate_swift_sdks(preset.args)

    if args.distcc and (args.cmake_c_launcher or args.cmake_cxx_launcher):
        fatal_error(
            '--distcc can not be used with' +
            ' --cmake-c-launcher or --cmake-cxx-launcher')
    if args.sccache and (args.cmake_c_launcher or args.cmake_cxx_launcher):
        fatal_error(
            '--sccache can not be used with' +
            ' --cmake-c-launcher or --cmake-cxx-launcher')

    build_script_args = [sys.argv[0]]
    if args.dry_run:
        build_script_args += ["--dry-run"]
    if args.dump_config:
        build_script_args += ["--dump-config"]

    build_script_args += preset_args
    if args.distcc:
        build_script_args += ["--distcc"]
    if args.sccache:
        build_script_args += ["--sccache"]
    if args.build_jobs:
        build_script_args += ["--jobs", str(args.build_jobs)]
    if args.swiftsyntax_install_prefix:
        build_script_args += ["--install-swiftsyntax",
                              "--install-destdir",
                              args.swiftsyntax_install_prefix]
    if args.build_dir:
        build_script_args += ["--build-dir", args.build_dir]
    if args.cmake_c_launcher:
        build_script_args += ["--cmake-c-launcher", args.cmake_c_launcher]
    if args.cmake_cxx_launcher:
        build_script_args += ["--cmake-cxx-launcher", args.cmake_cxx_launcher]
    printable_command = shell.quote_command(build_script_args)
    if args.expand_invocation:
        print(printable_command)
        return 0

    if args.reconfigure:
        build_script_args += ["--reconfigure"]

    print_note('using preset "{}", which expands to \n\n{}\n'.format(
        args.preset, printable_command))
    shell.call_without_sleeping(build_script_args)
    return 0


# -----------------------------------------------------------------------------
# Main (normal)

def main_normal():
    parser = driver_arguments.create_argument_parser()

    args = migration.parse_args(parser, sys.argv[1:])

    if args.build_script_impl_args:
        # If we received any impl args, check if `build-script-impl` would
        # accept them or not before any further processing.
        try:
            migration.check_impl_args(BUILD_SCRIPT_IMPL_PATH,
                                      args.build_script_impl_args)
        except ValueError as e:
            exit_rejecting_arguments(e, parser)

        if '--check-args-only' in args.build_script_impl_args:
            return 0

    shell.dry_run = args.dry_run

    # Prepare and validate toolchain
    if args.darwin_xcrun_toolchain is None:
        xcrun_toolchain = os.environ.get('TOOLCHAINS',
                                         defaults.DARWIN_XCRUN_TOOLCHAIN)

        print_note('Using toolchain {}'.format(xcrun_toolchain))
        args.darwin_xcrun_toolchain = xcrun_toolchain

    toolchain = host_toolchain(xcrun_toolchain=args.darwin_xcrun_toolchain)
    os.environ['TOOLCHAINS'] = args.darwin_xcrun_toolchain

    if args.host_cc is not None:
        toolchain.cc = args.host_cc
    if args.host_cxx is not None:
        toolchain.cxx = args.host_cxx
    if args.host_lipo is not None:
        toolchain.lipo = args.host_lipo
    if args.host_libtool is not None:
        toolchain.libtool = args.host_libtool
    if args.cmake is not None:
        toolchain.cmake = args.cmake
    if args.sccache:
        print("Ensuring the sccache server is running...")
        # Use --show-stats to ensure the server is started, because using
        # --start-server will fail if the server is already running.
        # Capture the output because we don't want to see the stats.
        shell.capture([toolchain.sccache, "--show-stats"])

    cmake = CMake(args=args, toolchain=toolchain)
    # Check the CMake version is sufficient on Linux and build from source
    # if not.
    cmake_path = cmake.check_cmake_version(SWIFT_SOURCE_ROOT, SWIFT_BUILD_ROOT)
    if cmake_path is not None:
        toolchain.cmake = cmake_path
        args.cmake = cmake_path

    # Preprocess the arguments to apply defaults.
    apply_default_arguments(toolchain, args)

    # Validate the arguments.
    validate_arguments(toolchain, args)

    # Create the build script invocation.
    invocation = build_script_invocation.BuildScriptInvocation(toolchain, args)

    # Sanitize the runtime environment.
    initialize_runtime_environment()

    # Show SDKs, if requested.
    if args.show_sdks:
        print_xcodebuild_versions()

    if args.dump_config:
        print(JSONDumper().encode(invocation))
        return 0

    # Clean build directory if requested.
    if args.clean:
        clean_delay()
        shell.rmtree(invocation.workspace.build_root)

    # Create build directory.
    shell.makedirs(invocation.workspace.build_root)

    # Create .build_script_log
    if not args.dry_run:
        build_script_log = os.path.join(invocation.workspace.build_root,
                                        ".build_script_log")
        open(build_script_log, 'w').close()

    if args.clean_install_destdir:
        sys.stdout.write("Cleaning install destdir!\n")
        shell.rmtree(args.install_destdir)

    # Build ninja if required, which will update the toolchain.
    if args.build_ninja:
        invocation.build_ninja()

    # Execute the underlying build script implementation.
    invocation.execute()

    if args.symbols_package:
        print('--- Creating symbols package ---')
        print('-- Package file: {} --'.format(args.symbols_package))

        if platform.system() == 'Darwin':
            prefix = targets.darwin_toolchain_prefix(args.install_prefix)
            prefix = os.path.join(args.host_target, prefix.lstrip('/'))
        else:
            prefix = args.install_prefix

        # As a security measure, `tar` normally strips leading '/' from paths
        # it is archiving. To stay safe, we change working directories, then
        # run `tar` without the leading '/' (we remove it ourselves to keep
        # `tar` from emitting a warning).
        with shell.pushd(args.install_symroot):
            tar(source=prefix.lstrip('/'),
                destination=args.symbols_package)

    return 0


# -----------------------------------------------------------------------------

def main():
    if not SWIFT_SOURCE_ROOT:
        fatal_error(
            "could not infer source root directory " +
            "(forgot to set $SWIFT_SOURCE_ROOT environment variable?)")

    if not os.path.isdir(SWIFT_SOURCE_ROOT):
        fatal_error(
            "source root directory \'" + SWIFT_SOURCE_ROOT +
            "\' does not exist " +
            "(forgot to set $SWIFT_SOURCE_ROOT environment variable?)")

    validate_xcode_compatibility()

    # Determine if we are invoked in the preset mode and dispatch accordingly.
    if any([(opt.startswith("--preset") or opt == "--show-presets")
            for opt in sys.argv[1:]]):
        return main_preset()
    else:
        return main_normal()


if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        sys.exit(1)