#!/usr/bin/env python3 # This source file is part of the Swift.org open source project # # Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors # Licensed under Apache License v2.0 with Runtime Library Exception # # See https://swift.org/LICENSE.txt for license information # See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors """ The ultimate tool for building Swift. """ import json import os import platform import re import sys import time from build_swift.build_swift import argparse from build_swift.build_swift import defaults from build_swift.build_swift import driver_arguments from build_swift.build_swift import migration from build_swift.build_swift import presets from build_swift.build_swift.constants import BUILD_SCRIPT_IMPL_PATH from build_swift.build_swift.constants import SWIFT_BUILD_ROOT from build_swift.build_swift.constants import SWIFT_REPO_NAME from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT from swift_build_support.swift_build_support import build_script_invocation from swift_build_support.swift_build_support import shell from swift_build_support.swift_build_support import targets from swift_build_support.swift_build_support import workspace from swift_build_support.swift_build_support.cmake import CMake from swift_build_support.swift_build_support.targets import StdlibDeploymentTarget from swift_build_support.swift_build_support.toolchain import host_toolchain from swift_build_support.swift_build_support.utils \ import exit_rejecting_arguments from swift_build_support.swift_build_support.utils import fatal_error from swift_build_support.swift_build_support.utils import log_analyzer # ----------------------------------------------------------------------------- # Helpers def print_note(message, stream=sys.stdout): """Writes a diagnostic message to the given stream. By default this function outputs to stdout. """ stream.write('[{}] NOTE: {}\n'.format(sys.argv[0], message)) stream.flush() def print_warning(message, stream=sys.stdout): """Writes a warning to the given stream. By default this function outputs to stdout. """ warning = "[{}] WARNING: {}\n".format(sys.argv[0], message) magenta_warning = "\033[35m" + warning + "\033[0m" stream.write(magenta_warning) stream.flush() def clean_delay(): """Provide a short delay so accidentally invoked clean builds can be canceled. """ sys.stdout.write('Starting clean build in ') for i in range(3, 0, -1): sys.stdout.write('\b%d' % i) sys.stdout.flush() time.sleep(1) print('\b\b\b\bnow.') def initialize_runtime_environment(): """Change the program environment for building. """ # Set an appropriate default umask. os.umask(0o022) # Unset environment variables that might affect how tools behave. for v in [ 'MAKEFLAGS', 'SDKROOT', 'MACOSX_DEPLOYMENT_TARGET', 'IPHONEOS_DEPLOYMENT_TARGET', 'TVOS_DEPLOYMENT_TARGET', 'WATCHOS_DEPLOYMENT_TARGET', 'XROS_DEPLOYMENT_TARGET']: os.environ.pop(v, None) # Provide a default NINJA_STATUS to format ninja output. if 'NINJA_STATUS' not in os.environ: os.environ['NINJA_STATUS'] = '[%f/%t][%p][%es] ' class JSONDumper(json.JSONEncoder): def __init__(self, *args, **kwargs): json.JSONEncoder.__init__( self, indent=2, separators=(',', ': '), sort_keys=True, *args, **kwargs) def default(self, o): if hasattr(o, '__dict__'): return vars(o) return str(o) def print_xcodebuild_versions(file=sys.stdout): """ Print the host machine's `xcodebuild` version, as well as version information for all available SDKs. """ version = shell.capture( ['xcodebuild', '-version'], dry_run=False, echo=False).rstrip() # Allow non-zero exit codes. Under certain obscure circumstances # xcodebuild can exit with a non-zero exit code even when the SDK is # usable. sdks = shell.capture( ['xcodebuild', '-version', '-sdk'], dry_run=False, echo=False, allow_non_zero_exit=True).rstrip() fmt = '{version}\n\n--- SDK versions ---\n{sdks}\n' print(fmt.format(version=version, sdks=sdks), file=file) file.flush() def validate_xcode_compatibility(): if sys.platform != 'darwin': return if os.getenv("SKIP_XCODE_VERSION_CHECK"): print("note: skipping Xcode version check") return output = shell.capture( ['xcodebuild', '-version'], dry_run=False, echo=False).strip() # Capture only version ex: '14.x' from full output match = re.match(r"Xcode (\d+\.\d+(?:\.\d)?)", output) if not match: print(f"warning: unexpected xcodebuild output format: '{output}', " "skipping Xcode compatibility check, please report this issue", file=sys.stderr) return version_match = match.group(1) current_version = tuple(int(v) for v in version_match.replace('.', ' ').split()) minimum_version = (13, 0) if current_version < minimum_version: raise SystemExit( f"error: using unsupported Xcode version '{version_match}'. " f"Install Xcode {'.'.join(str(x) for x in minimum_version)} or newer, " "or set 'SKIP_XCODE_VERSION_CHECK=1' in the environment" ) def tar(source, destination): """ Create a gzip archive of the file at 'source' at the given 'destination' path. """ # We do not use `tarfile` here because: # - We wish to support LZMA2 compression while also supporting Python 2.7. # - We wish to explicitly set the owner and group of the archive. args = ['tar', '-c', '-z', '-f', destination] if platform.system() != 'Darwin' and platform.system() != 'Windows': args += ['--owner=0', '--group=0'] # Discard stderr output such as 'tar: Failed to open ...'. We'll detect # these cases using the exit code, which should cause 'shell.call' to # raise. shell.call(args + [source], stderr=shell.DEVNULL) # ----------------------------------------------------------------------------- # Argument Validation def validate_arguments(toolchain, args): if toolchain.cc is None or toolchain.cxx is None: fatal_error( "can't find clang (please install clang-3.5 or a " "later version)") if toolchain.cmake is None: fatal_error("can't find CMake (please install CMake)") if args.distcc: if toolchain.distcc is None: fatal_error( "can't find distcc (please install distcc)") if toolchain.distcc_pump is None: fatal_error( "can't find distcc-pump (please install distcc-pump)") if args.sccache: if toolchain.sccache is None: fatal_error( "can't find sccache (please install sccache)") # Xcode project generation is no longer supported. if args.cmake_generator == "Xcode": fatal_error( "build-script no longer supports Xcode project generation, you " "should use utils/generate-xcode instead, see " "https://github.com/swiftlang/swift/blob/main/docs/HowToGuides/" "GettingStarted.md#using-ninja-with-xcode for more info" ) if args.host_target is None or args.stdlib_deployment_targets is None: fatal_error("unknown operating system") if args.symbols_package: if not os.path.isabs(args.symbols_package): print( '--symbols-package must be an absolute path ' '(was \'{}\')'.format(args.symbols_package)) return 1 if not args.install_symroot: fatal_error( "--install-symroot is required when specifying " "--symbols-package.") if args.android: if args.android_ndk is None or \ args.android_api_level is None: fatal_error( "when building for Android, --android-ndk " "and --android-api-level must be specified") targets_needing_toolchain = [ 'build_indexstoredb', 'build_playgroundsupport', 'build_sourcekitlsp', 'build_toolchainbenchmarks', 'build_swift_inspect', 'tsan_libdispatch_test', ] has_target_needing_toolchain = \ bool(sum(getattr(args, x) for x in targets_needing_toolchain)) if args.legacy_impl and has_target_needing_toolchain: fatal_error( "--legacy-impl is incompatible with building packages needing " "a toolchain (%s)" % ", ".join(targets_needing_toolchain)) def default_stdlib_deployment_targets(args): """ Return targets for the Swift stdlib, based on the build machine. If the build machine is not one of the recognized ones, return None. """ host_target = StdlibDeploymentTarget.host_target() if host_target is None: return None targets = [host_target] # OS X build machines configure all Darwin platforms by default. # Put iOS native targets last so that we test them last # (it takes a long time). if host_target == StdlibDeploymentTarget.OSX.x86_64 or \ host_target == StdlibDeploymentTarget.OSX.arm64: if args.build_ios and args.build_ios_simulator: targets.extend(StdlibDeploymentTarget.iOSSimulator.targets) if args.build_ios and args.build_ios_device: targets.extend(StdlibDeploymentTarget.iOS.targets) if args.build_tvos and args.build_tvos_simulator: targets.extend(StdlibDeploymentTarget.AppleTVSimulator.targets) if args.build_tvos and args.build_tvos_device: targets.extend(StdlibDeploymentTarget.AppleTV.targets) if args.build_watchos and args.build_watchos_simulator: targets.extend(StdlibDeploymentTarget .AppleWatchSimulator.targets) if args.build_watchos and args.build_watchos_device: targets.extend(StdlibDeploymentTarget.AppleWatch.targets) if StdlibDeploymentTarget.XROSSimulator and \ args.build_xros and args.build_xros_simulator: targets.extend(StdlibDeploymentTarget.XROSSimulator.targets) if StdlibDeploymentTarget.XROS and \ args.build_xros and args.build_xros_device: targets.extend(StdlibDeploymentTarget.XROS.targets) return targets elif host_target.platform.name == 'linux': if args.build_linux_static: targets.append(getattr(StdlibDeploymentTarget.Musl, host_target.arch)) return targets def apply_default_arguments(toolchain, args): # Infer if ninja is required ninja_required = ( args.build_foundation or args.build_indexstoredb or args.build_sourcekitlsp or args.cmake_generator == 'Ninja' ) if ninja_required and toolchain.ninja is None: args.build_ninja = True # Enable test colors if we're on a tty. if args.color_in_tests and sys.stdout.isatty(): args.lit_args += " --param color_output" # Set the default stdlib-deployment-targets, if none were provided. if args.stdlib_deployment_targets is None: stdlib_targets = default_stdlib_deployment_targets(args) args.stdlib_deployment_targets = [ target.name for target in stdlib_targets] # SwiftPM and XCTest have a dependency on Foundation. # On OS X, Foundation is built automatically using xcodebuild. # On Linux, we must ensure that it is built manually. if ((args.build_swiftpm or args.build_xctest) and platform.system() != "Darwin"): args.build_foundation = True # Foundation has a dependency on libdispatch. # On OS X, libdispatch is provided by the OS. # On Linux, we must ensure that it is built manually. if (args.build_foundation and platform.system() != "Darwin"): args.build_libdispatch = True if args.build_subdir is None: args.build_subdir = \ workspace.compute_build_subdir(args) if args.relocate_xdg_cache_home_under_build_subdir: cache_under_build_subdir = os.path.join( SWIFT_BUILD_ROOT, args.build_subdir, '.cache') if args.verbose_build: print("Relocating XDG_CACHE_HOME to {}".format(cache_under_build_subdir)) workspace.relocate_xdg_cache_home_under(cache_under_build_subdir) if args.install_destdir is None: args.install_destdir = os.path.join( SWIFT_BUILD_ROOT, args.build_subdir, '{}-{}'.format('toolchain', args.host_target)) # Add optional stdlib-deployment-targets if args.android: if args.android_arch == "armv7": args.stdlib_deployment_targets.append( StdlibDeploymentTarget.Android.armv7.name) elif args.android_arch == "aarch64": args.stdlib_deployment_targets.append( StdlibDeploymentTarget.Android.aarch64.name) elif args.android_arch == "x86_64": args.stdlib_deployment_targets.append( StdlibDeploymentTarget.Android.x86_64.name) # Infer platform flags from manually-specified configure targets. # This doesn't apply to Darwin platforms, as they are # already configured. No building without the platform flag, though. android_tgts = [tgt for tgt in args.stdlib_deployment_targets if StdlibDeploymentTarget.Android.contains(tgt)] if not args.android and len(android_tgts) > 0: # If building natively on an Android host, avoid the NDK # cross-compilation configuration. if not StdlibDeploymentTarget.Android.contains(StdlibDeploymentTarget .host_target().name): args.android = True args.build_android = False # Include the Darwin supported architectures in the CMake options. if args.swift_darwin_supported_archs: args.extra_cmake_options.append( '-DSWIFT_DARWIN_SUPPORTED_ARCHS:STRING={}'.format( args.swift_darwin_supported_archs)) # Remove unsupported Darwin archs from the standard library # deployment targets. supported_archs = args.swift_darwin_supported_archs.split(';') targets = StdlibDeploymentTarget.get_targets_by_name( args.stdlib_deployment_targets) args.stdlib_deployment_targets = [ target.name for target in targets if (target.platform.is_darwin and target.arch in supported_archs) ] else: # Clear the value explicitly from CMakeCache.txt # to avoid picking up a stale value that can cause # build failures args.extra_cmake_options.append( '-USWIFT_DARWIN_SUPPORTED_ARCHS') # Filter out any macOS stdlib deployment targets that are not supported # by the macOS SDK. targets = StdlibDeploymentTarget.get_targets_by_name( args.stdlib_deployment_targets) args.stdlib_deployment_targets = [ target.name for target in targets if (not target.platform.is_darwin or target.platform.sdk_supports_architecture( target.arch, args.darwin_xcrun_toolchain, args)) ] # Include the Darwin module-only architectures in the CMake options. if args.swift_darwin_module_archs: args.extra_cmake_options.append( '-DSWIFT_DARWIN_MODULE_ARCHS:STRING={}'.format( args.swift_darwin_module_archs)) if (args.infer_cross_compile_hosts_on_darwin and platform.system() == "Darwin"): args.cross_compile_hosts = _infer_cross_compile_hosts_on_darwin( get_running_arch(args.dry_run)) print("Inferred the following hosts for cross compilations: " f"{args.cross_compile_hosts}") sys.stdout.flush() def _infer_cross_compile_hosts_on_darwin(arch_we_are_running_on): if arch_we_are_running_on == "x86_64": return ["macosx-arm64"] else: return ["macosx-x86_64"] def get_running_arch(dry_run): # We can override the detected arch to support # BuildSystem/infer-cross-compile-hosts-on-darwin-macosx.test # Given the narrow scenario, we preferred using # an environment variable instead of a build-script # argument to make this less discoverable on purpose if dry_run: injected_arch = os.environ.get( 'ARCH_FOR_BUILDSYSTEM_TESTS', platform.machine()) print("DRY RUN: assume build-script is being run on a " f"{injected_arch} machine") return injected_arch else: return platform.machine() # ----------------------------------------------------------------------------- # Main (preset) def parse_preset_args(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description="""Builds Swift using a preset.""") parser.add_argument( "-n", "--dry-run", help="print the commands that would be executed, but do not execute " "them", action="store_true", default=False) parser.add_argument( "--preset-file", help="load presets from the specified file", metavar="PATH", action="append", dest="preset_file_names", default=[]) parser.add_argument( "--preset", help="use the specified option preset", metavar="NAME") parser.add_argument( "--show-presets", help="list all presets and exit", action=argparse.actions.StoreTrueAction, nargs=argparse.Nargs.OPTIONAL) parser.add_argument( "--preset-vars-file", help="load preset vars from the specified file", metavar="PATH", action="append", dest="preset_vars_file_names", default=[]) parser.add_argument( "--distcc", help="use distcc", action=argparse.actions.StoreTrueAction, nargs=argparse.Nargs.OPTIONAL, default=os.environ.get('USE_DISTCC') == '1') parser.add_argument( "--sccache", help="use sccache", action=argparse.actions.StoreTrueAction, nargs=argparse.Nargs.OPTIONAL, default=os.environ.get('SWIFT_USE_SCCACHE') == '1') parser.add_argument( "--cmake-c-launcher", help="the absolute path to set CMAKE_C_COMPILER_LAUNCHER", metavar="PATH") parser.add_argument( "--cmake-cxx-launcher", help="the absolute path to set CMAKE_CXX_COMPILER_LAUNCHER", metavar="PATH") parser.add_argument( "-j", "--jobs", help="the number of parallel build jobs to use", type=int, dest="build_jobs") parser.add_argument( "--lit-jobs", help="the number of workers to use when testing with lit", type=int, dest="lit_jobs") parser.add_argument( "preset_substitutions_raw", help="'name=value' pairs that are substituted in the preset", nargs="*", metavar="SUBSTITUTION") parser.add_argument( "--expand-invocation", "--expand-build-script-invocation", help="Print the expanded build-script invocation generated " "by the preset, but do not run the preset", action=argparse.actions.StoreTrueAction, nargs=argparse.Nargs.OPTIONAL) parser.add_argument( "--swiftsyntax-install-prefix", help="specify the directory to where SwiftSyntax should be installed") parser.add_argument( "--build-dir", help="specify the directory where build artifact should be stored") parser.add_argument( "--dump-config", help="instead of building, write JSON to stdout containing " "various values used to build in this configuration", action="store_true", default=False) parser.add_argument( "--reconfigure", help="Reconfigure all projects as we build", action="store_true", default=False) return parser.parse_args() def main_preset(): args = parse_preset_args() if len(args.preset_file_names) == 0: args.preset_file_names = [ os.path.join( SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, "utils", "build-presets.ini") ] user_presets_file = os.path.join(os.path.expanduser("~"), '.swift-build-presets') if os.path.isfile(user_presets_file): args.preset_file_names.append(user_presets_file) preset_parser = presets.PresetParser() try: preset_parser.read_files(args.preset_file_names) except presets.PresetError as e: fatal_error(str(e)) if args.show_presets: for name in sorted(preset_parser.preset_names, key=lambda name: name.lower()): print(name) return 0 if not args.preset: fatal_error("missing --preset option") args.preset_substitutions = {} for file in args.preset_vars_file_names: if os.path.isfile(file): with open(file, 'r') as f: for _, line in enumerate(f): name, value = line.split("=", 1) args.preset_substitutions[name] = value.strip() for arg in args.preset_substitutions_raw: name, value = arg.split("=", 1) args.preset_substitutions[name] = value try: preset = preset_parser.get_preset( args.preset, vars=args.preset_substitutions) except presets.PresetError as e: fatal_error(str(e)) preset_args = migration.migrate_swift_sdks(preset.args) if args.distcc and (args.cmake_c_launcher or args.cmake_cxx_launcher): fatal_error( '--distcc can not be used with' + ' --cmake-c-launcher or --cmake-cxx-launcher') if args.sccache and (args.cmake_c_launcher or args.cmake_cxx_launcher): fatal_error( '--sccache can not be used with' + ' --cmake-c-launcher or --cmake-cxx-launcher') build_script_args = [sys.argv[0]] if args.dry_run: build_script_args += ["--dry-run"] if args.dump_config: build_script_args += ["--dump-config"] build_script_args += preset_args if args.distcc: build_script_args += ["--distcc"] if args.sccache: build_script_args += ["--sccache"] if args.build_jobs: build_script_args += ["--jobs", str(args.build_jobs)] if args.lit_jobs: build_script_args += ["--lit-jobs", str(args.lit_jobs)] if args.swiftsyntax_install_prefix: build_script_args += ["--install-swiftsyntax", "--install-destdir", args.swiftsyntax_install_prefix] if args.build_dir: build_script_args += ["--build-dir", args.build_dir] if args.cmake_c_launcher: build_script_args += ["--cmake-c-launcher", args.cmake_c_launcher] if args.cmake_cxx_launcher: build_script_args += ["--cmake-cxx-launcher", args.cmake_cxx_launcher] printable_command = shell.quote_command(build_script_args) if args.expand_invocation: print(printable_command) return 0 if args.reconfigure: build_script_args += ["--reconfigure"] print_note('using preset "{}", which expands to \n\n{}\n'.format( args.preset, printable_command)) shell.call_without_sleeping(build_script_args) return 0 # ----------------------------------------------------------------------------- # Main (normal) def main_normal(): parser = driver_arguments.create_argument_parser() args = migration.parse_args(parser, sys.argv[1:]) if args.build_script_impl_args: # If we received any impl args, check if `build-script-impl` would # accept them or not before any further processing. try: migration.check_impl_args(BUILD_SCRIPT_IMPL_PATH, args.build_script_impl_args) except ValueError as e: exit_rejecting_arguments(e, parser) if '--check-args-only' in args.build_script_impl_args: return 0 shell.dry_run = args.dry_run # Prepare and validate toolchain if args.darwin_xcrun_toolchain is None: xcrun_toolchain = os.environ.get('TOOLCHAINS', defaults.DARWIN_XCRUN_TOOLCHAIN) print_note('Using toolchain {}'.format(xcrun_toolchain)) args.darwin_xcrun_toolchain = xcrun_toolchain toolchain = host_toolchain(xcrun_toolchain=args.darwin_xcrun_toolchain) os.environ['TOOLCHAINS'] = args.darwin_xcrun_toolchain if args.host_cc is not None: toolchain.cc = args.host_cc if args.host_cxx is not None: toolchain.cxx = args.host_cxx if args.host_lipo is not None: toolchain.lipo = args.host_lipo if args.host_libtool is not None: toolchain.libtool = args.host_libtool if args.cmake is not None: toolchain.cmake = args.cmake # Preprocess the arguments to apply defaults. apply_default_arguments(toolchain, args) cmake = CMake(args=args, toolchain=toolchain) toolchain.cmake = cmake.get_cmake_path(source_root=SWIFT_SOURCE_ROOT, build_root=SWIFT_BUILD_ROOT) args.cmake = toolchain.cmake # Create the build script invocation. invocation = build_script_invocation.BuildScriptInvocation(toolchain, args) # Validate the arguments. validate_arguments(toolchain, args) if args.sccache: print("Ensuring the sccache server is running...") # Use --show-stats to ensure the server is started, because using # --start-server will fail if the server is already running. # Capture the output because we don't want to see the stats. shell.capture([toolchain.sccache, "--show-stats"]) # Sanitize the runtime environment. initialize_runtime_environment() # Show SDKs, if requested. if args.show_sdks: print_xcodebuild_versions() if args.dump_config: print(JSONDumper().encode(invocation)) return 0 # Clean build directory if requested. if args.clean: clean_delay() shell.rmtree(invocation.workspace.build_root) # Create build directory. shell.makedirs(invocation.workspace.build_root) # Create .build_script_log if not args.dry_run: build_script_log = os.path.join(invocation.workspace.build_root, ".build_script_log") open(build_script_log, 'w').close() if args.clean_install_destdir: sys.stdout.write("Cleaning install destdir!\n") shell.rmtree(args.install_destdir) # Execute the underlying build script implementation. invocation.execute() if args.symbols_package: print('--- Creating symbols package ---') print('-- Package file: {} --'.format(args.symbols_package)) if platform.system() == 'Darwin': prefix = targets.darwin_toolchain_prefix(args.install_prefix) prefix = os.path.join(args.host_target, prefix.lstrip('/')) else: prefix = args.install_prefix # As a security measure, `tar` normally strips leading '/' from paths # it is archiving. To stay safe, we change working directories, then # run `tar` without the leading '/' (we remove it ourselves to keep # `tar` from emitting a warning). with shell.pushd(args.install_symroot): tar(source=prefix.lstrip('/'), destination=args.symbols_package) return 0 # ----------------------------------------------------------------------------- def main(): if not SWIFT_SOURCE_ROOT: fatal_error( "could not infer source root directory " + "(forgot to set $SWIFT_SOURCE_ROOT environment variable?)") if not os.path.isdir(SWIFT_SOURCE_ROOT): fatal_error( "source root directory \'" + SWIFT_SOURCE_ROOT + "\' does not exist " + "(forgot to set $SWIFT_SOURCE_ROOT environment variable?)") validate_xcode_compatibility() # Determine if we are invoked in the preset mode and dispatch accordingly. if any([(opt.startswith("--preset") or opt == "--show-presets") for opt in sys.argv[1:]]): return main_preset() else: return main_normal() if __name__ == "__main__": try: exit_code = main() log_analyzer() except SystemExit: raise except KeyboardInterrupt: sys.exit(1) except Exception: log_analyzer() raise sys.exit(exit_code)