[build-script] Determine HOST_CC in build-script

https://bugs.swift.org/browse/SR-237 calls for `build-script` and
`build-script-impl` to be merged. This commit takes another step towards
that goal by moving the logic that finds the path to the `clang` and
`clang++` executables up into Python-land.

Rather than simply moving all of the logic into `utils/build-script`,
this commit moves relevant functions into a new Python module, named
`swift_build_support`. This has several benefits:

- The logic can be tested. Whereas `build-script-impl` needed to be run
  in order to verify its behavior, the logic extracted out of it into
  `swift_build_support` can be tested in isolation.
- The logic can be split up into several files without polluting the
  `utils` directory, which now contains many different files that are
  unrelated to `build-script`.

Author: modocache

utils/build-script

@@ -18,8 +18,11 @@ import shutil
 import sys
 import textwrap
 
+# FIXME: Instead of modifying the system path in order to enable imports from
+#        other directories, all Python modules related to the build script
+#        should be moved to the `swift_build_support` module.
+#        For additional information, see: https://bugs.swift.org/browse/SR-237.
 sys.path.append(os.path.dirname(__file__))
-
 from SwiftBuildSupport import (
     HOME,
     SWIFT_BUILD_ROOT,
@@ -31,6 +34,11 @@ from SwiftBuildSupport import (
     quote_shell_command,
 )
 
+sys.path.append(os.path.join(os.path.dirname(__file__), 'swift_build_support'))
+import swift_build_support.clang
+from swift_build_support.migration import migrate_impl_args
+
+
 # Main entry point for the preset mode.
 def main_preset():
     parser = argparse.ArgumentParser(
@@ -495,6 +503,10 @@ the number of parallel build jobs to use""",
         dest="build_jobs",
         default=multiprocessing.cpu_count())
 
+    parser.add_argument("--darwin-xcrun-toolchain",
+                        help="the name of the toolchain to use on Darwin",
+                        default="default")
+
     parser.add_argument("--extra-swift-args", help=textwrap.dedent("""
     Pass through extra flags to swift in the form of a cmake list 'module_regexp;flag'. Can
     be called multiple times to add multiple such module_regexp flag pairs. All semicolons
@@ -505,7 +517,8 @@ the number of parallel build jobs to use""",
         help="",
         nargs="*")
 
-    args = parser.parse_args()
+    args = parser.parse_args(migrate_impl_args(sys.argv[1:], [
+        '--darwin-xcrun-toolchain']))
 
     # Build cmark if any cmark-related options were specified.
     if (args.cmark_build_variant is not None):
@@ -709,9 +722,18 @@ the number of parallel build jobs to use""",
     if args.clean:
         shutil.rmtree(build_dir)
 
+    host_clang = swift_build_support.clang.host_clang(
+        xcrun_toolchain=args.darwin_xcrun_toolchain)
+    if not host_clang:
+        print_with_argv0(
+            "Can't find clang.  Please install clang-3.5 or a later version.")
+        return 1
     build_script_impl_args = [
         os.path.join(SWIFT_SOURCE_ROOT, "swift", "utils", "build-script-impl"),
         "--build-dir", build_dir,
+        "--host-cc", host_clang.cc,
+        "--host-cxx", host_clang.cxx,
+        "--darwin-xcrun-toolchain", args.darwin_xcrun_toolchain,
         "--cmark-build-type", args.cmark_build_variant,
         "--llvm-build-type", args.llvm_build_variant,
         "--swift-build-type", args.swift_build_variant,

utils/build-script-impl

@@ -57,6 +57,8 @@ KNOWN_SETTINGS=(
     # name                      default          description
     build-args                  ""               "arguments to the build tool; defaults to -j8 when CMake generator is \"Unix Makefiles\""
     build-dir                   ""               "out-of-tree build directory; default is in-tree. **This argument is required**"
+    host-cc                     ""               "the path to CC, the 'clang' compiler for the host platform. **This argument is requied**"
+    host-cxx                    ""               "the path to CXX, the 'clang++' compiler for the host platform. **This argument is requied**"
     darwin-xcrun-toolchain      "default"        "the name of the toolchain to use on Darwin"
     build-ninja                 ""               "build the Ninja tool"
     cmark-build-type            "Debug"          "the CMake build variant for CommonMark (Debug, RelWithDebInfo, Release, MinSizeRel).  Defaults to Debug."
@@ -1037,33 +1039,10 @@ if [[ "${EXPORT_COMPILE_COMMANDS}" ]] ; then
     )
 fi
 
-if [ -z "${HOST_CC}" ] ; then
-    if [ "$(uname -s)" == "Darwin" ] ; then
-        HOST_CC="$(xcrun_find_tool clang)"
-        HOST_CXX="$(xcrun_find_tool clang++)"
-    elif [ "$(uname -s)" == "FreeBSD" ]; then
-        if [ $(sysctl -n kern.osreldate) -ge 1100000 ]; then
-            HOST_CC="clang"
-            HOST_CXX="clang++"
-        else
-            for clang_candidate_suffix in "38" "37" "36" "35" ; do
-                if which "clang${clang_candidate_suffix}" > /dev/null ; then
-                    HOST_CC="clang${clang_candidate_suffix}"
-                    HOST_CXX="clang++${clang_candidate_suffix}"
-                    break
-                fi
-            done
-        fi
-    else
-        for clang_candidate_suffix in "" "-3.8" "-3.7" "-3.6" "-3.5" ; do
-            if which "clang${clang_candidate_suffix}" > /dev/null ; then
-                HOST_CC="clang${clang_candidate_suffix}"
-                HOST_CXX="clang++${clang_candidate_suffix}"
-                break
-            fi
-        done
-    fi
-fi
+# FIXME: HOST_CC is set and its presence is validated by utils/build-script.
+#        This check is redundant, but must remain until build-script-impl
+#        is merged completely with utils/build-script.
+#        For additional information, see: https://bugs.swift.org/browse/SR-237
 if [ -z "${HOST_CC}" ] ; then
     echo "Can't find clang.  Please install clang-3.5 or a later version."
     exit 1

utils/swift_build_support/README.md

@@ -0,0 +1,10 @@
+# swift_build_support
+
+`swift_build_support` is a Python module containing functions and data
+structures used by the Swift build script.
+
+You may run unit tests for `swift_build_support` from the command line:
+
+```sh
+apple/swift $ python -m unittest discover -s utils/swift_build_support
+```

utils/swift_build_support/swift_build_support/__init__.py

@@ -0,0 +1,16 @@
+# swift_build_support/__init__.py - Helpers for building Swift -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+#
+# This file needs to be here in order for Python to treat the
+# utils/swift_build_support/ directory as a module.
+#
+# ----------------------------------------------------------------------------

utils/swift_build_support/swift_build_support/clang.py

@@ -0,0 +1,90 @@
+# swift_build_support/clang.py - Detect host machine's Clang -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+#
+# Find the path to a Clang executable on the host machine that is most
+# suitable for building Swift.
+#
+# ----------------------------------------------------------------------------
+
+from __future__ import absolute_import
+
+import collections
+import platform
+import subprocess
+
+from . import xcrun
+from .which import which
+
+
+# A named tuple consisting of two paths:
+# 1. 'cc' is the path to a program used for compiling C.
+# 2. 'cxx' is the path to a program used for compiling C++.
+CompilerExecutable = collections.namedtuple('CompilerExecutable', 'cc cxx')
+
+
+def _freebsd_release_date():
+    """
+    Return the release date for FreeBSD operating system on this host.
+    If the release date cannot be ascertained, return None.
+    """
+    try:
+        # For details on `sysctl`, see:
+        # http://www.freebsd.org/cgi/man.cgi?sysctl(8)
+        return int(subprocess.check_output(
+            ['sysctl', '-n', 'kern.osreldate']).rstrip())
+    except subprocess.CalledProcessError:
+        return None
+
+
+def _first_clang(suffixes):
+    """
+    Return a CompilerExecutable with the first available versions of clang
+    and clang++, searching in the order of the given suffixes.
+
+    If no Clang executables are found, return None.
+    """
+    for suffix in suffixes:
+        cc_path = which('clang{}'.format(suffix))
+        cxx_path = which('clang++{}'.format(suffix))
+        if cc_path and cxx_path:
+            return CompilerExecutable(cc=cc_path, cxx=cxx_path)
+
+    return None
+
+
+def host_clang(xcrun_toolchain):
+    """
+    Return a CompilerExecutable for the host platform.
+    If no appropriate compilers can be found, return None.
+    """
+    if platform.system() == 'Darwin':
+        cc = xcrun.find(xcrun_toolchain, 'clang')
+        cxx = xcrun.find(xcrun_toolchain, 'clang++')
+        if cc and cxx:
+            return CompilerExecutable(cc=cc, cxx=cxx)
+        else:
+            return None
+    elif platform.system() == 'FreeBSD':
+        # See: https://github.com/apple/swift/pull/169
+        # Building Swift from source requires a recent version of the Clang
+        # compiler with C++14 support.
+        freebsd_release_date = _freebsd_release_date()
+        if freebsd_release_date and freebsd_release_date >= 1100000:
+            # On newer releases of FreeBSD, the default Clang is sufficient.
+            return CompilerExecutable(cc='clang', cxx='clang++')
+        else:
+            # On older releases, or on releases for which we cannot determine
+            # the release date, we search for the most modern version
+            # available.
+            return _first_clang(['38', '37', '36', '35'])
+    else:
+        return _first_clang(['', '-3.8', '-3.7', '-3.6', '-3.5'])

utils/swift_build_support/swift_build_support/migration.py

@@ -0,0 +1,56 @@
+# swift_build_support/migration.py - Migrating build-script -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+#
+# utils/build-script takes arguments for its argument parser, as well as
+# arguments that are meant to be passed directly to utils/build-script-impl.
+# In order to gradually migrate away from build-script-impl, this module
+# provides tools to handle parsing of these args.
+#
+# ----------------------------------------------------------------------------
+
+
+def migrate_impl_args(argv, migrate_args):
+    """
+    Given a list of arguments of the form:
+
+        --foo --bar=baz -- --flim=flam
+
+    And a list of arguments to migrate, return a list in which the arguments
+    to migrate come before the '--' separator. For example, were we to migrate
+    '--flim', we would return:
+
+        --foo --bar=baz --flim=flam --
+
+    Note that we do not attempt to remove the '--' separator if it is no longer
+    necessary, nor do we replace '--flim' if it already appears before the
+    separator. In these cases, argparse "does the right thing": it can handle
+    a trailing separator, and when options that are specified twice argparse
+    uses the second value.
+    """
+    try:
+        split_index = argv.index('--')
+    except ValueError:
+        # If there is no separator, then we have nothing to migrate.
+        return argv
+
+    args = argv[:split_index]
+    impl_args = argv[split_index:]
+    impl_args_to_remove = []
+    for index, impl_arg in enumerate(impl_args):
+        if impl_arg.split('=')[0] in migrate_args:
+            args.append(impl_arg)
+            impl_args_to_remove.append(impl_arg)
+
+    for impl_arg_to_remove in impl_args_to_remove:
+        impl_args.remove(impl_arg_to_remove)
+
+    return args + impl_args

utils/swift_build_support/swift_build_support/which.py

@@ -0,0 +1,36 @@
+# swift_build_support/which.py - shutil.which() for Python 2.7 -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+#
+# A naive reimplementation of shutil.which() for Python 2.7. This can be
+# removed if shutil.which() is backported, or if the Swift build toolchain
+# migrates completely to Python 3.3+.
+#
+# ----------------------------------------------------------------------------
+
+import subprocess
+
+
+def which(cmd):
+    """
+    Return the path to an executable which would be run if
+    the given cmd was called. If no cmd would be called, return None.
+
+    Python 3.3+ provides this behavior via the shutil.which() function;
+    see: https://docs.python.org/3.3/library/shutil.html#shutil.which
+
+    We provide our own implementation because shutil.which() has not
+    been backported to Python 2.7, which we support.
+    """
+    try:
+        return subprocess.check_output(['which', cmd]).rstrip()
+    except subprocess.CalledProcessError:
+        return None

utils/swift_build_support/swift_build_support/xcrun.py

@@ -0,0 +1,34 @@
+# swift_build_support/xcrun.py - Invoke xcrun from Python -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+#
+# Python wrappers for invoking `xcrun` on the command-line.
+#
+# ----------------------------------------------------------------------------
+
+import subprocess
+
+
+def find(toolchain, tool):
+    """
+    Return the path for the given tool, according to `xcrun --find`, using
+    the given toolchain. If `xcrun --find` cannot find the tool, return None.
+    """
+    try:
+        # `xcrun --find` prints to stderr when it fails to find the
+        # given tool. We swallow that output with a pipe.
+        out = subprocess.check_output(['xcrun', '--sdk', 'macosx',
+                                       '--toolchain', toolchain,
+                                       '--find', tool],
+                                      stderr=subprocess.PIPE)
+        return out.rstrip()
+    except subprocess.CalledProcessError:
+        return None

utils/swift_build_support/tests/__init__.py

@@ -0,0 +1,16 @@
+# swift_build_support/tests/__init__.py - Test module -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+#
+# This file needs to be here in order for Python to treat the
+# utils/swift_build_support/tests/ directory as a module.
+#
+# ----------------------------------------------------------------------------