[build-script] Allow specifying the number of lit workers

Allow tests to run with a different number of
workers than build jobs if desired.

Author: hamishknight

utils/build-script

@@ -500,6 +500,11 @@ def parse_preset_args():
         help="the number of parallel build jobs to use",
         type=int,
         dest="build_jobs")
+    parser.add_argument(
+        "--lit-jobs",
+        help="the number of workers to use when testing with lit",
+        type=int,
+        dest="lit_jobs")
     parser.add_argument(
         "preset_substitutions_raw",
         help="'name=value' pairs that are substituted in the preset",
@@ -599,6 +604,8 @@ def main_preset():
         build_script_args += ["--sccache"]
     if args.build_jobs:
         build_script_args += ["--jobs", str(args.build_jobs)]
+    if args.lit_jobs:
+        build_script_args += ["--lit-jobs", str(args.lit_jobs)]
     if args.swiftsyntax_install_prefix:
         build_script_args += ["--install-swiftsyntax",
                               "--install-destdir",

utils/build-script-impl

@@ -62,6 +62,7 @@ KNOWN_SETTINGS=(
     build-args                                    ""                "arguments to the build tool; defaults to -j8 when CMake generator is \"Unix Makefiles\""
     build-dir                                     ""                "out-of-tree build directory; default is in-tree. **This argument is required**"
     build-jobs                                    ""                "The number of parallel build jobs to use"
+    lit-jobs                                      ""                "The number of workers to use when testing with lit"
     build-runtime-with-host-compiler              ""                "use the host c++ compiler to build everything"
     build-stdlib-deployment-targets               "all"             "space-separated list that filters which of the configured targets to build the Swift standard library for, or 'all'"
     build-toolchain-only                          ""                "If set, only build the necessary tools to build an external toolchain"
@@ -829,10 +830,10 @@ function set_build_options_for_host() {
     )
 
     llvm_cmake_options+=(
-        -DLLVM_LIT_ARGS="${LLVM_LIT_ARGS} -j ${BUILD_JOBS}"
+        -DLLVM_LIT_ARGS="${LLVM_LIT_ARGS} -j ${LIT_JOBS}"
     )
     swift_cmake_options+=(
-        -DLLVM_LIT_ARGS="${LLVM_LIT_ARGS} -j ${BUILD_JOBS}"
+        -DLLVM_LIT_ARGS="${LLVM_LIT_ARGS} -j ${LIT_JOBS}"
     )
 
     if [[ "${CLANG_PROFILE_INSTR_USE}" ]]; then
@@ -2949,7 +2950,7 @@ for host in "${ALL_HOSTS[@]}"; do
 
                 if [[ "${ENABLE_ASAN}" ]] ; then
                     # Limit the number of parallel tests
-                    LLVM_LIT_ARGS="${LLVM_LIT_ARGS} -j $(sysctl hw.physicalcpu | awk -v N=${BUILD_JOBS} '{ print (N < $2) ? N : int($2/2) }')"
+                    LLVM_LIT_ARGS="${LLVM_LIT_ARGS} -j $(sysctl hw.physicalcpu | awk -v N=${LIT_JOBS} '{ print (N < $2) ? N : int($2/2) }')"
                 fi
 
                 FILTER_SWIFT_OPTION="--filter=[sS]wift"

utils/build_swift/build_swift/driver_arguments.py

@@ -153,6 +153,10 @@ def _apply_default_arguments(args):
     if not args.android or not args.build_android:
         args.build_android = False
 
+    # By default use the same number of lit workers as build jobs.
+    if not args.lit_jobs:
+        args.lit_jobs = args.build_jobs
+
     # --test-paths implies --test and/or --validation-test
     # depending on what directories/files have been specified.
     if args.test_paths:
@@ -379,6 +383,8 @@ def create_argument_parser():
     option(['-j', '--jobs'], store_int('build_jobs'),
            default=multiprocessing.cpu_count(),
            help='the number of parallel build jobs to use')
+    option(['--lit-jobs'], store_int('lit_jobs'),
+           help='the number of workers to use when testing with lit')
 
     option('--darwin-xcrun-toolchain', store,
            help='the name of the toolchain to use on Darwin')

utils/build_swift/tests/expected_options.py

@@ -186,6 +186,7 @@
     'libdispatch_build_variant': 'Debug',
     'libicu_build_variant': 'Debug',
     'libxml2_build_variant': 'Debug',
+    'lit_jobs': multiprocessing.cpu_count(),
     'zlib_build_variant': 'Debug',
     'curl_build_variant': 'Debug',
     'bootstrapping_mode': None,
@@ -732,6 +733,7 @@ class BuildScriptImplOption(_BaseOption):
     IntOption('--swift-tools-max-parallel-lto-link-jobs'),
     EnableOption('--swift-tools-ld64-lto-codegen-only-for-supporting-targets'),
     IntOption('-j', dest='build_jobs'),
+    IntOption('--lit-jobs', dest='lit_jobs'),
     IntOption('--dsymutil-jobs', dest='dsymutil_jobs'),
 
     AppendOption('--cross-compile-hosts'),

utils/swift_build_support/swift_build_support/build_script_invocation.py

@@ -129,6 +129,7 @@ def convert_to_impl_arguments(self):
             "--cross-compile-append-host-target-to-destdir", str(
                 args.cross_compile_append_host_target_to_destdir).lower(),
             "--build-jobs", str(args.build_jobs),
+            "--lit-jobs", str(args.lit_jobs),
             "--common-cmake-options=%s" % ' '.join(
                 pipes.quote(opt) for opt in cmake.common_options()),
             "--build-args=%s" % ' '.join(