Add option to include submodules from vendoring

This adds a config option `exclude-submodules-from-vendoring` that
explicitly filters out listed submodules from vendoring. This allows us
to exclude submodules like `rustc-perf`, which vendors a number of
libraries for benchmarking, and has a complicated licensing story.

Closes #137272

Author: erickt

config.example.toml

@@ -981,3 +981,6 @@
 
 # Whether to vendor dependencies for the dist tarball.
 #vendor = if "is a tarball source" || "is a git repository" { true } else { false }
+
+# Exclude these submodules from vendoring.
+# exclude-submodules-from-vendoring = ["src/tools/rustc-perf"]

src/bootstrap/src/core/build_steps/run.rs

@@ -212,6 +212,9 @@ impl Step for GenerateCopyright {
         let paths_to_vendor = default_paths_to_vendor(builder);
         for (_, submodules) in &paths_to_vendor {
             for submodule in submodules {
+                if builder.config.dist_exclude_submodules_from_vendoring.contains(&**submodule) {
+                    continue;
+                }
                 builder.build.require_submodule(submodule, None);
             }
         }

src/bootstrap/src/core/build_steps/vendor.rs

@@ -86,6 +86,9 @@ impl Step for Vendor {
         // These submodules must be present for `x vendor` to work.
         for (_, submodules) in &to_vendor {
             for submodule in submodules {
+                if builder.config.dist_exclude_submodules_from_vendoring.contains(&**submodule) {
+                    continue;
+                }
                 builder.build.require_submodule(submodule, None);
             }
         }

src/bootstrap/src/core/config/config.rs

@@ -339,6 +339,7 @@ pub struct Config {
     pub dist_compression_profile: String,
     pub dist_include_mingw_linker: bool,
     pub dist_vendor: bool,
+    pub dist_exclude_submodules_from_vendoring: BTreeSet<String>,
 
     // libstd features
     pub backtrace: bool, // support for RUST_BACKTRACE
@@ -1001,6 +1002,7 @@ define_config! {
         compression_profile: Option<String> = "compression-profile",
         include_mingw_linker: Option<bool> = "include-mingw-linker",
         vendor: Option<bool> = "vendor",
+        exclude_submodules_from_vendoring: Option<BTreeSet<String>> = "exclude-submodules-from-vendoring",
     }
 }
 
@@ -2227,10 +2229,13 @@ impl Config {
                 compression_profile,
                 include_mingw_linker,
                 vendor,
+                exclude_submodules_from_vendoring,
             } = dist;
             config.dist_sign_folder = sign_folder.map(PathBuf::from);
             config.dist_upload_addr = upload_addr;
             config.dist_compression_formats = compression_formats;
+            config.dist_exclude_submodules_from_vendoring =
+                exclude_submodules_from_vendoring.unwrap_or_else(BTreeSet::new);
             set(&mut config.dist_compression_profile, compression_profile);
             set(&mut config.rust_dist_src, src_tarball);
             set(&mut config.dist_include_mingw_linker, include_mingw_linker);

src/bootstrap/src/utils/change_tracker.rs

@@ -360,4 +360,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[
         severity: ChangeSeverity::Info,
         summary: "Added `build.test-stage = 2` to 'tools' profile defaults",
     },
+    ChangeInfo {
+        change_id: 137583,
+        severity: ChangeSeverity::Info,
+        summary: "It is now possible to filter out submodules from vendoring with `dist.exclude-submodules-from-vendoring`",
+    },
 ];