add 'x.py miri', and make it work for 'library/{core,alloc,std}'

Author: RalfJung

library/alloc/benches/vec_deque_append.rs

@@ -5,6 +5,11 @@ const WARMUP_N: usize = 100;
 const BENCH_N: usize = 1000;
 
 fn main() {
+    if cfg!(miri) {
+        // Don't benchmark Miri...
+        // (Due to bootstrap quirks, this gets picked up by `x.py miri library/alloc --no-doc`.)
+        return;
+    }
     let a: VecDeque<i32> = (0..VECDEQUE_LEN).collect();
     let b: VecDeque<i32> = (0..VECDEQUE_LEN).collect();
 

library/alloc/src/lib.miri.rs

@@ -0,0 +1,4 @@
+//! Grep bootstrap for `MIRI_REPLACE_LIBRS_IF_NOT_TEST` to learn what this is about.
+#![no_std]
+extern crate alloc as realalloc;
+pub use realalloc::*;

library/core/src/lib.miri.rs

@@ -0,0 +1,4 @@
+//! Grep bootstrap for `MIRI_REPLACE_LIBRS_IF_NOT_TEST` to learn what this is about.
+#![no_std]
+extern crate core as realcore;
+pub use realcore::*;

library/std/src/lib.miri.rs

@@ -0,0 +1,4 @@
+//! Grep bootstrap for `MIRI_REPLACE_LIBRS_IF_NOT_TEST` to learn what this is about.
+#![no_std]
+extern crate std as realstd;
+pub use realstd::*;

src/bootstrap/src/core/build_steps/test.rs

@@ -2536,9 +2536,14 @@ fn prepare_cargo_test(
     //
     // Note that to run the compiler we need to run with the *host* libraries,
     // but our wrapper scripts arrange for that to be the case anyway.
-    let mut dylib_path = dylib_path();
-    dylib_path.insert(0, PathBuf::from(&*builder.sysroot_libdir(compiler, target)));
-    cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
+    //
+    // We skip everything on Miri as then this overwrites the libdir set up
+    // by `Cargo::new` and that actually makes things go wrong.
+    if builder.kind != Kind::Miri {
+        let mut dylib_path = dylib_path();
+        dylib_path.insert(0, PathBuf::from(&*builder.sysroot_libdir(compiler, target)));
+        cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
+    }
 
     if builder.remote_tested(target) {
         cargo.env(
@@ -2594,28 +2599,62 @@ impl Step for Crate {
         let target = self.target;
         let mode = self.mode;
 
+        // Prepare sysroot
         // See [field@compile::Std::force_recompile].
         builder.ensure(compile::Std::force_recompile(compiler, compiler.host));
 
-        if builder.config.build != target {
-            builder.ensure(compile::Std::force_recompile(compiler, target));
-            builder.ensure(RemoteCopyLibs { compiler, target });
-        }
-
         // If we're not doing a full bootstrap but we're testing a stage2
         // version of libstd, then what we're actually testing is the libstd
         // produced in stage1. Reflect that here by updating the compiler that
         // we're working with automatically.
         let compiler = builder.compiler_for(compiler.stage, compiler.host, target);
 
-        let mut cargo = builder::Cargo::new(
-            builder,
-            compiler,
-            mode,
-            SourceType::InTree,
-            target,
-            builder.kind.as_str(),
-        );
+        let mut cargo = if builder.kind == Kind::Miri {
+            if builder.top_stage == 0 {
+                eprintln!("ERROR: `x.py miri` requires stage 1 or higher");
+                std::process::exit(1);
+            }
+
+            // Build `cargo miri test` command
+            // (Implicitly prepares target sysroot)
+            let mut cargo = builder::Cargo::new(
+                builder,
+                compiler,
+                mode,
+                SourceType::InTree,
+                target,
+                "miri-test",
+            );
+            // This hack helps bootstrap run standard library tests in Miri. The issue is as
+            // follows: when running `cargo miri test` on libcore, cargo builds a local copy of core
+            // and makes it a dependency of the integration test crate. This copy duplicates all the
+            // lang items, so the build fails. (Regular testing avoids this because the sysroot is a
+            // literal copy of what `cargo build` produces, but since Miri builds its own sysroot
+            // this does not work for us.) So we need to make it so that the locally built libcore
+            // contains all the items from `core`, but does not re-define them -- we want to replace
+            // the entire crate but a re-export of the sysroot crate. We do this by swapping out the
+            // source file: if `MIRI_REPLACE_LIBRS_IF_NOT_TEST` is set and we are building a
+            // `lib.rs` file, and a `lib.miri.rs` file exists in the same folder, we build that
+            // instead. But crucially we only do that for the library, not the test builds.
+            cargo.env("MIRI_REPLACE_LIBRS_IF_NOT_TEST", "1");
+            cargo
+        } else {
+            // Also prepare a sysroot for the target.
+            if builder.config.build != target {
+                builder.ensure(compile::Std::force_recompile(compiler, target));
+                builder.ensure(RemoteCopyLibs { compiler, target });
+            }
+
+            // Build `cargo test` command
+            builder::Cargo::new(
+                builder,
+                compiler,
+                mode,
+                SourceType::InTree,
+                target,
+                builder.kind.as_str(),
+            )
+        };
 
         match mode {
             Mode::Std => {

src/bootstrap/src/core/builder.rs

@@ -631,6 +631,7 @@ pub enum Kind {
     Format,
     #[value(alias = "t")]
     Test,
+    Miri,
     Bench,
     #[value(alias = "d")]
     Doc,
@@ -673,6 +674,7 @@ impl Kind {
             Kind::Fix => "fix",
             Kind::Format => "fmt",
             Kind::Test => "test",
+            Kind::Miri => "miri",
             Kind::Bench => "bench",
             Kind::Doc => "doc",
             Kind::Clean => "clean",
@@ -822,6 +824,7 @@ impl<'a> Builder<'a> {
                 // Run run-make last, since these won't pass without make on Windows
                 test::RunMake,
             ),
+            Kind::Miri => describe!(test::Crate),
             Kind::Bench => describe!(test::Crate, test::CrateLibrustc),
             Kind::Doc => describe!(
                 doc::UnstableBook,
@@ -970,6 +973,7 @@ impl<'a> Builder<'a> {
             Subcommand::Fix => (Kind::Fix, &paths[..]),
             Subcommand::Doc { .. } => (Kind::Doc, &paths[..]),
             Subcommand::Test { .. } => (Kind::Test, &paths[..]),
+            Subcommand::Miri { .. } => (Kind::Miri, &paths[..]),
             Subcommand::Bench { .. } => (Kind::Bench, &paths[..]),
             Subcommand::Dist => (Kind::Dist, &paths[..]),
             Subcommand::Install => (Kind::Install, &paths[..]),
@@ -1309,7 +1313,11 @@ impl<'a> Builder<'a> {
         if cmd == "clippy" {
             cargo = self.cargo_clippy_cmd(compiler);
             cargo.arg(cmd);
-        } else if let Some(subcmd) = cmd.strip_prefix("miri-") {
+        } else if let Some(subcmd) = cmd.strip_prefix("miri") {
+            // Command must be "miri-X".
+            let subcmd = subcmd
+                .strip_prefix("-")
+                .unwrap_or_else(|| panic!("expected `miri-$subcommand`, but got {}", cmd));
             cargo = self.cargo_miri_cmd(compiler);
             cargo.arg("miri").arg(subcmd);
         } else {

src/bootstrap/src/core/config/config.rs

@@ -2022,7 +2022,7 @@ impl Config {
             Subcommand::Build { .. } => {
                 flags.stage.or(build_stage).unwrap_or(if download_rustc { 2 } else { 1 })
             }
-            Subcommand::Test { .. } => {
+            Subcommand::Test { .. } | Subcommand::Miri { .. } => {
                 flags.stage.or(test_stage).unwrap_or(if download_rustc { 2 } else { 1 })
             }
             Subcommand::Bench { .. } => flags.stage.or(bench_stage).unwrap_or(2),
@@ -2044,6 +2044,7 @@ impl Config {
         if flags.stage.is_none() && crate::CiEnv::current() != crate::CiEnv::None {
             match config.cmd {
                 Subcommand::Test { .. }
+                | Subcommand::Miri { .. }
                 | Subcommand::Doc { .. }
                 | Subcommand::Build { .. }
                 | Subcommand::Bench { .. }
@@ -2099,7 +2100,9 @@ impl Config {
 
     pub(crate) fn test_args(&self) -> Vec<&str> {
         let mut test_args = match self.cmd {
-            Subcommand::Test { ref test_args, .. } | Subcommand::Bench { ref test_args, .. } => {
+            Subcommand::Test { ref test_args, .. }
+            | Subcommand::Bench { ref test_args, .. }
+            | Subcommand::Miri { ref test_args, .. } => {
                 test_args.iter().flat_map(|s| s.split_whitespace()).collect()
             }
             _ => vec![],

src/bootstrap/src/core/config/flags.rs

@@ -382,6 +382,25 @@ pub enum Subcommand {
         /// `/<build_base>/rustfix_missing_coverage.txt`
         rustfix_coverage: bool,
     },
+    /// Build and run some test suites *in Miri*
+    Miri {
+        #[arg(long)]
+        /// run all tests regardless of failure
+        no_fail_fast: bool,
+        #[arg(long, value_name = "ARGS", allow_hyphen_values(true))]
+        /// extra arguments to be passed for the test tool being used
+        /// (e.g. libtest, compiletest or rustdoc)
+        test_args: Vec<String>,
+        /// extra options to pass the compiler when running tests
+        #[arg(long, value_name = "ARGS", allow_hyphen_values(true))]
+        rustc_args: Vec<String>,
+        #[arg(long)]
+        /// do not run doc tests
+        no_doc: bool,
+        #[arg(long)]
+        /// only run doc tests
+        doc: bool,
+    },
     /// Build and run some benchmarks
     Bench {
         #[arg(long, allow_hyphen_values(true))]
@@ -453,6 +472,7 @@ impl Subcommand {
             Subcommand::Fix { .. } => Kind::Fix,
             Subcommand::Format { .. } => Kind::Format,
             Subcommand::Test { .. } => Kind::Test,
+            Subcommand::Miri { .. } => Kind::Miri,
             Subcommand::Clean { .. } => Kind::Clean,
             Subcommand::Dist { .. } => Kind::Dist,
             Subcommand::Install { .. } => Kind::Install,
@@ -464,7 +484,7 @@ impl Subcommand {
 
     pub fn rustc_args(&self) -> Vec<&str> {
         match *self {
-            Subcommand::Test { ref rustc_args, .. } => {
+            Subcommand::Test { ref rustc_args, .. } | Subcommand::Miri { ref rustc_args, .. } => {
                 rustc_args.iter().flat_map(|s| s.split_whitespace()).collect()
             }
             _ => vec![],
@@ -473,14 +493,16 @@ impl Subcommand {
 
     pub fn fail_fast(&self) -> bool {
         match *self {
-            Subcommand::Test { no_fail_fast, .. } => !no_fail_fast,
+            Subcommand::Test { no_fail_fast, .. } | Subcommand::Miri { no_fail_fast, .. } => {
+                !no_fail_fast
+            }
             _ => false,
         }
     }
 
     pub fn doc_tests(&self) -> DocTests {
         match *self {
-            Subcommand::Test { doc, no_doc, .. } => {
+            Subcommand::Test { doc, no_doc, .. } | Subcommand::Miri { no_doc, doc, .. } => {
                 if doc {
                     DocTests::Only
                 } else if no_doc {

src/etc/completions/x.py.fish

@@ -39,6 +39,7 @@ complete -c x.py -n "__fish_use_subcommand" -f -a "fix" -d 'Run cargo fix'
 complete -c x.py -n "__fish_use_subcommand" -f -a "fmt" -d 'Run rustfmt'
 complete -c x.py -n "__fish_use_subcommand" -f -a "doc" -d 'Build documentation'
 complete -c x.py -n "__fish_use_subcommand" -f -a "test" -d 'Build and run some test suites'
+complete -c x.py -n "__fish_use_subcommand" -f -a "miri" -d 'Build and run some test suites *in Miri*'
 complete -c x.py -n "__fish_use_subcommand" -f -a "bench" -d 'Build and run some benchmarks'
 complete -c x.py -n "__fish_use_subcommand" -f -a "clean" -d 'Clean out build directories'
 complete -c x.py -n "__fish_use_subcommand" -f -a "dist" -d 'Build distribution artifacts'
@@ -308,6 +309,45 @@ complete -c x.py -n "__fish_seen_subcommand_from test" -l llvm-profile-generate
 complete -c x.py -n "__fish_seen_subcommand_from test" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from test" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from test" -s h -l help -d 'Print help (see more with \'--help\')'
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l test-args -d 'extra arguments to be passed for the test tool being used (e.g. libtest, compiletest or rustdoc)' -r
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l rustc-args -d 'extra options to pass the compiler when running tests' -r
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l config -d 'TOML configuration file for build' -r -F
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l build -d 'build target of the stage0 compiler' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l host -d 'host targets to build' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l target -d 'target targets to build' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l exclude -d 'build paths to exclude' -r -F
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l skip -d 'build paths to skip' -r -F
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l rustc-error-format -r -f
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l on-fail -d 'command to run on failure' -r -f -a "(__fish_complete_command)"
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l stage -d 'stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l keep-stage -d 'stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)"
+complete -c x.py -n "__fish_seen_subcommand_from miri" -s j -l jobs -d 'number of jobs to run in parallel' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny	'',warn	'',default	''}"
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l error-format -d 'rustc error format' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always	'',never	'',auto	''}"
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true	'',false	''}"
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l reproducible-artifact -d 'Additional reproducible artifacts that should be added to the reproducible artifacts archive' -r
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l set -d 'override options in config.toml' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l no-fail-fast -d 'run all tests regardless of failure'
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l no-doc -d 'do not run doc tests'
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l doc -d 'only run doc tests'
+complete -c x.py -n "__fish_seen_subcommand_from miri" -s v -l verbose -d 'use verbose output (-vv for very verbose)'
+complete -c x.py -n "__fish_seen_subcommand_from miri" -s i -l incremental -d 'use incremental compilation'
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l include-default-paths -d 'include default paths in addition to the provided ones'
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l dry-run -d 'dry run; don\'t build anything'
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l dump-bootstrap-shims -d 'Indicates whether to dump the work done from bootstrap shims'
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from miri" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
+complete -c x.py -n "__fish_seen_subcommand_from miri" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_seen_subcommand_from bench" -l test-args -r
 complete -c x.py -n "__fish_seen_subcommand_from bench" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from bench" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"