Rollup merge of #150875 - bootstrap-refactor-build-outputs, r=bjorn3

Refactor artifact keep mode in bootstrap

This makes it easier to understand which artifacts will be stored in the build stamp. Should help with #145343.

Before, there were two booleans (keep .rmeta files and a special mode for rustc). I created an explicit enum for it instead. The mapping is:
- `(false, false)` => `ArtifactKeepMode::OnlyRlib`
- `(true, false)` => `ArtifactKeepMode::OnlyRmeta` (used for check/clippy builds)
- `(false, true)` => special rustc mode

r? @bjorn3

Author: matthiaskrgr

src/bootstrap/src/core/build_steps/check.rs

@@ -4,7 +4,8 @@ use std::fs;
 use std::path::{Path, PathBuf};
 
 use crate::core::build_steps::compile::{
-    add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, std_crates_for_run_make,
+    ArtifactKeepMode, add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo,
+    std_crates_for_run_make,
 };
 use crate::core::build_steps::tool;
 use crate::core::build_steps::tool::{
@@ -111,8 +112,7 @@ impl Step for Std {
             builder.config.free_args.clone(),
             &check_stamp,
             vec![],
-            true,
-            false,
+            ArtifactKeepMode::OnlyRmeta,
         );
 
         drop(_guard);
@@ -148,7 +148,14 @@ impl Step for Std {
             build_compiler,
             target,
         );
-        run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false);
+        run_cargo(
+            builder,
+            cargo,
+            builder.config.free_args.clone(),
+            &stamp,
+            vec![],
+            ArtifactKeepMode::OnlyRmeta,
+        );
         check_stamp
     }
 
@@ -368,7 +375,14 @@ impl Step for Rustc {
         let stamp =
             build_stamp::librustc_stamp(builder, build_compiler, target).with_prefix("check");
 
-        run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false);
+        run_cargo(
+            builder,
+            cargo,
+            builder.config.free_args.clone(),
+            &stamp,
+            vec![],
+            ArtifactKeepMode::OnlyRmeta,
+        );
 
         stamp
     }
@@ -568,7 +582,14 @@ impl Step for CraneliftCodegenBackend {
         )
         .with_prefix("check");
 
-        run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false);
+        run_cargo(
+            builder,
+            cargo,
+            builder.config.free_args.clone(),
+            &stamp,
+            vec![],
+            ArtifactKeepMode::OnlyRmeta,
+        );
     }
 
     fn metadata(&self) -> Option<StepMetadata> {
@@ -639,7 +660,14 @@ impl Step for GccCodegenBackend {
         )
         .with_prefix("check");
 
-        run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false);
+        run_cargo(
+            builder,
+            cargo,
+            builder.config.free_args.clone(),
+            &stamp,
+            vec![],
+            ArtifactKeepMode::OnlyRmeta,
+        );
     }
 
     fn metadata(&self) -> Option<StepMetadata> {
@@ -777,7 +805,14 @@ fn run_tool_check_step(
         .with_prefix(&format!("{display_name}-check"));
 
     let _guard = builder.msg(builder.kind, display_name, mode, build_compiler, target);
-    run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false);
+    run_cargo(
+        builder,
+        cargo,
+        builder.config.free_args.clone(),
+        &stamp,
+        vec![],
+        ArtifactKeepMode::OnlyRmeta,
+    );
 }
 
 tool_check_step!(Rustdoc {

src/bootstrap/src/core/build_steps/clippy.rs

@@ -15,7 +15,7 @@
 
 use build_helper::exit;
 
-use super::compile::{run_cargo, rustc_cargo, std_cargo};
+use super::compile::{ArtifactKeepMode, run_cargo, rustc_cargo, std_cargo};
 use super::tool::{SourceType, prepare_tool_cargo};
 use crate::builder::{Builder, ShouldRun};
 use crate::core::build_steps::check::{CompilerForCheck, prepare_compiler_for_check};
@@ -214,8 +214,7 @@ impl Step for Std {
             lint_args(builder, &self.config, IGNORED_RULES_FOR_STD_AND_RUSTC),
             &build_stamp::libstd_stamp(builder, build_compiler, target),
             vec![],
-            true,
-            false,
+            ArtifactKeepMode::OnlyRmeta,
         );
     }
 
@@ -309,8 +308,7 @@ impl Step for Rustc {
             lint_args(builder, &self.config, IGNORED_RULES_FOR_STD_AND_RUSTC),
             &build_stamp::librustc_stamp(builder, build_compiler, target),
             vec![],
-            true,
-            false,
+            ArtifactKeepMode::OnlyRmeta,
         );
     }
 
@@ -380,7 +378,7 @@ impl Step for CodegenGcc {
             .with_prefix("rustc_codegen_gcc-check");
 
         let args = lint_args(builder, &self.config, &[]);
-        run_cargo(builder, cargo, args.clone(), &stamp, vec![], true, false);
+        run_cargo(builder, cargo, args.clone(), &stamp, vec![], ArtifactKeepMode::OnlyRmeta);
 
         // Same but we disable the features enabled by default.
         let mut cargo = prepare_tool_cargo(
@@ -396,7 +394,7 @@ impl Step for CodegenGcc {
         self.build_compiler.configure_cargo(&mut cargo);
         println!("Now running clippy on `rustc_codegen_gcc` with `--no-default-features`");
         cargo.arg("--no-default-features");
-        run_cargo(builder, cargo, args, &stamp, vec![], true, false);
+        run_cargo(builder, cargo, args, &stamp, vec![], ArtifactKeepMode::OnlyRmeta);
     }
 
     fn metadata(&self) -> Option<StepMetadata> {
@@ -478,8 +476,7 @@ macro_rules! lint_any {
                     lint_args(builder, &self.config, &[]),
                     &stamp,
                     vec![],
-                    true,
-                    false,
+                    ArtifactKeepMode::OnlyRmeta
                 );
             }
 

src/bootstrap/src/core/build_steps/compile.rs

@@ -296,8 +296,11 @@ impl Step for Std {
             vec![],
             &stamp,
             target_deps,
-            self.is_for_mir_opt_tests, // is_check
-            false,
+            if self.is_for_mir_opt_tests {
+                ArtifactKeepMode::OnlyRmeta
+            } else {
+                ArtifactKeepMode::OnlyRlib
+            },
         );
 
         builder.ensure(StdLink::from_std(
@@ -1167,14 +1170,28 @@ impl Step for Rustc {
             target,
         );
         let stamp = build_stamp::librustc_stamp(builder, build_compiler, target);
+
         run_cargo(
             builder,
             cargo,
             vec![],
             &stamp,
             vec![],
-            false,
-            true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files.
+            ArtifactKeepMode::Custom(Box::new(|filename| {
+                if filename.contains("jemalloc_sys")
+                    || filename.contains("rustc_public_bridge")
+                    || filename.contains("rustc_public")
+                {
+                    // jemalloc_sys and rustc_public_bridge are not linked into librustc_driver.so,
+                    // so we need to distribute them as rlib to be able to use them.
+                    filename.ends_with(".rlib")
+                } else {
+                    // Distribute the rest of the rustc crates as rmeta files only to reduce
+                    // the tarball sizes by about 50%. The object files are linked into
+                    // librustc_driver.so, so it is still possible to link against them.
+                    filename.ends_with(".rmeta")
+                }
+            })),
         );
 
         let target_root_dir = stamp.path().parent().unwrap();
@@ -1714,7 +1731,7 @@ impl Step for GccCodegenBackend {
 
         let _guard =
             builder.msg(Kind::Build, "codegen backend gcc", Mode::Codegen, build_compiler, host);
-        let files = run_cargo(builder, cargo, vec![], &stamp, vec![], false, false);
+        let files = run_cargo(builder, cargo, vec![], &stamp, vec![], ArtifactKeepMode::OnlyRlib);
 
         GccCodegenBackendOutput {
             stamp: write_codegen_backend_stamp(stamp, files, builder.config.dry_run()),
@@ -1790,7 +1807,7 @@ impl Step for CraneliftCodegenBackend {
             build_compiler,
             target,
         );
-        let files = run_cargo(builder, cargo, vec![], &stamp, vec![], false, false);
+        let files = run_cargo(builder, cargo, vec![], &stamp, vec![], ArtifactKeepMode::OnlyRlib);
         write_codegen_backend_stamp(stamp, files, builder.config.dry_run())
     }
 
@@ -2620,14 +2637,26 @@ pub fn add_to_sysroot(
     }
 }
 
+/// Specifies which rlib/rmeta artifacts outputted by Cargo should be put into the resulting
+/// build stamp, and thus be included in dist archives and copied into sysroots by default.
+/// Note that some kinds of artifacts are copied automatically (e.g. native libraries).
+pub enum ArtifactKeepMode {
+    /// Only keep .rlib files, ignore .rmeta files
+    OnlyRlib,
+    /// Only keep .rmeta files, ignore .rlib files
+    OnlyRmeta,
+    /// Custom logic for keeping an artifact
+    /// It receives the filename of an artifact, and returns true if it should be kept.
+    Custom(Box<dyn Fn(&str) -> bool>),
+}
+
 pub fn run_cargo(
     builder: &Builder<'_>,
     cargo: Cargo,
     tail_args: Vec<String>,
     stamp: &BuildStamp,
     additional_target_deps: Vec<(PathBuf, DependencyType)>,
-    is_check: bool,
-    rlib_only_metadata: bool,
+    artifact_keep_mode: ArtifactKeepMode,
 ) -> Vec<PathBuf> {
     // `target_root_dir` looks like $dir/$target/release
     let target_root_dir = stamp.path().parent().unwrap();
@@ -2661,36 +2690,20 @@ pub fn run_cargo(
         };
         for filename in filenames_vec {
             // Skip files like executables
-            let mut keep = false;
-            if filename.ends_with(".lib")
+            let keep = if filename.ends_with(".lib")
                 || filename.ends_with(".a")
                 || is_debug_info(&filename)
                 || is_dylib(Path::new(&*filename))
             {
                 // Always keep native libraries, rust dylibs and debuginfo
-                keep = true;
-            }
-            if is_check && filename.ends_with(".rmeta") {
-                // During check builds we need to keep crate metadata
-                keep = true;
-            } else if rlib_only_metadata {
-                if filename.contains("jemalloc_sys")
-                    || filename.contains("rustc_public_bridge")
-                    || filename.contains("rustc_public")
-                {
-                    // jemalloc_sys and rustc_public_bridge are not linked into librustc_driver.so,
-                    // so we need to distribute them as rlib to be able to use them.
-                    keep |= filename.ends_with(".rlib");
-                } else {
-                    // Distribute the rest of the rustc crates as rmeta files only to reduce
-                    // the tarball sizes by about 50%. The object files are linked into
-                    // librustc_driver.so, so it is still possible to link against them.
-                    keep |= filename.ends_with(".rmeta");
-                }
+                true
             } else {
-                // In all other cases keep all rlibs
-                keep |= filename.ends_with(".rlib");
-            }
+                match &artifact_keep_mode {
+                    ArtifactKeepMode::OnlyRlib => filename.ends_with(".rlib"),
+                    ArtifactKeepMode::OnlyRmeta => filename.ends_with(".rmeta"),
+                    ArtifactKeepMode::Custom(func) => func(&filename),
+                }
+            };
 
             if !keep {
                 continue;

src/bootstrap/src/core/build_steps/test.rs

@@ -14,7 +14,7 @@ use std::{env, fs, iter};
 
 use build_helper::exit;
 
-use crate::core::build_steps::compile::{Std, run_cargo};
+use crate::core::build_steps::compile::{ArtifactKeepMode, Std, run_cargo};
 use crate::core::build_steps::doc::{DocumentationFormat, prepare_doc_compiler};
 use crate::core::build_steps::gcc::{Gcc, GccTargetPair, add_cg_gcc_cargo_flags};
 use crate::core::build_steps::llvm::get_llvm_version;
@@ -2587,7 +2587,8 @@ impl BookTest {
                 let stamp = BuildStamp::new(&builder.cargo_out(test_compiler, mode, target))
                     .with_prefix(PathBuf::from(dep).file_name().and_then(|v| v.to_str()).unwrap());
 
-                let output_paths = run_cargo(builder, cargo, vec![], &stamp, vec![], false, false);
+                let output_paths =
+                    run_cargo(builder, cargo, vec![], &stamp, vec![], ArtifactKeepMode::OnlyRlib);
                 let directories = output_paths
                     .into_iter()
                     .filter_map(|p| p.parent().map(ToOwned::to_owned))