feat(exec-harness): implement the walltime benchmarking algorithm

Author: GuillaumeLagrange

Cargo.lock

@@ -40,7 +40,7 @@ async-compression = { version = "0.4.5", features = ["tokio", "gzip"] }
 simplelog = { version = "0.12.1", default-features = false, features = [
     "termcolor",
 ] }
-tempfile = "3.10.0"
+tempfile = { workspace = true }
 git2 = "0.20.2"
 nestify = "0.3.3"
 gql_client = { git = "https://github.com/CodSpeedHQ/gql-client-rs" }
@@ -59,6 +59,7 @@ nix = { version = "0.29.0", features = ["fs", "time", "user"] }
 futures = "0.3.31"
 runner-shared = { path = "crates/runner-shared" }
 memtrack = { path = "crates/memtrack", default-features = false }
+exec-harness = { path = "crates/exec-harness" }
 ipc-channel = "0.18"
 shellexpand = { version = "3.1.1", features = ["tilde"] }
 addr2line = "0.25"
@@ -85,14 +86,15 @@ members = ["crates/runner-shared", "crates/memtrack", "crates/exec-harness"]
 
 [workspace.dependencies]
 anyhow = "1.0"
-clap = { version = "4.5", features = ["derive"] }
+clap = { version = "4.5", features = ["derive", "env"] }
 libc = "0.2"
 log = "0.4.28"
 serde_json = "1.0"
 serde = { version = "1.0.228", features = ["derive"] }
 ipc-channel = "0.18"
 itertools = "0.14.0"
 env_logger = "0.11.8"
+tempfile = "3.10.0"
 
 [workspace.metadata.release]
 sign-tag = true

Cargo.toml

@@ -5,12 +5,27 @@ edition = "2024"
 repository = "https://github.com/CodSpeedHQ/runner"
 publish = false
 
+[lib]
+name = "exec_harness"
+path = "src/lib.rs"
+
+[[bin]]
+name = "exec-harness"
+path = "src/main.rs"
+
 [dependencies]
 anyhow = { workspace = true }
 codspeed = "4.1.0"
+log = { workspace = true }
+env_logger = { workspace = true }
 clap = { workspace = true }
 serde_json = { workspace = true }
 serde = { workspace = true }
+humantime = "2.1"
+runner-shared = { path = "../runner-shared" }
+
+[dev-dependencies]
+tempfile = { workspace = true }
 
 [package.metadata.dist]
 targets = ["aarch64-unknown-linux-musl", "x86_64-unknown-linux-musl"]

crates/exec-harness/Cargo.toml

@@ -0,0 +1,7 @@
+//! CodSpeed exec-harness library
+//!
+//! This library provides the core functionality for wrapping commands
+//! with CodSpeed performance instrumentation.
+
+mod prelude;
+pub mod walltime;

crates/exec-harness/src/lib.rs

@@ -1,13 +1,11 @@
+use crate::prelude::*;
 use crate::walltime::WalltimeResults;
-use anyhow::Context;
-use anyhow::Result;
-use anyhow::bail;
 use clap::Parser;
 use codspeed::instrument_hooks::InstrumentHooks;
-use codspeed::walltime_results::WalltimeBenchmark;
+use runner_shared::walltime_results::WalltimeBenchmark;
 use std::path::PathBuf;
-use std::process;
 
+mod prelude;
 mod walltime;
 
 #[derive(Parser, Debug)]
@@ -21,11 +19,19 @@ struct Args {
     #[arg(long)]
     name: Option<String>,
 
+    #[command(flatten)]
+    execution_args: walltime::WalltimeExecutionArgs,
+
     /// The command and arguments to execute
     command: Vec<String>,
 }
 
 fn main() -> Result<()> {
+    env_logger::builder()
+        .parse_env(env_logger::Env::new().filter_or("CODSPEED_LOG", "info"))
+        .format_timestamp(None)
+        .init();
+
     let args = Args::parse();
 
     if args.command.is_empty() {
@@ -49,46 +55,18 @@ fn main() -> Result<()> {
         .set_integration("codspeed-rust", env!("CARGO_PKG_VERSION"))
         .unwrap();
 
-    const NUM_ITERATIONS: usize = 1;
-    let mut times_per_round_ns = Vec::with_capacity(NUM_ITERATIONS);
-
-    hooks.start_benchmark().unwrap();
-    for _ in 0..NUM_ITERATIONS {
-        // Spawn the command
-        let mut child = process::Command::new(&args.command[0])
-            .args(&args.command[1..])
-            .spawn()
-            .context("Failed to spawn command")?;
-
-        // Start monotonic timer for this iteration
-        let bench_start = InstrumentHooks::current_timestamp();
-
-        // Wait for the process to complete
-        let status = child.wait().context("Failed to wait for command")?;
-
-        // Measure elapsed time
-        let bench_end = InstrumentHooks::current_timestamp();
-        hooks.add_benchmark_timestamps(bench_start, bench_end);
-
-        // Exit immediately if any iteration fails
-        if !status.success() {
-            bail!("Command failed with exit code: {:?}", status.code());
-        }
-
-        // Calculate and store the elapsed time in nanoseconds
-        let elapsed_ns = (bench_end - bench_start) as u128;
-        times_per_round_ns.push(elapsed_ns);
-    }
+    // Build execution options from CLI args
+    let execution_options: walltime::ExecutionOptions = args.execution_args.try_into()?;
 
-    hooks.stop_benchmark().unwrap();
-    hooks.set_executed_benchmark(&bench_uri).unwrap();
+    let times_per_round_ns =
+        walltime::perform(bench_uri.clone(), args.command, &execution_options)?;
 
     // Collect walltime results
     let max_time_ns = times_per_round_ns.iter().copied().max();
     let walltime_benchmark = WalltimeBenchmark::from_runtime_data(
         bench_name.clone(),
         bench_uri.clone(),
-        vec![1; NUM_ITERATIONS],
+        vec![1; times_per_round_ns.len()],
         times_per_round_ns,
         max_time_ns,
     );

crates/exec-harness/src/main.rs

@@ -0,0 +1,6 @@
+pub use anyhow::Context;
+pub use anyhow::Result;
+pub use anyhow::bail;
+
+#[allow(unused_imports)]
+pub use log::{debug, error, info, trace, warn};