diff --git a/Cargo.lock b/Cargo.lock index 819ecbc878ef..bb042b8cd7fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -193,6 +193,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a6904aef64d73cf10ab17ebace7befb918b82164785cb89907993be7f83813" + [[package]] name = "block-buffer" version = "0.9.0" @@ -456,7 +462,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "190814073e85d238f31ff738fcb0bf6910cedeb73376c87cd69291028966fd83" dependencies = [ "atty", - "bitflags", + "bitflags 1.3.2", "clap_derive", "clap_lex", "indexmap", @@ -998,6 +1004,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + [[package]] name = "der" version = "0.4.5" @@ -1395,6 +1410,19 @@ dependencies = [ "byteorder", ] +[[package]] +name = "fxprof-processed-profile" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27d12c0aed7f1e24276a241aadc4cb8ea9f83000f34bc062b7cc2d51e3b0fabd" +dependencies = [ + "bitflags 2.2.1", + "debugid", + "fxhash", + "serde", + "serde_json", +] + [[package]] name = "generic-array" version = "0.14.5" @@ -2392,7 +2420,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0d9cc07f18492d879586c92b485def06bc850da3118075cd45d50e9c95b0e5" dependencies = [ "bit-set", - "bitflags", + "bitflags 1.3.2", "byteorder", "lazy_static", "num-traits", @@ -2420,7 +2448,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffade02495f22453cd593159ea2f59827aae7f53fa8323f756799b670881dcf8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "memchr", "unicase", ] @@ -2562,7 +2590,7 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -2619,7 +2647,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877e54ea2adcd70d80e9179344c97f93ef0dffd6b03e1f4529e6e83ab2fa9ae0" dependencies = [ - "bitflags", + "bitflags 1.3.2", "libc", "mach", "winapi", @@ -2688,7 +2716,7 @@ version = "0.37.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f79bef90eb6d984c72722595b5b1348ab39275a5e5123faca6863bf07d75a4e0" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "itoa", @@ -2996,7 +3024,7 @@ version = "0.25.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "928ebd55ab758962e230f51ca63735c5b283f26292297c81404289cda5d78631" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cap-fs-ext", "cap-std", "fd-lock", @@ -3327,7 +3355,7 @@ version = "0.44.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3f92c29dd66c7342443280695afc5bb79d773c3aa3eb02978cf24f058ae2b3d" dependencies = [ - "bitflags", + "bitflags 1.3.2", "fslock", "lazy_static", "libc", @@ -3422,7 +3450,7 @@ name = "wasi-common" version = "9.0.0" dependencies = [ "anyhow", - "bitflags", + "bitflags 1.3.2", "cap-rand", "cap-std", "io-extras", @@ -3685,6 +3713,7 @@ dependencies = [ "bumpalo", "cfg-if", "encoding_rs", + "fxprof-processed-profile", "indexmap", "libc", "log", @@ -3694,6 +3723,7 @@ dependencies = [ "psm", "rayon", "serde", + "serde_json", "target-lexicon", "tempfile", "wasi-cap-std-sync", @@ -4258,7 +4288,7 @@ version = "9.0.0" dependencies = [ "anyhow", "async-trait", - "bitflags", + "bitflags 1.3.2", "proptest", "thiserror", "tokio", @@ -4547,7 +4577,7 @@ version = "0.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c52a121f0fbf9320d5f2a9a5d82f6cb7557eda5e8b47fc3e7f359ec866ae960" dependencies = [ - "bitflags", + "bitflags 1.3.2", "io-lifetimes", "windows-sys 0.48.0", ] diff --git a/crates/runtime/src/traphandlers/backtrace.rs b/crates/runtime/src/traphandlers/backtrace.rs index b7a6ec0c60dc..a1abdeb8ada7 100644 --- a/crates/runtime/src/traphandlers/backtrace.rs +++ b/crates/runtime/src/traphandlers/backtrace.rs @@ -304,7 +304,9 @@ impl Backtrace { } /// Iterate over the frames inside this backtrace. - pub fn frames<'a>(&'a self) -> impl ExactSizeIterator + 'a { + pub fn frames<'a>( + &'a self, + ) -> impl ExactSizeIterator + DoubleEndedIterator + 'a { self.0.iter() } } diff --git a/crates/wasi-common/src/snapshots/preview_1.rs b/crates/wasi-common/src/snapshots/preview_1.rs index ef54c0e448cc..0b7fd584ca60 100644 --- a/crates/wasi-common/src/snapshots/preview_1.rs +++ b/crates/wasi-common/src/snapshots/preview_1.rs @@ -1355,7 +1355,7 @@ fn dirent_bytes(dirent: types::Dirent) -> Vec { use wiggle::GuestType; assert_eq!( types::Dirent::guest_size(), - std::mem::size_of::() as _, + std::mem::size_of::() as u32, "Dirent guest repr and host repr should match" ); assert_eq!( diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index d6a04d2f5c7a..0ebab2787c3a 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -35,6 +35,7 @@ cfg-if = "1.0" log = { workspace = true } wat = { workspace = true, optional = true } serde = { version = "1.0.94", features = ["derive"] } +serde_json = { workspace = true } bincode = "1.2.1" indexmap = "1.6" paste = "1.0.3" @@ -45,6 +46,7 @@ object = { workspace = true } async-trait = { workspace = true, optional = true } encoding_rs = { version = "0.8.31", optional = true } bumpalo = "3.11.0" +fxprof-processed-profile = "0.6.0" [target.'cfg(target_os = "windows")'.dependencies.windows-sys] workspace = true diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs index b1ba2004be3e..8c6879206697 100644 --- a/crates/wasmtime/src/lib.rs +++ b/crates/wasmtime/src/lib.rs @@ -399,6 +399,7 @@ mod limits; mod linker; mod memory; mod module; +mod profiling; mod r#ref; mod signatures; mod store; @@ -416,6 +417,7 @@ pub use crate::limits::*; pub use crate::linker::*; pub use crate::memory::*; pub use crate::module::Module; +pub use crate::profiling::GuestProfiler; pub use crate::r#ref::ExternRef; #[cfg(feature = "async")] pub use crate::store::CallHookHandler; diff --git a/crates/wasmtime/src/profiling.rs b/crates/wasmtime/src/profiling.rs new file mode 100644 index 000000000000..b0620789a4eb --- /dev/null +++ b/crates/wasmtime/src/profiling.rs @@ -0,0 +1,213 @@ +use anyhow::Result; +use fxprof_processed_profile::debugid::DebugId; +use fxprof_processed_profile::{ + CategoryHandle, CpuDelta, Frame, FrameFlags, FrameInfo, LibraryInfo, Profile, + ReferenceTimestamp, Symbol, SymbolTable, Timestamp, +}; +use std::ops::Range; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use wasmtime_jit::CompiledModule; +use wasmtime_runtime::Backtrace; + +use crate::Module; + +// TODO: collect more data +// - Provide additional hooks for recording host-guest transitions, to be +// invoked from a Store::call_hook +// - On non-Windows, measure thread-local CPU usage between events with +// rustix::time::clock_gettime(ClockId::ThreadCPUTime) +// - Report which wasm module, and maybe instance, each frame came from + +/// Collects basic profiling data for a single WebAssembly guest. +/// +/// This profiler can't provide measurements that are as accurate or detailed +/// as a platform-specific profiler, such as `perf` on Linux. On the other +/// hand, this profiler works on every platform that Wasmtime supports. Also, +/// as an embedder you can use this profiler selectively on individual guest +/// instances rather than profiling the entire process. +/// +/// To use this, you'll need to arrange to call [`GuestProfiler::sample`] at +/// regular intervals while the guest is on the stack. The most straightforward +/// way to do that is to call it from a callback registered with +/// [`Store::epoch_deadline_callback()`](crate::Store::epoch_deadline_callback). +/// +/// # Accuracy +/// +/// The data collection granularity is limited by the mechanism you use to +/// interrupt guest execution and collect a profiling sample. +/// +/// If you use epoch interruption, then samples will only be collected at +/// function entry points and loop headers. This introduces some bias to the +/// results. In addition, samples will only be taken at times when WebAssembly +/// functions are running, not during host-calls. +/// +/// It is technically possible to use fuel interruption instead. That +/// introduces worse bias since samples occur after a certain number of +/// WebAssembly instructions, which can take different amounts of time. +/// +/// You may instead be able to use platform-specific methods, such as +/// `setitimer(ITIMER_VIRTUAL, ...)` on POSIX-compliant systems, to sample on +/// a more accurate interval. The only current requirement is that the guest +/// you wish to profile must be on the same stack where you call `sample`, +/// and executing within the same thread. However, the `GuestProfiler::sample` +/// method is not currently async-signal-safe, so doing this correctly is not +/// easy. +/// +/// # Security +/// +/// Profiles produced using this profiler do not include any configuration +/// details from the host, such as virtual memory addresses, or from any +/// WebAssembly modules that you haven't specifically allowed. So for +/// example, these profiles should be safe to share with untrusted users +/// who have provided untrusted code that you are running in a multi-tenancy +/// environment. +/// +/// However, the profile does include byte offsets into the text section of +/// the compiled module, revealing some information about the size of the code +/// generated for each module. For user-provided modules, the user could get +/// the same information by compiling the module for themself using a similar +/// version of Wasmtime on the same target architecture, but for any module +/// where they don't already have the WebAssembly module binary available this +/// could theoretically lead to an undesirable information disclosure. So you +/// should only include user-provided modules in profiles. +#[derive(Debug)] +pub struct GuestProfiler { + profile: Profile, + modules: Vec<(Range, fxprof_processed_profile::LibraryHandle)>, + process: fxprof_processed_profile::ProcessHandle, + thread: fxprof_processed_profile::ThreadHandle, + start: Instant, +} + +impl GuestProfiler { + /// Begin profiling a new guest. When this function is called, the current + /// wall-clock time is recorded as the start time for the guest. + /// + /// The `module_name` parameter is recorded in the profile to help identify + /// where the profile came from. + /// + /// The `interval` parameter should match the rate at which you intend + /// to call `sample`. However, this is used as a hint and not required to + /// exactly match the real sample rate. + /// + /// Only modules which are present in the `modules` vector will appear in + /// stack traces in this profile. Any stack frames which were executing + /// host code or functions from other modules will be omitted. See the + /// "Security" section of the [`GuestProfiler`] documentation for guidance + /// on what modules should not be included in this list. + pub fn new(module_name: &str, interval: Duration, modules: Vec<(String, Module)>) -> Self { + let zero = ReferenceTimestamp::from_millis_since_unix_epoch(0.0); + let mut profile = Profile::new(module_name, zero, interval.into()); + + let mut modules: Vec<_> = modules + .into_iter() + .filter_map(|(name, module)| { + let compiled = module.compiled_module(); + let text = compiled.text().as_ptr_range(); + let address_range = text.start as usize..text.end as usize; + module_symbols(name, compiled).map(|lib| (address_range, profile.add_lib(lib))) + }) + .collect(); + + modules.sort_unstable_by_key(|(range, _)| range.start); + + profile.set_reference_timestamp(std::time::SystemTime::now().into()); + let process = profile.add_process(module_name, 0, Timestamp::from_nanos_since_reference(0)); + let thread = profile.add_thread(process, 0, Timestamp::from_nanos_since_reference(0), true); + let start = Instant::now(); + Self { + profile, + modules, + process, + thread, + start, + } + } + + /// Add a sample to the profile. This function collects a backtrace from + /// any stack frames for allowed modules on the current stack. It should + /// typically be called from a callback registered using + /// [`Store::epoch_deadline_callback()`](crate::Store::epoch_deadline_callback). + pub fn sample(&mut self) { + let now = Timestamp::from_nanos_since_reference( + self.start.elapsed().as_nanos().try_into().unwrap(), + ); + + let backtrace = Backtrace::new(); + let frames = backtrace + .frames() + // Samply needs to see the oldest frame first, but we list the newest + // first, so iterate in reverse. + .rev() + .filter_map(|frame| { + // Find the first module whose start address includes this PC. + let module_idx = self + .modules + .partition_point(|(range, _)| range.start > frame.pc()); + if let Some((range, lib)) = self.modules.get(module_idx) { + if range.contains(&frame.pc()) { + return Some(FrameInfo { + frame: Frame::RelativeAddressFromReturnAddress( + *lib, + u32::try_from(frame.pc() - range.start).unwrap(), + ), + category_pair: CategoryHandle::OTHER.into(), + flags: FrameFlags::empty(), + }); + } + } + None + }); + + self.profile + .add_sample(self.thread, now, frames, CpuDelta::ZERO, 1); + } + + /// When the guest finishes running, call this function to write the + /// profile to the given `output`. The output is a JSON-formatted object in + /// the [Firefox "processed profile format"][fmt]. Files in this format may + /// be visualized at . + /// + /// [fmt]: https://github.com/firefox-devtools/profiler/blob/main/docs-developer/processed-profile-format.md + pub fn finish(mut self, output: impl std::io::Write) -> Result<()> { + let now = Timestamp::from_nanos_since_reference( + self.start.elapsed().as_nanos().try_into().unwrap(), + ); + self.profile.set_thread_end_time(self.thread, now); + self.profile.set_process_end_time(self.process, now); + + serde_json::to_writer(output, &self.profile)?; + Ok(()) + } +} + +fn module_symbols(name: String, compiled: &CompiledModule) -> Option { + let symbols = Vec::from_iter(compiled.finished_functions().map(|(defined_idx, _)| { + let loc = compiled.func_loc(defined_idx); + let func_idx = compiled.module().func_index(defined_idx); + let name = match compiled.func_name(func_idx) { + None => format!("wasm_function_{}", defined_idx.as_u32()), + Some(name) => name.to_string(), + }; + Symbol { + address: loc.start, + size: Some(loc.length), + name, + } + })); + if symbols.is_empty() { + return None; + } + + Some(LibraryInfo { + name, + debug_name: String::new(), + path: String::new(), + debug_path: String::new(), + debug_id: DebugId::nil(), + code_id: None, + arch: None, + symbol_table: Some(Arc::new(SymbolTable::new(symbols))), + }) +} diff --git a/src/commands/run.rs b/src/commands/run.rs index 740dd566fbfb..c7a4ac4833ca 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -10,7 +10,8 @@ use std::path::{Component, Path, PathBuf}; use std::thread; use std::time::Duration; use wasmtime::{ - Engine, Func, Linker, Module, Store, StoreLimits, StoreLimitsBuilder, Val, ValType, + Engine, Func, GuestProfiler, Linker, Module, Store, StoreLimits, StoreLimitsBuilder, Val, + ValType, }; use wasmtime_cli_flags::{CommonOptions, WasiModules}; use wasmtime_wasi::maybe_exit_on_error; @@ -168,6 +169,23 @@ pub struct RunCommand { )] wasm_timeout: Option, + /// Enable in-process sampling profiling of the guest WebAssembly program, + /// and write the captured profile to the given path. The resulting file + /// can be viewed using https://profiler.firefox.com/. + #[clap(long, value_name = "PATH")] + profile_guest: Option, + + /// Sampling interval for in-process profiling with `--profile-guest`. When + /// used together with `--wasm-timeout`, the timeout will be rounded up to + /// the nearest multiple of this interval. + #[clap( + long, + default_value = "10ms", + value_name = "TIME", + parse(try_from_str = parse_dur), + )] + profile_guest_interval: Duration, + /// Enable coredump generation after a WebAssembly trap. #[clap(long = "coredump-on-trap", value_name = "PATH")] coredump_on_trap: Option, @@ -216,9 +234,11 @@ impl RunCommand { self.common.init_logging(); let mut config = self.common.config(None)?; - if self.wasm_timeout.is_some() { + + if self.wasm_timeout.is_some() || self.profile_guest.is_some() { config.epoch_interruption(true); } + let engine = Engine::new(&config)?; let preopen_sockets = self.compute_preopen_sockets()?; @@ -239,6 +259,7 @@ impl RunCommand { // Read the wasm module binary either as `*.wat` or a raw binary. let module = self.load_module(linker.engine(), &self.module)?; + let mut modules = vec![(String::new(), module.clone())]; let host = Host::default(); let mut store = Store::new(&engine, host); @@ -285,6 +306,7 @@ impl RunCommand { for (name, path) in self.preloads.iter() { // Read the wasm module binary either as `*.wat` or a raw binary let module = self.load_module(&engine, path)?; + modules.push((name.clone(), module.clone())); // Add the module's functions to the linker. linker.module(&mut store, name, &module).context(format!( @@ -296,7 +318,7 @@ impl RunCommand { // Load the main wasm module. match self - .load_main_module(&mut store, &mut linker, module) + .load_main_module(&mut store, &mut linker, module, modules, &argv[0]) .with_context(|| format!("failed to run main module `{}`", self.module.display())) { Ok(()) => (), @@ -370,12 +392,63 @@ impl RunCommand { result } - fn load_main_module( + fn setup_epoch_handler( &self, store: &mut Store, - linker: &mut Linker, - module: Module, - ) -> Result<()> { + module_name: &str, + modules: Vec<(String, Module)>, + ) -> Box)> { + if let Some(path) = &self.profile_guest { + let interval = self.profile_guest_interval; + store.data_mut().guest_profiler = + Some(Arc::new(GuestProfiler::new(module_name, interval, modules))); + + if let Some(timeout) = self.wasm_timeout { + let mut timeout = (timeout.as_secs_f64() / interval.as_secs_f64()).ceil() as u64; + assert!(timeout > 0); + store.epoch_deadline_callback(move |mut store| { + Arc::get_mut(store.data_mut().guest_profiler.as_mut().unwrap()) + .expect("profiling doesn't support threads yet") + .sample(); + timeout -= 1; + if timeout == 0 { + bail!("timeout exceeded"); + } + Ok(1) + }); + } else { + store.epoch_deadline_callback(move |mut store| { + Arc::get_mut(store.data_mut().guest_profiler.as_mut().unwrap()) + .expect("profiling doesn't support threads yet") + .sample(); + Ok(1) + }); + } + + store.set_epoch_deadline(1); + let engine = store.engine().clone(); + thread::spawn(move || loop { + thread::sleep(interval); + engine.increment_epoch(); + }); + + let path = path.clone(); + return Box::new(move |store| { + let profiler = Arc::try_unwrap(store.data_mut().guest_profiler.take().unwrap()) + .expect("profiling doesn't support threads yet"); + if let Err(e) = std::fs::File::create(&path) + .map_err(anyhow::Error::new) + .and_then(|output| profiler.finish(std::io::BufWriter::new(output))) + { + eprintln!("failed writing profile at {}: {e:#}", path.display()); + } else { + eprintln!(); + eprintln!("Profile written to: {}", path.display()); + eprintln!("View this profile at https://profiler.firefox.com/."); + } + }); + } + if let Some(timeout) = self.wasm_timeout { store.set_epoch_deadline(1); let engine = store.engine().clone(); @@ -385,6 +458,17 @@ impl RunCommand { }); } + Box::new(|_store| {}) + } + + fn load_main_module( + &self, + store: &mut Store, + linker: &mut Linker, + module: Module, + modules: Vec<(String, Module)>, + module_name: &str, + ) -> Result<()> { // The main module might be allowed to have unknown imports, which // should be defined as traps: if self.trap_unknown_imports { @@ -402,20 +486,25 @@ impl RunCommand { .context(format!("failed to instantiate {:?}", self.module))?; // If a function to invoke was given, invoke it. - if let Some(name) = self.invoke.as_ref() { - self.invoke_export(store, linker, name) + let func = if let Some(name) = &self.invoke { + self.find_export(store, linker, name)? } else { - let func = linker.get_default(&mut *store, "")?; - self.invoke_func(store, func, None) - } + linker.get_default(&mut *store, "")? + }; + + // Finish all lookups before starting any epoch timers. + let finish_epoch_handler = self.setup_epoch_handler(store, module_name, modules); + let result = self.invoke_func(store, func); + finish_epoch_handler(store); + result } - fn invoke_export( + fn find_export( &self, store: &mut Store, linker: &Linker, name: &str, - ) -> Result<()> { + ) -> Result { let func = match linker .get(&mut *store, "", name) .ok_or_else(|| anyhow!("no export named `{}` found", name))? @@ -424,10 +513,10 @@ impl RunCommand { Some(func) => func, None => bail!("export of `{}` wasn't a function", name), }; - self.invoke_func(store, func, Some(name)) + Ok(func) } - fn invoke_func(&self, store: &mut Store, func: Func, name: Option<&str>) -> Result<()> { + fn invoke_func(&self, store: &mut Store, func: Func) -> Result<()> { let ty = func.ty(&store); if ty.params().len() > 0 { eprintln!( @@ -441,7 +530,7 @@ impl RunCommand { let val = match args.next() { Some(s) => s, None => { - if let Some(name) = name { + if let Some(name) = &self.invoke { bail!("not enough arguments for `{}`", name) } else { bail!("not enough arguments for command default") @@ -464,7 +553,7 @@ impl RunCommand { // out, if there are any. let mut results = vec![Val::null(); ty.results().len()]; let invoke_res = func.call(store, &values, &mut results).with_context(|| { - if let Some(name) = name { + if let Some(name) = &self.invoke { format!("failed to invoke `{}`", name) } else { format!("failed to invoke command default") @@ -536,6 +625,7 @@ struct Host { #[cfg(feature = "wasi-http")] wasi_http: Option, limits: StoreLimits, + guest_profiler: Option>, } /// Populates the given `Linker` with WASI APIs. diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index acf48ae7d8b9..fa1a40987b87 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -179,6 +179,13 @@ criteria = "safe-to-deploy" delta = "0.1.3 -> 0.1.6" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.debugid]] +who = "Gabriele Svelto " +criteria = "safe-to-deploy" +version = "0.8.0" +notes = "This crates was written by Sentry and I've fully audited it as Firefox crash reporting machinery relies on it." +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.either]] who = "Nika Layzell " criteria = "safe-to-deploy"