diff --git a/crates/winch/src/builder.rs b/crates/winch/src/builder.rs index e4f70a6ec9d5..3e342a8b3c91 100644 --- a/crates/winch/src/builder.rs +++ b/crates/winch/src/builder.rs @@ -69,10 +69,6 @@ impl CompilerBuilder for Builder { bail!("Winch does not currently support epoch based interruption"); } - if tunables.consume_fuel { - bail!("Winch does not currently support fuel based interruption"); - } - if tunables.generate_native_debuginfo { bail!("Winch does not currently support generating native debug information"); } diff --git a/crates/winch/src/compiler.rs b/crates/winch/src/compiler.rs index f6438723dc6b..1725b94c28c7 100644 --- a/crates/winch/src/compiler.rs +++ b/crates/winch/src/compiler.rs @@ -109,6 +109,7 @@ impl wasmtime_environ::Compiler for Compiler { types, &mut context.builtins, &mut validator, + &self.tunables, ) .map_err(|e| CompileError::Codegen(format!("{e:?}"))); self.save_context(context, validator.into_allocations()); diff --git a/tests/all/fuel.rs b/tests/all/fuel.rs index 882235e3a654..1d209a30954a 100644 --- a/tests/all/fuel.rs +++ b/tests/all/fuel.rs @@ -24,14 +24,15 @@ impl<'a> Parse<'a> for FuelWast<'a> { } } -#[test] +#[wasmtime_test] #[cfg_attr(miri, ignore)] -fn run() -> Result<()> { +fn run(config: &mut Config) -> Result<()> { + config.consume_fuel(true); let test = std::fs::read_to_string("tests/all/fuel.wast")?; let buf = ParseBuffer::new(&test)?; let mut wast = parser::parse::>(&buf)?; for (span, fuel, module) in wast.assertions.iter_mut() { - let consumed = fuel_consumed(&module.encode()?); + let consumed = fuel_consumed(&config, &module.encode()?); if consumed == *fuel { continue; } @@ -47,9 +48,7 @@ fn run() -> Result<()> { Ok(()) } -fn fuel_consumed(wasm: &[u8]) -> u64 { - let mut config = Config::new(); - config.consume_fuel(true); +fn fuel_consumed(config: &Config, wasm: &[u8]) -> u64 { let engine = Engine::new(&config).unwrap(); let module = Module::new(&engine, wasm).unwrap(); let mut store = Store::new(&engine, ()); @@ -58,10 +57,12 @@ fn fuel_consumed(wasm: &[u8]) -> u64 { u64::MAX - store.get_fuel().unwrap() } -#[test] +#[wasmtime_test] #[cfg_attr(miri, ignore)] -fn iloop() { +fn iloop(config: &mut Config) -> Result<()> { + config.consume_fuel(true); iloop_aborts( + &config, r#" (module (start 0) @@ -70,6 +71,7 @@ fn iloop() { "#, ); iloop_aborts( + &config, r#" (module (start 0) @@ -78,6 +80,7 @@ fn iloop() { "#, ); iloop_aborts( + &config, r#" (module (start 0) @@ -86,6 +89,7 @@ fn iloop() { "#, ); iloop_aborts( + &config, r#" (module (start 0) @@ -110,9 +114,7 @@ fn iloop() { "#, ); - fn iloop_aborts(wat: &str) { - let mut config = Config::new(); - config.consume_fuel(true); + fn iloop_aborts(config: &Config, wat: &str) { let engine = Engine::new(&config).unwrap(); let module = Module::new(&engine, wat).unwrap(); let mut store = Store::new(&engine, ()); @@ -120,11 +122,12 @@ fn iloop() { let error = Instance::new(&mut store, &module, &[]).err().unwrap(); assert_eq!(error.downcast::().unwrap(), Trap::OutOfFuel); } + + Ok(()) } -#[test] -fn manual_fuel() { - let mut config = Config::new(); +#[wasmtime_test] +fn manual_fuel(config: &mut Config) { config.consume_fuel(true); let engine = Engine::new(&config).unwrap(); let mut store = Store::new(&engine, ()); @@ -134,11 +137,10 @@ fn manual_fuel() { assert_eq!(store.get_fuel().ok(), Some(1)); } -#[test] +#[wasmtime_test] #[cfg_attr(miri, ignore)] -fn host_function_consumes_all() { +fn host_function_consumes_all(config: &mut Config) { const FUEL: u64 = 10_000; - let mut config = Config::new(); config.consume_fuel(true); let engine = Engine::new(&config).unwrap(); let module = Module::new( @@ -167,9 +169,8 @@ fn host_function_consumes_all() { assert_eq!(trap.downcast::().unwrap(), Trap::OutOfFuel); } -#[test] -fn manual_edge_cases() { - let mut config = Config::new(); +#[wasmtime_test] +fn manual_edge_cases(config: &mut Config) { config.consume_fuel(true); let engine = Engine::new(&config).unwrap(); let mut store = Store::new(&engine, ()); @@ -177,10 +178,9 @@ fn manual_edge_cases() { assert_eq!(store.get_fuel().unwrap(), u64::MAX); } -#[test] +#[wasmtime_test] #[cfg_attr(miri, ignore)] -fn unconditionally_trapping_memory_accesses_save_fuel_before_trapping() { - let mut config = Config::new(); +fn unconditionally_trapping_memory_accesses_save_fuel_before_trapping(config: &mut Config) { config.consume_fuel(true); config.static_memory_maximum_size(0x1_0000); @@ -221,10 +221,11 @@ fn unconditionally_trapping_memory_accesses_save_fuel_before_trapping() { assert!(consumed_fuel > 0); } -#[test] +#[wasmtime_test] #[cfg_attr(miri, ignore)] -fn get_fuel_clamps_at_zero() -> Result<()> { - let engine = Engine::new(Config::new().consume_fuel(true))?; +fn get_fuel_clamps_at_zero(config: &mut Config) -> Result<()> { + config.consume_fuel(true); + let engine = Engine::new(config)?; let mut store = Store::new(&engine, ()); let module = Module::new( &engine, @@ -257,21 +258,3 @@ fn get_fuel_clamps_at_zero() -> Result<()> { Ok(()) } - -#[wasmtime_test(strategies(not(Cranelift)))] -#[cfg_attr(miri, ignore)] -fn ensure_compatibility_between_winch_and_fuel(config: &mut Config) -> Result<()> { - config.consume_fuel(true); - let result = Engine::new(&config); - match result { - Ok(_) => anyhow::bail!("Expected incompatibility between fuel and Winch"), - Err(e) => { - assert_eq!( - e.to_string(), - "Winch does not currently support fuel based interruption" - ); - } - } - - Ok(()) -} diff --git a/winch/codegen/src/codegen/env.rs b/winch/codegen/src/codegen/env.rs index c7afe0e7fdcb..e88669987039 100644 --- a/winch/codegen/src/codegen/env.rs +++ b/winch/codegen/src/codegen/env.rs @@ -62,7 +62,7 @@ pub enum HeapStyle { #[derive(Debug, Copy, Clone)] pub struct HeapData { /// The offset to the base of the heap. - /// Relative to the VMContext pointer if the WebAssembly memory is locally + /// Relative to the `VMContext` pointer if the WebAssembly memory is locally /// defined. Else this is relative to the location of the imported WebAssembly /// memory location. pub offset: u32, diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs index 362da834c187..bd074d8ef4d6 100644 --- a/winch/codegen/src/codegen/mod.rs +++ b/winch/codegen/src/codegen/mod.rs @@ -13,7 +13,7 @@ use wasmparser::{ BinaryReader, FuncValidator, MemArg, Operator, ValidatorResources, VisitOperator, }; use wasmtime_environ::{ - GlobalIndex, MemoryIndex, PtrSize, TableIndex, TypeIndex, WasmHeapType, WasmValType, + GlobalIndex, MemoryIndex, PtrSize, TableIndex, Tunables, TypeIndex, WasmHeapType, WasmValType, FUNCREF_MASK, }; @@ -77,6 +77,12 @@ where /// Flag indicating whether during translation an unsupported instruction /// was found. pub found_unsupported_instruction: Option<&'static str>, + + /// Compilation settings for code generation. + pub tunables: &'a Tunables, + + /// Local counter to track fuel consumption. + pub fuel_consumed: i64, } impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M> @@ -84,6 +90,7 @@ where M: MacroAssembler, { pub fn new( + tunables: &'a Tunables, masm: &'a mut M, context: CodeGenContext<'a>, env: FuncEnv<'a, 'translation, 'data, M::Ptr>, @@ -94,9 +101,12 @@ where context, masm, env, + tunables, source_location: Default::default(), control_frames: Default::default(), found_unsupported_instruction: None, + // Empty functions should consume at least 1 fuel unit. + fuel_consumed: 1, } } @@ -134,6 +144,7 @@ where self.masm.start_source_loc(Default::default()); // We need to use the vmctx parameter before pinning it for stack checking. self.masm.prologue(vmctx); + // Pin the `VMContext` pointer. self.masm.mov( writable!(vmctx!(M)), @@ -145,6 +156,10 @@ where self.masm.end_source_loc(); + if self.tunables.consume_fuel { + self.emit_fuel_check(); + } + // Once we have emitted the epilogue and reserved stack space for the locals, we push the // base control flow block. self.control_frames.push(ControlStackFrame::block( @@ -250,17 +265,11 @@ where $( fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { self.0.$visit($($($arg.clone()),*)?)?; - // Only visit operators if the compiler is in a reachable code state. If - // the compiler is in an unreachable code state, most of the operators are - // ignored except for If, Block, Loop, Else and End. These operators need - // to be observed in order to keep the control stack frames balanced and to - // determine if reachability should be restored. - let visit_when_unreachable = visit_op_when_unreachable(Operator::$op $({ $($arg: $arg.clone()),* })?); - if self.1.is_reachable() || visit_when_unreachable { - let location = SourceLoc::new(self.2 as u32); - self.1.start(location); + let op = Operator::$op $({ $($arg: $arg.clone()),* })?; + if self.1.visit(&op) { + self.1.before_visit_op(&op, self.2); let res = Ok(self.1.$visit($($($arg),*)?)); - self.1.end(); + self.1.after_visit_op(); res } else { Ok(U::Output::default()) @@ -270,7 +279,7 @@ where }; } - fn visit_op_when_unreachable(op: Operator) -> bool { + fn visit_op_when_unreachable(op: &Operator) -> bool { use Operator::*; match op { If { .. } | Block { .. } | Loop { .. } | Else | End => true, @@ -278,51 +287,52 @@ where } } - /// Trait to handle reachability state. - trait ReachableState { - /// Returns true if the current state of the program is reachable. - fn is_reachable(&self) -> bool; - } - - /// Trait to map source locations to machine code. - trait SourceLocator { - fn start(&mut self, loc: SourceLoc); - fn end(&mut self); + /// Trait to handle hooks that must happen before and after visiting an + /// operator. + trait VisitorHooks { + /// Hook prior to visiting an operator. + fn before_visit_op(&mut self, operator: &Operator, offset: usize); + /// Hook after visiting an operator. + fn after_visit_op(&mut self); + + /// Returns `true` if the operator will be visited. + /// + /// Operators will be visited if the following invariants are met: + /// * The compiler is in a reachable state. + /// * The compiler is in an unreachable state, but the current + /// operator is a control flow operator. These operators need to be + /// visited in order to keep the control stack frames balanced and + /// to determine if the reachability state must be restored. + fn visit(&self, op: &Operator) -> bool; } - impl<'a, 'translation, 'data, M: MacroAssembler> ReachableState + impl<'a, 'translation, 'data, M: MacroAssembler> VisitorHooks for CodeGen<'a, 'translation, 'data, M> { - fn is_reachable(&self) -> bool { - self.context.reachable + fn visit(&self, op: &Operator) -> bool { + self.context.reachable || visit_op_when_unreachable(op) } - } - impl<'a, 'translation, 'data, M: MacroAssembler> SourceLocator - for CodeGen<'a, 'translation, 'data, M> - { - fn start(&mut self, loc: SourceLoc) { - let rel = self.source_loc_from(loc); - self.source_location.current = self.masm.start_source_loc(rel); - } + fn before_visit_op(&mut self, operator: &Operator, offset: usize) { + // Handle source location mapping. + self.source_location_before_visit_op(offset); - fn end(&mut self) { - // Because in Winch binary emission is done in a single pass - // and because the MachBuffer performs optimizations during - // emission, we have to be careful when calling - // [MacroAssembler::end_source_location] to avoid breaking the - // invariant that checks that the end [CodeOffset] must be equal - // or greater than the start [CodeOffset]. - if self.masm.current_code_offset() >= self.source_location.current.0 { - self.masm.end_source_loc(); + // Handle fuel. + if self.tunables.consume_fuel { + self.fuel_before_visit_op(operator); } } + + fn after_visit_op(&mut self) { + // Handle source code location mapping. + self.source_location_after_visit_op(); + } } impl<'a, T, U> VisitOperator<'a> for ValidateThenVisit<'_, T, U> where T: VisitOperator<'a, Output = wasmparser::Result<()>>, - U: VisitOperator<'a> + ReachableState + SourceLocator, + U: VisitOperator<'a> + VisitorHooks, U::Output: Default, { type Output = Result; @@ -665,6 +675,7 @@ where // reachability is restored or when reaching the end of the // function. HeapStyle::Static { bound } if offset_with_access_size > bound => { + self.emit_fuel_increment(); self.masm.trap(TrapCode::HEAP_OUT_OF_BOUNDS); self.context.reachable = false; None @@ -907,6 +918,176 @@ where ); self.context.stack.push(dst.into()); } + + /// Emit a series of instructions that check the current fuel usage by + /// performing a zero-comparison with the number of units stored in + /// `VMRuntimeLimits`. + pub fn emit_fuel_check(&mut self) { + let fuel_var = self.emit_load_fuel_consumed(); + let continuation = self.masm.get_label(); + + // Fuel is stored as a negative i64, so if the number is less than zero, + // we're still under the fuel limits. + self.masm.branch( + IntCmpKind::LtS, + fuel_var, + RegImm::i64(0), + continuation, + OperandSize::S64, + ); + // Out-of-fuel branch. + let out_of_fuel = self.env.builtins.out_of_gas::(); + FnCall::emit::( + &mut self.env, + self.masm, + &mut self.context, + Callee::Builtin(out_of_fuel.clone()), + ); + // Under fuel limits branch. + self.masm.bind(continuation); + self.context.free_reg(fuel_var); + } + + /// Increments the fuel consumed in `VMRuntimeLimits` by flushing + /// `self.fuel_consumed` to memory. + fn emit_fuel_increment(&mut self) { + let fuel_at_point = std::mem::replace(&mut self.fuel_consumed, 0); + if fuel_at_point == 0 { + return; + } + + let limits_offset = self.env.vmoffsets.ptr.vmctx_runtime_limits(); + let fuel_offset = self.env.vmoffsets.ptr.vmruntime_limits_fuel_consumed(); + let limits_var = self.context.any_gpr(self.masm); + + // Load `VMRuntimeLimits` into the `limits_var` reg. + self.masm.load_ptr( + self.masm.address_at_vmctx(u32::from(limits_offset)), + writable!(limits_var), + ); + + // Load the fuel consumed at point into the scratch register. + self.masm.load( + self.masm.address_at_reg(limits_var, u32::from(fuel_offset)), + writable!(scratch!(M)), + OperandSize::S64, + ); + + // Add the fuel consumed at point with the value in the scratch + // register. + self.masm.add( + writable!(scratch!(M)), + scratch!(M), + RegImm::i64(fuel_at_point), + OperandSize::S64, + ); + + // Store the updated fuel consumed to `VMRuntimeLimits`. + self.masm.store( + scratch!(M).into(), + self.masm.address_at_reg(limits_var, u32::from(fuel_offset)), + OperandSize::S64, + ); + + self.context.free_reg(limits_var); + } + + /// Emits a series of instructions that load the `fuel_consumed` field from + /// `VMRuntimeLimits`. + fn emit_load_fuel_consumed(&mut self) -> Reg { + let limits_offset = self.env.vmoffsets.ptr.vmctx_runtime_limits(); + let fuel_offset = self.env.vmoffsets.ptr.vmruntime_limits_fuel_consumed(); + let fuel_var = self.context.any_gpr(self.masm); + self.masm.load_ptr( + self.masm.address_at_vmctx(u32::from(limits_offset)), + writable!(fuel_var), + ); + + self.masm.load( + self.masm.address_at_reg(fuel_var, u32::from(fuel_offset)), + writable!(fuel_var), + // Fuel is an i64. + OperandSize::S64, + ); + + fuel_var + } + + /// Hook to handle fuel before visiting an operator. + fn fuel_before_visit_op(&mut self, op: &Operator) { + if !self.context.reachable { + // `self.fuel_consumed` must be correctly flushed to memory when + // entering an unreachable state. + debug_assert_eq!(self.fuel_consumed, 0); + return; + } + + // Generally, most instructions require 1 fuel unit. + // + // However, there are exceptions, which are detailed in the code below. + // Note that the fuel accounting semantics align with those of + // Cranelift; for further information, refer to + // `crates/cranelift/src/func_environ.rs`. + // + // The primary distinction between the two implementations is that Winch + // does not utilize a local-based cache to track fuel consumption. + // Instead, each increase in fuel necessitates loading from and storing + // to memory. + // + // Memory traffic will undoubtedly impact runtime performance. One + // potential optimization is to designate a register as non-allocatable, + // when fuel consumption is enabled, effectively using it as a local + // fuel cache. + self.fuel_consumed += match op { + Operator::Nop | Operator::Drop => 0, + Operator::Block { .. } + | Operator::Loop { .. } + | Operator::Unreachable + | Operator::Return + | Operator::Else + | Operator::End => 0, + _ => 1, + }; + + match op { + Operator::Unreachable + | Operator::Loop { .. } + | Operator::If { .. } + | Operator::Else { .. } + | Operator::Br { .. } + | Operator::BrIf { .. } + | Operator::BrTable { .. } + | Operator::End + | Operator::Return + | Operator::CallIndirect { .. } + | Operator::Call { .. } + | Operator::ReturnCall { .. } + | Operator::ReturnCallIndirect { .. } => { + self.emit_fuel_increment(); + } + _ => {} + } + } + + // Hook to handle source location mapping before visiting an operator. + fn source_location_before_visit_op(&mut self, offset: usize) { + let loc = SourceLoc::new(offset as u32); + let rel = self.source_loc_from(loc); + self.source_location.current = self.masm.start_source_loc(rel); + } + + // Hook to handle source location mapping after visiting an operator. + fn source_location_after_visit_op(&mut self) { + // Because in Winch binary emission is done in a single pass + // and because the MachBuffer performs optimizations during + // emission, we have to be careful when calling + // [`MacroAssembler::end_source_location`] to avoid breaking the + // invariant that checks that the end [CodeOffset] must be equal + // or greater than the start [CodeOffset]. + if self.masm.current_code_offset() >= self.source_location.current.0 { + self.masm.end_source_loc(); + } + } } /// Returns the index of the [`ControlStackFrame`] for the given diff --git a/winch/codegen/src/isa/aarch64/mod.rs b/winch/codegen/src/isa/aarch64/mod.rs index ea6bc0ff5e63..6cd04f68feb6 100644 --- a/winch/codegen/src/isa/aarch64/mod.rs +++ b/winch/codegen/src/isa/aarch64/mod.rs @@ -19,7 +19,7 @@ use masm::MacroAssembler as Aarch64Masm; use target_lexicon::Triple; use wasmparser::{FuncValidator, FunctionBody, ValidatorResources}; use wasmtime_cranelift::CompiledFunction; -use wasmtime_environ::{ModuleTranslation, ModuleTypesBuilder, VMOffsets, WasmFuncType}; +use wasmtime_environ::{ModuleTranslation, ModuleTypesBuilder, Tunables, VMOffsets, WasmFuncType}; mod abi; mod address; @@ -90,6 +90,7 @@ impl TargetIsa for Aarch64 { types: &ModuleTypesBuilder, builtins: &mut BuiltinFunctions, validator: &mut FuncValidator, + tunables: &Tunables, ) -> Result { let pointer_bytes = self.pointer_bytes(); let vmoffsets = VMOffsets::new(pointer_bytes, &translation.module); @@ -122,7 +123,7 @@ impl TargetIsa for Aarch64 { ); let regalloc = RegAlloc::from(gpr, fpr); let codegen_context = CodeGenContext::new(regalloc, stack, frame, &vmoffsets); - let mut codegen = CodeGen::new(&mut masm, codegen_context, env, abi_sig); + let mut codegen = CodeGen::new(tunables, &mut masm, codegen_context, env, abi_sig); codegen.emit(&mut body, validator)?; let names = codegen.env.take_name_map(); diff --git a/winch/codegen/src/isa/mod.rs b/winch/codegen/src/isa/mod.rs index 96071eec6c57..a77183c12f77 100644 --- a/winch/codegen/src/isa/mod.rs +++ b/winch/codegen/src/isa/mod.rs @@ -12,7 +12,7 @@ use std::{ use target_lexicon::{Architecture, Triple}; use wasmparser::{FuncValidator, FunctionBody, ValidatorResources}; use wasmtime_cranelift::CompiledFunction; -use wasmtime_environ::{ModuleTranslation, ModuleTypesBuilder, WasmFuncType}; +use wasmtime_environ::{ModuleTranslation, ModuleTypesBuilder, Tunables, WasmFuncType}; #[cfg(feature = "x64")] pub(crate) mod x64; @@ -163,6 +163,7 @@ pub trait TargetIsa: Send + Sync { types: &ModuleTypesBuilder, builtins: &mut BuiltinFunctions, validator: &mut FuncValidator, + tunables: &Tunables, ) -> Result; /// Get the default calling convention of the underlying target triple. diff --git a/winch/codegen/src/isa/x64/mod.rs b/winch/codegen/src/isa/x64/mod.rs index 68f91c76dfb8..7bef612ca0d4 100644 --- a/winch/codegen/src/isa/x64/mod.rs +++ b/winch/codegen/src/isa/x64/mod.rs @@ -19,7 +19,7 @@ use cranelift_codegen::{MachTextSectionBuilder, TextSectionBuilder}; use target_lexicon::Triple; use wasmparser::{FuncValidator, FunctionBody, ValidatorResources}; use wasmtime_cranelift::CompiledFunction; -use wasmtime_environ::{ModuleTranslation, ModuleTypesBuilder, VMOffsets, WasmFuncType}; +use wasmtime_environ::{ModuleTranslation, ModuleTypesBuilder, Tunables, VMOffsets, WasmFuncType}; use self::regs::{ALL_FPR, ALL_GPR, MAX_FPR, MAX_GPR, NON_ALLOCATABLE_FPR, NON_ALLOCATABLE_GPR}; @@ -94,6 +94,7 @@ impl TargetIsa for X64 { types: &ModuleTypesBuilder, builtins: &mut BuiltinFunctions, validator: &mut FuncValidator, + tunables: &Tunables, ) -> Result { let pointer_bytes = self.pointer_bytes(); let vmoffsets = VMOffsets::new(pointer_bytes, &translation.module); @@ -133,7 +134,7 @@ impl TargetIsa for X64 { let regalloc = RegAlloc::from(gpr, fpr); let codegen_context = CodeGenContext::new(regalloc, stack, frame, &vmoffsets); - let mut codegen = CodeGen::new(&mut masm, codegen_context, env, abi_sig); + let mut codegen = CodeGen::new(tunables, &mut masm, codegen_context, env, abi_sig); codegen.emit(&mut body, validator)?; let base = codegen.source_location.base; diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs index fe7b868dc9c4..70d9659cf88d 100644 --- a/winch/codegen/src/visitor.rs +++ b/winch/codegen/src/visitor.rs @@ -1721,6 +1721,11 @@ where self.masm, &mut self.context, )); + + // Emit fuel check right after binding the loop header. + if self.tunables.consume_fuel { + self.emit_fuel_check(); + } } fn visit_br(&mut self, depth: u32) {