fuzzgen: Statistics framework (#4868)

* cranelift: Add non user trap codes function

* cranelift: Add Fuzzgen stats

* cranelift: Use `once_cell` and cleanup some stuff

* fuzzgen: Remove total_inputs metric

* fuzzgen: Filter empty trap codes

Author: afonso360

Cargo.lock

@@ -53,6 +53,25 @@ pub enum TrapCode {
     User(u16),
 }
 
+impl TrapCode {
+    /// Returns a slice of all traps except `TrapCode::User` traps
+    pub const fn non_user_traps() -> &'static [TrapCode] {
+        &[
+            TrapCode::StackOverflow,
+            TrapCode::HeapOutOfBounds,
+            TrapCode::HeapMisaligned,
+            TrapCode::TableOutOfBounds,
+            TrapCode::IndirectCallToNull,
+            TrapCode::BadSignature,
+            TrapCode::IntegerOverflow,
+            TrapCode::IntegerDivisionByZero,
+            TrapCode::BadConversionToInteger,
+            TrapCode::UnreachableCodeReached,
+            TrapCode::Interrupt,
+        ]
+    }
+}
+
 impl Display for TrapCode {
     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
         use self::TrapCode::*;
@@ -102,24 +121,9 @@ mod tests {
     use super::*;
     use alloc::string::ToString;
 
-    // Everything but user-defined codes.
-    const CODES: [TrapCode; 11] = [
-        TrapCode::StackOverflow,
-        TrapCode::HeapOutOfBounds,
-        TrapCode::HeapMisaligned,
-        TrapCode::TableOutOfBounds,
-        TrapCode::IndirectCallToNull,
-        TrapCode::BadSignature,
-        TrapCode::IntegerOverflow,
-        TrapCode::IntegerDivisionByZero,
-        TrapCode::BadConversionToInteger,
-        TrapCode::UnreachableCodeReached,
-        TrapCode::Interrupt,
-    ];
-
     #[test]
     fn display() {
-        for r in &CODES {
+        for r in TrapCode::non_user_traps() {
             let tc = *r;
             assert_eq!(tc.to_string().parse(), Ok(tc));
         }

cranelift/codegen/src/ir/trapcode.rs

@@ -1385,7 +1385,7 @@ impl<'a, V> ControlFlow<'a, V> {
     }
 }
 
-#[derive(Error, Debug, PartialEq)]
+#[derive(Error, Debug, PartialEq, Eq, Hash)]
 pub enum CraneliftTrap {
     #[error("user code: {0}")]
     User(TrapCode),

cranelift/interpreter/src/step.rs

@@ -9,6 +9,7 @@ cargo-fuzz = true
 
 [dependencies]
 anyhow = { workspace = true }
+once_cell = { workspace = true }
 cranelift-codegen = { workspace = true, features = ["incremental-cache"] }
 cranelift-reader = { workspace = true }
 cranelift-wasm = { workspace = true }

fuzz/Cargo.toml

@@ -1,9 +1,13 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
+use once_cell::sync::Lazy;
+use std::collections::HashMap;
+use std::sync::atomic::AtomicU64;
+use std::sync::atomic::Ordering;
 
 use cranelift_codegen::data_value::DataValue;
-use cranelift_codegen::ir::LibCall;
+use cranelift_codegen::ir::{LibCall, TrapCode};
 use cranelift_codegen::settings;
 use cranelift_codegen::settings::Configurable;
 use cranelift_filetests::function_runner::{TestFileCompiler, Trampoline};
@@ -19,6 +23,87 @@ use smallvec::smallvec;
 
 const INTERPRETER_FUEL: u64 = 4096;
 
+/// Gather statistics about the fuzzer executions
+struct Statistics {
+    /// Inputs that fuzzgen can build a function with
+    /// This is also how many compiles we executed
+    pub valid_inputs: AtomicU64,
+
+    /// Total amount of runs that we tried in the interpreter
+    /// One fuzzer input can have many runs
+    pub total_runs: AtomicU64,
+    /// How many runs were successful?
+    /// This is also how many runs were run in the backend
+    pub run_result_success: AtomicU64,
+    /// How many runs resulted in a timeout?
+    pub run_result_timeout: AtomicU64,
+    /// How many runs ended with a trap?
+    pub run_result_trap: HashMap<CraneliftTrap, AtomicU64>,
+}
+
+impl Statistics {
+    pub fn print(&self, valid_inputs: u64) {
+        // We get valid_inputs as a param since we already loaded it previously.
+        let total_runs = self.total_runs.load(Ordering::SeqCst);
+        let run_result_success = self.run_result_success.load(Ordering::SeqCst);
+        let run_result_timeout = self.run_result_timeout.load(Ordering::SeqCst);
+
+        println!("== FuzzGen Statistics  ====================");
+        println!("Valid Inputs: {}", valid_inputs);
+        println!("Total Runs: {}", total_runs);
+        println!(
+            "Successful Runs: {} ({:.1}% of Total Runs)",
+            run_result_success,
+            (run_result_success as f64 / total_runs as f64) * 100.0
+        );
+        println!(
+            "Timed out Runs: {} ({:.1}% of Total Runs)",
+            run_result_timeout,
+            (run_result_timeout as f64 / total_runs as f64) * 100.0
+        );
+        println!("Traps:");
+        // Load and filter out empty trap codes.
+        let mut traps = self
+            .run_result_trap
+            .iter()
+            .map(|(trap, count)| (trap, count.load(Ordering::SeqCst)))
+            .filter(|(_, count)| *count != 0)
+            .collect::<Vec<_>>();
+
+        // Sort traps by count in a descending order
+        traps.sort_by_key(|(_, count)| -(*count as i64));
+
+        for (trap, count) in traps.into_iter() {
+            println!(
+                "\t{}: {} ({:.1}% of Total Runs)",
+                trap,
+                count,
+                (count as f64 / total_runs as f64) * 100.0
+            );
+        }
+    }
+}
+
+impl Default for Statistics {
+    fn default() -> Self {
+        // Pre-Register all trap codes since we can't modify this hashmap atomically.
+        let mut run_result_trap = HashMap::new();
+        run_result_trap.insert(CraneliftTrap::Debug, AtomicU64::new(0));
+        run_result_trap.insert(CraneliftTrap::Resumable, AtomicU64::new(0));
+        for trapcode in TrapCode::non_user_traps() {
+            run_result_trap.insert(CraneliftTrap::User(*trapcode), AtomicU64::new(0));
+        }
+
+        Self {
+            valid_inputs: AtomicU64::new(0),
+            total_runs: AtomicU64::new(0),
+            run_result_success: AtomicU64::new(0),
+            run_result_timeout: AtomicU64::new(0),
+            run_result_trap,
+        }
+    }
+}
+
 #[derive(Debug)]
 enum RunResult {
     Success(Vec<DataValue>),
@@ -79,7 +164,15 @@ fn build_interpreter(testcase: &TestCase) -> Interpreter {
     interpreter
 }
 
+static STATISTICS: Lazy<Statistics> = Lazy::new(Statistics::default);
+
 fuzz_target!(|testcase: TestCase| {
+    // Periodically print statistics
+    let valid_inputs = STATISTICS.valid_inputs.fetch_add(1, Ordering::SeqCst);
+    if valid_inputs != 0 && valid_inputs % 10000 == 0 {
+        STATISTICS.print(valid_inputs);
+    }
+
     // Native fn
     let flags = {
         let mut builder = settings::builder();
@@ -101,13 +194,18 @@ fuzz_target!(|testcase: TestCase| {
     let trampoline = compiled.get_trampoline(&testcase.func).unwrap();
 
     for args in &testcase.inputs {
+        STATISTICS.total_runs.fetch_add(1, Ordering::SeqCst);
+
         // We rebuild the interpreter every run so that we don't accidentally carry over any state
         // between runs, such as fuel remaining.
         let mut interpreter = build_interpreter(&testcase);
         let int_res = run_in_interpreter(&mut interpreter, args);
         match int_res {
-            RunResult::Success(_) => {}
-            RunResult::Trap(_) => {
+            RunResult::Success(_) => {
+                STATISTICS.run_result_success.fetch_add(1, Ordering::SeqCst);
+            }
+            RunResult::Trap(trap) => {
+                STATISTICS.run_result_trap[&trap].fetch_add(1, Ordering::SeqCst);
                 // If this input traps, skip it and continue trying other inputs
                 // for this function. We've already compiled it anyway.
                 //
@@ -120,6 +218,7 @@ fuzz_target!(|testcase: TestCase| {
             RunResult::Timeout => {
                 // We probably generated an infinite loop, we should drop this entire input.
                 // We could `continue` like we do on traps, but timeouts are *really* expensive.
+                STATISTICS.run_result_timeout.fetch_add(1, Ordering::SeqCst);
                 return;
             }
             RunResult::Error(_) => panic!("interpreter failed: {:?}", int_res),