Add an unsafe API to unload process trap handlers

crates/wasmtime/src/engine.rs

@@ -699,6 +699,60 @@ impl Engine {
         code.publish()?;
         Ok(Arc::new(code))
     }
+
+    /// Unload process-related trap/signal handlers and destroy this engine.
+    ///
+    /// This method is not safe and is not widely applicable. It is not required
+    /// to be called and is intended for use cases such as unloading a dynamic
+    /// library from a process. It is difficult to invoke this method correctly
+    /// and it requires careful coordination to do so.
+    ///
+    /// # Panics
+    ///
+    /// This method will panic if this `Engine` handle is not the last remaining
+    /// engine handle.
+    ///
+    /// # Aborts
+    ///
+    /// This method will abort the process on some platforms in some situations
+    /// where unloading the handler cannot be performed and an unrecoverable
+    /// state is reached. For example on Unix platforms with signal handling
+    /// the process will be aborted if the current signal handlers are not
+    /// Wasmtime's.
+    ///
+    /// # Unsafety
+    ///
+    /// This method is not generally safe to call and has a number of
+    /// preconditions that must be met to even possibly be safe. Even with these
+    /// known preconditions met there may be other unknown invariants to uphold
+    /// as well.
+    ///
+    /// * There must be no other instances of `Engine` elsewhere in the process.
+    ///   Note that this isn't just copies of this `Engine` but it's any other
+    ///   `Engine` at all. This unloads global state that is used by all
+    ///   `Engine`s so this instance must be the last.
+    ///
+    /// * On Unix platforms no other signal handlers could have been installed
+    ///   for signals that Wasmtime catches. In this situation Wasmtime won't
+    ///   know how to restore signal handlers that Wasmtime possibly overwrote
+    ///   when Wasmtime was initially loaded. If possible initialize other
+    ///   libraries first and then initialize Wasmtime last (e.g. defer creating
+    ///   an `Engine`).
+    ///
+    /// * All existing threads which have used this DLL or copy of Wasmtime may
+    ///   no longer use this copy of Wasmtime. Per-thread state is not iterated
+    ///   and destroyed. Only future threads may use future instances of this
+    ///   Wasmtime itself.
+    ///
+    /// If other crashes are seen from using this method please feel free to
+    /// file an issue to update the documentation here with more preconditions
+    /// that must be met.
+    pub unsafe fn unload_process_handlers(self) {
+        assert_eq!(Arc::weak_count(&self.inner), 0);
+        assert_eq!(Arc::strong_count(&self.inner), 1);
+
+        crate::runtime::vm::deinit_traps();
+    }
 }
 
 /// A weak reference to an [`Engine`].

crates/wasmtime/src/runtime/vm/sys/custom/traphandlers.rs

@@ -20,8 +20,15 @@ pub unsafe fn wasmtime_setjmp(
     capi::wasmtime_setjmp(jmp_buf, callback, payload, callee.cast())
 }
 
-pub unsafe fn platform_init(_macos_use_mach_ports: bool) {
-    capi::wasmtime_init_traps(handle_trap);
+pub struct TrapHandler;
+
+impl TrapHandler {
+    pub unsafe fn new(_macos_use_mach_ports: bool) -> TrapHandler {
+        capi::wasmtime_init_traps(handle_trap);
+        TrapHandler
+    }
+
+    pub fn validate_config(&self, _macos_use_mach_ports: bool) {}
 }
 
 extern "C" fn handle_trap(ip: usize, fp: usize, has_faulting_addr: bool, faulting_addr: usize) {

crates/wasmtime/src/runtime/vm/sys/miri/traphandlers.rs

@@ -30,7 +30,15 @@ pub fn wasmtime_longjmp(_jmp_buf: *const u8) -> ! {
 #[allow(missing_docs)]
 pub type SignalHandler<'a> = dyn Fn() + Send + Sync + 'a;
 
-pub unsafe fn platform_init(_macos_use_mach_ports: bool) {}
+pub struct TrapHandler;
+
+impl TrapHandler {
+    pub unsafe fn new(_macos_use_mach_ports: bool) -> TrapHandler {
+        TrapHandler
+    }
+
+    pub fn validate_config(&self, _macos_use_mach_ports: bool) {}
+}
 
 pub fn lazy_per_thread_init() {}
 

crates/wasmtime/src/runtime/vm/sys/unix/machports.rs

@@ -57,28 +57,49 @@ static mut WASMTIME_PORT: mach_port_name_t = MACH_PORT_NULL;
 
 static mut CHILD_OF_FORKED_PROCESS: bool = false;
 
-pub unsafe fn platform_init() {
-    // Mach ports do not currently work across forks, so configure Wasmtime to
-    // panic in `lazy_per_thread_init` if the child attempts to invoke
-    // WebAssembly.
-    unsafe extern "C" fn child() {
-        CHILD_OF_FORKED_PROCESS = true;
+pub struct TrapHandler {
+    thread: Option<thread::JoinHandle<()>>,
+}
+
+impl TrapHandler {
+    pub unsafe fn new() -> TrapHandler {
+        // Mach ports do not currently work across forks, so configure Wasmtime to
+        // panic in `lazy_per_thread_init` if the child attempts to invoke
+        // WebAssembly.
+        unsafe extern "C" fn child() {
+            CHILD_OF_FORKED_PROCESS = true;
+        }
+        let rc = libc::pthread_atfork(None, None, Some(child));
+        assert_eq!(rc, 0, "failed to configure `pthread_atfork` handler");
+
+        // Allocate our WASMTIME_PORT and make sure that it can be sent to so we
+        // can receive exceptions.
+        let me = mach_task_self();
+        let kret = mach_port_allocate(me, MACH_PORT_RIGHT_RECEIVE, addr_of_mut!(WASMTIME_PORT));
+        assert_eq!(kret, KERN_SUCCESS, "failed to allocate port");
+        let kret =
+            mach_port_insert_right(me, WASMTIME_PORT, WASMTIME_PORT, MACH_MSG_TYPE_MAKE_SEND);
+        assert_eq!(kret, KERN_SUCCESS, "failed to insert right");
+
+        // Spin up our handler thread which will solely exist to service exceptions
+        // generated by other threads. Note that this is a background thread that
+        // we're not very interested in so it's detached here.
+        let thread = thread::spawn(|| handler_thread());
+
+        TrapHandler {
+            thread: Some(thread),
+        }
+    }
+}
+
+impl Drop for TrapHandler {
+    fn drop(&mut self) {
+        unsafe {
+            let kret = mach_port_destroy(mach_task_self(), WASMTIME_PORT);
+            assert_eq!(kret, KERN_SUCCESS, "failed to destroy port");
+            self.thread.take().unwrap().join().unwrap();
+        }
     }
-    let rc = libc::pthread_atfork(None, None, Some(child));
-    assert_eq!(rc, 0, "failed to configure `pthread_atfork` handler");
-
-    // Allocate our WASMTIME_PORT and make sure that it can be sent to so we
-    // can receive exceptions.
-    let me = mach_task_self();
-    let kret = mach_port_allocate(me, MACH_PORT_RIGHT_RECEIVE, addr_of_mut!(WASMTIME_PORT));
-    assert_eq!(kret, KERN_SUCCESS, "failed to allocate port");
-    let kret = mach_port_insert_right(me, WASMTIME_PORT, WASMTIME_PORT, MACH_MSG_TYPE_MAKE_SEND);
-    assert_eq!(kret, KERN_SUCCESS, "failed to insert right");
-
-    // Spin up our handler thread which will solely exist to service exceptions
-    // generated by other threads. Note that this is a background thread that
-    // we're not very interested in so it's detached here.
-    thread::spawn(|| handler_thread());
 }
 
 // Note that this is copied from Gecko at
@@ -133,13 +154,17 @@ unsafe fn handler_thread() {
         let mut request: ExceptionRequest = mem::zeroed();
         let kret = mach_msg(
             &mut request.body.Head,
-            MACH_RCV_MSG,
+            MACH_RCV_MSG | MACH_RCV_INTERRUPT,
             0,
             mem::size_of_val(&request) as u32,
             WASMTIME_PORT,
             MACH_MSG_TIMEOUT_NONE,
             MACH_PORT_NULL,
         );
+        if kret == MACH_RCV_PORT_CHANGED {
+            // this port has been closed;
+            break;
+        }
         if kret != KERN_SUCCESS {
             eprintln!("mach_msg failed with {} ({0:x})", kret);
             libc::abort();

crates/wasmtime/src/runtime/vm/sys/unix/macos_traphandlers.rs

@@ -4,17 +4,28 @@ static mut USE_MACH_PORTS: bool = false;
 
 pub use super::signals::{wasmtime_longjmp, wasmtime_setjmp, SignalHandler};
 
-pub unsafe fn platform_init(macos_use_mach_ports: bool) {
-    USE_MACH_PORTS = macos_use_mach_ports;
-    if macos_use_mach_ports {
-        super::machports::platform_init();
-    } else {
-        super::signals::platform_init(false);
-    }
+pub enum TrapHandler {
+    Signals(super::signals::TrapHandler),
+    #[allow(dead_code)] // used for its drop
+    MachPorts(super::machports::TrapHandler),
 }
 
-pub fn using_mach_ports() -> bool {
-    unsafe { USE_MACH_PORTS }
+impl TrapHandler {
+    pub unsafe fn new(macos_use_mach_ports: bool) -> TrapHandler {
+        USE_MACH_PORTS = macos_use_mach_ports;
+        if macos_use_mach_ports {
+            TrapHandler::MachPorts(super::machports::TrapHandler::new())
+        } else {
+            TrapHandler::Signals(super::signals::TrapHandler::new(false))
+        }
+    }
+
+    pub fn validate_config(&self, macos_use_mach_ports: bool) {
+        match self {
+            TrapHandler::Signals(t) => t.validate_config(macos_use_mach_ports),
+            TrapHandler::MachPorts(_) => assert!(macos_use_mach_ports),
+        }
+    }
 }
 
 pub fn lazy_per_thread_init() {

crates/wasmtime/src/runtime/vm/sys/unix/signals.rs

@@ -31,58 +31,114 @@ static mut PREV_SIGBUS: MaybeUninit<libc::sigaction> = MaybeUninit::uninit();
 static mut PREV_SIGILL: MaybeUninit<libc::sigaction> = MaybeUninit::uninit();
 static mut PREV_SIGFPE: MaybeUninit<libc::sigaction> = MaybeUninit::uninit();
 
-pub unsafe fn platform_init(macos_use_mach_ports: bool) {
-    // Either mach ports shouldn't be in use or we shouldn't be on macOS,
-    // otherwise the `machports.rs` module should be used instead.
-    assert!(!macos_use_mach_ports || !cfg!(target_os = "macos"));
-
-    let register = |slot: *mut libc::sigaction, signal: i32| {
-        let mut handler: libc::sigaction = mem::zeroed();
-        // The flags here are relatively careful, and they are...
-        //
-        // SA_SIGINFO gives us access to information like the program
-        // counter from where the fault happened.
-        //
-        // SA_ONSTACK allows us to handle signals on an alternate stack,
-        // so that the handler can run in response to running out of
-        // stack space on the main stack. Rust installs an alternate
-        // stack with sigaltstack, so we rely on that.
-        //
-        // SA_NODEFER allows us to reenter the signal handler if we
-        // crash while handling the signal, and fall through to the
-        // Breakpad handler by testing handlingSegFault.
-        handler.sa_flags = libc::SA_SIGINFO | libc::SA_NODEFER | libc::SA_ONSTACK;
-        handler.sa_sigaction = trap_handler as usize;
-        libc::sigemptyset(&mut handler.sa_mask);
-        if libc::sigaction(signal, &handler, slot) != 0 {
-            panic!(
-                "unable to install signal handler: {}",
-                io::Error::last_os_error(),
-            );
-        }
-    };
+pub struct TrapHandler;
+
+impl TrapHandler {
+    /// Installs all trap handlers.
+    ///
+    /// # Unsafety
+    ///
+    /// This function is unsafe because it's not safe to call concurrently and
+    /// it's not safe to call if the trap handlers have already been initialized
+    /// for this process.
+    pub unsafe fn new(macos_use_mach_ports: bool) -> TrapHandler {
+        // Either mach ports shouldn't be in use or we shouldn't be on macOS,
+        // otherwise the `machports.rs` module should be used instead.
+        assert!(!macos_use_mach_ports || !cfg!(target_os = "macos"));
+
+        foreach_handler(|slot, signal| {
+            let mut handler: libc::sigaction = mem::zeroed();
+            // The flags here are relatively careful, and they are...
+            //
+            // SA_SIGINFO gives us access to information like the program
+            // counter from where the fault happened.
+            //
+            // SA_ONSTACK allows us to handle signals on an alternate stack,
+            // so that the handler can run in response to running out of
+            // stack space on the main stack. Rust installs an alternate
+            // stack with sigaltstack, so we rely on that.
+            //
+            // SA_NODEFER allows us to reenter the signal handler if we
+            // crash while handling the signal, and fall through to the
+            // Breakpad handler by testing handlingSegFault.
+            handler.sa_flags = libc::SA_SIGINFO | libc::SA_NODEFER | libc::SA_ONSTACK;
+            handler.sa_sigaction = trap_handler as usize;
+            libc::sigemptyset(&mut handler.sa_mask);
+            if libc::sigaction(signal, &handler, slot) != 0 {
+                panic!(
+                    "unable to install signal handler: {}",
+                    io::Error::last_os_error(),
+                );
+            }
+        });
+
+        TrapHandler
+    }
+
+    pub fn validate_config(&self, macos_use_mach_ports: bool) {
+        assert!(!macos_use_mach_ports || !cfg!(target_os = "macos"));
+    }
+}
 
+unsafe fn foreach_handler(mut f: impl FnMut(*mut libc::sigaction, i32)) {
     // Allow handling OOB with signals on all architectures
-    register(PREV_SIGSEGV.as_mut_ptr(), libc::SIGSEGV);
+    f(PREV_SIGSEGV.as_mut_ptr(), libc::SIGSEGV);
 
     // Handle `unreachable` instructions which execute `ud2` right now
-    register(PREV_SIGILL.as_mut_ptr(), libc::SIGILL);
+    f(PREV_SIGILL.as_mut_ptr(), libc::SIGILL);
 
     // x86 and s390x use SIGFPE to report division by zero
     if cfg!(target_arch = "x86_64") || cfg!(target_arch = "s390x") {
-        register(PREV_SIGFPE.as_mut_ptr(), libc::SIGFPE);
+        f(PREV_SIGFPE.as_mut_ptr(), libc::SIGFPE);
     }
 
     // Sometimes we need to handle SIGBUS too:
     // - On Darwin, guard page accesses are raised as SIGBUS.
     if cfg!(target_os = "macos") || cfg!(target_os = "freebsd") {
-        register(PREV_SIGBUS.as_mut_ptr(), libc::SIGBUS);
+        f(PREV_SIGBUS.as_mut_ptr(), libc::SIGBUS);
     }
 
     // TODO(#1980): x86-32, if we support it, will also need a SIGFPE handler.
     // TODO(#1173): ARM32, if we support it, will also need a SIGBUS handler.
 }
 
+impl Drop for TrapHandler {
+    fn drop(&mut self) {
+        unsafe {
+            foreach_handler(|slot, signal| {
+                let mut prev: libc::sigaction = mem::zeroed();
+
+                // Restore the previous handler that this signal had.
+                if libc::sigaction(signal, slot, &mut prev) != 0 {
+                    eprintln!(
+                        "unable to reinstall signal handler: {}",
+                        io::Error::last_os_error(),
+                    );
+                    libc::abort();
+                }
+
+                // If our trap handler wasn't currently listed for this process
+                // then that's a problem because we have just corrupted the
+                // signal handler state and don't know how to remove ourselves
+                // from the signal handling state. Inform the user of this and
+                // abort the process.
+                if prev.sa_sigaction != trap_handler as usize {
+                    eprintln!(
+                        "
+Wasmtime's signal handler was not the last signal handler to be installed
+in the process so it's not certain how to unload signal handlers. In this
+situation the Engine::unload_process_handlers API is not applicable and requires
+perhaps initializing libraries in a different order. The process will be aborted
+now.
+"
+                    );
+                    libc::abort();
+                }
+            });
+        }
+    }
+}
+
 unsafe extern "C" fn trap_handler(
     signum: libc::c_int,
     siginfo: *mut libc::siginfo_t,