openvmm: Add VFIO PCI device assignment

Cargo.lock

@@ -5382,6 +5382,7 @@ dependencies = [
  "uidevices_resources",
  "unicycle",
  "unix_socket",
+ "vfio_assigned_device_resources",
  "video_core",
  "virt_whp",
  "virtio",
@@ -5529,6 +5530,7 @@ dependencies = [
  "storvsp",
  "tpm_device",
  "uidevices",
+ "vfio_assigned_device",
  "vhost_user_frontend",
  "virtio",
  "virtio_blk",
@@ -5805,12 +5807,14 @@ dependencies = [
 name = "pci_core"
 version = "0.0.0"
 dependencies = [
+ "anyhow",
  "bitfield-struct 0.11.0",
  "chipset_device",
  "guestmem",
  "inspect",
  "mesh",
  "open_enum",
+ "pal_event",
  "parking_lot",
  "thiserror 2.0.16",
  "tracelimit",
@@ -8701,6 +8705,34 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7adc40b84dd8c9c8838758ea7611413a30a0f6c8c31aa865a6c13ac0ed089990"
 
+[[package]]
+name = "vfio_assigned_device"
+version = "0.0.0"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "chipset_device",
+ "inspect",
+ "pci_core",
+ "pci_resources",
+ "tracelimit",
+ "tracing",
+ "vfio-bindings",
+ "vfio_assigned_device_resources",
+ "vfio_sys",
+ "virt",
+ "vm_resource",
+ "vmcore",
+]
+
+[[package]]
+name = "vfio_assigned_device_resources"
+version = "0.0.0"
+dependencies = [
+ "mesh",
+ "vm_resource",
+]
+
 [[package]]
 name = "vfio_sys"
 version = "0.0.0"

Cargo.toml

@@ -284,6 +284,8 @@ storage_string = { path = "vm/devices/storage/storage_string" }
 tdisp = { path = "vm/devices/tdisp" }
 tdisp_proto = { path = "vm/devices/tdisp_proto" }
 vmswitch = { path = "vm/devices/net/vmswitch" }
+vfio_assigned_device = { path = "vm/devices/pci/vfio_assigned_device" }
+vfio_assigned_device_resources = { path = "vm/devices/pci/vfio_assigned_device_resources" }
 pci_bus = { path = "vm/devices/pci/pci_bus" }
 pci_core = { path = "vm/devices/pci/pci_core" }
 pci_resources = { path = "vm/devices/pci/pci_resources" }

Guide/src/SUMMARY.md

@@ -8,6 +8,7 @@
   - [Running OpenVMM](./user_guide/openvmm/run.md)
     - [Alpine Linux](./user_guide/openvmm/alpine.md)
   - [VM Configurations](./user_guide/openvmm/vm_configurations.md)
+  - [VFIO Device Assignment](./user_guide/openvmm/vfio.md)
   - [Troubleshooting](./user_guide/openvmm/troubleshooting.md)
   - [Snapshots](./user_guide/openvmm/snapshots.md)
   - [Next Steps](./user_guide/openvmm/next_steps.md)
@@ -118,7 +119,6 @@
       - [VGA]()
     - [PCIe]()
       - [Overview](./reference/emulated/pcie/overview.md)
-  - [Direct Assigned]()
 - [Device Backends]()
   - [Serial]()
   - [Graphics and Input]()

Guide/src/reference/openvmm/management/cli.md

@@ -128,3 +128,9 @@ For `--virtio-rng` and `--virtio-console`, use their separate PCIe port flags:
 --vhost-user /tmp/vhost-blk.sock,type=blk,pcie_port=rp0
 --vhost-user /tmp/virtiofsd.sock,type=fs,tag=myfs,pcie_port=rp0
 ```
+
+**VFIO device assignment** (Linux only): `--vfio`
+
+```sh
+--vfio rp0:0000:01:00.0
+```

Guide/src/user_guide/openvmm/vfio.md

@@ -0,0 +1,159 @@
+# VFIO Device Assignment
+
+This page explains how to assign a physical PCI device to an OpenVMM guest using Linux VFIO.
+
+VFIO device assignment lets a guest VM directly access a physical PCI device (such as an NVMe controller or GPU). The guest sees the real device in its PCI bus and can interact with its config space.
+
+```admonish warning
+VFIO device assignment is experimental. PCI config space, BAR MMIO passthrough, MSI-X interrupts (via irqfd), and DMA are functional. Devices such as NVMe controllers work end-to-end.
+```
+
+## Overview
+
+OpenVMM running on a Linux host can assign physical PCI devices to guest VMs using VFIO. The device is bound to the `vfio-pci` kernel driver, then OpenVMM opens it via VFIO and presents it to the guest as a PCIe endpoint.
+
+```text
+Linux Host
+└── OpenVMM
+    └── Guest VM
+        └── sees physical PCI device on its PCI bus
+```
+
+## Prerequisites
+
+- A Linux host with IOMMU support enabled (Intel VT-d or AMD-Vi)
+- A PCI device available for passthrough
+- The `vfio-pci` kernel module loaded
+- The `vfio_iommu_type1` kernel module loaded
+
+## Step 1: Identify the device
+
+Find the PCI device you want to assign:
+
+```bash
+lspci -D
+```
+
+Look for the device's PCI address in `domain:bus:device.function` format, for example `0000:01:00.0`.
+
+## Step 2: Enable unsafe interrupts
+
+Some IOMMU implementations do not support interrupt remapping. If VFIO fails to set up the IOMMU with an "interrupt remapping" error, allow it to proceed without:
+
+```bash
+echo 1 | sudo tee /sys/module/vfio_iommu_type1/parameters/allow_unsafe_interrupts
+```
+
+```admonish note
+This flag is required in environments where the IOMMU does not support interrupt remapping (e.g., some nested virtualization setups). The "unsafe" label means a device could theoretically forge interrupt messages. In practice, this is acceptable when the host platform already constrains device behavior.
+```
+
+To make this persistent across reboots, add a modprobe config:
+
+```bash
+echo "options vfio_iommu_type1 allow_unsafe_interrupts=1" | sudo tee /etc/modprobe.d/vfio.conf
+```
+
+## Step 3: Bind the device to vfio-pci
+
+If the device is currently bound to another driver (e.g., `nvme`), unbind it first:
+
+```bash
+echo "0000:01:00.0" | sudo tee /sys/bus/pci/devices/0000:01:00.0/driver/unbind
+```
+
+Then bind to `vfio-pci`:
+
+```bash
+echo "vfio-pci" | sudo tee /sys/bus/pci/devices/0000:01:00.0/driver_override
+echo "0000:01:00.0" | sudo tee /sys/bus/pci/drivers/vfio-pci/bind
+```
+
+Verify the binding:
+
+```bash
+ls -la /sys/bus/pci/devices/0000:01:00.0/driver
+# Should show: ... -> ../../../../bus/pci/drivers/vfio-pci
+```
+
+```admonish warning
+If the device is an NVMe controller backing a mounted filesystem, unbinding it will cause data loss. Make sure you are not using the device before unbinding.
+```
+
+## Step 4: Verify VFIO group
+
+Check that a VFIO group device was created:
+
+```bash
+ls /dev/vfio/
+```
+
+You should see a numbered group (e.g., `/dev/vfio/0`). An IOMMU group is required — NoIommu mode is not supported.
+
+## Step 5: Launch OpenVMM with the VFIO device
+
+Use the `--vfio` flag to assign the device to a PCIe root port. You also need to create a PCIe root complex and root port for the device to attach to:
+
+```bash
+sudo openvmm \
+  --pcie-root-complex rc0 \
+  --pcie-root-port rc0:rp0 \
+  --vfio rp0:0000:01:00.0 \
+  --kernel /path/to/vmlinux \
+  --initrd /path/to/initrd \
+  --cmdline "console=ttyS0" \
+  --com1 console \
+  --memory 256M \
+  --processors 2
+```
+
+The `--vfio` syntax is `<port_name>:<pci_bdf>`:
+
+- `rp0` — the name of the PCIe root port to attach the device to (must match a `--pcie-root-port` name)
+- `0000:01:00.0` — the PCI BDF of the VFIO device on the host
+
+```admonish tip
+You can assign multiple devices by adding more root ports and `--vfio` flags:
+
+    --pcie-root-port rc0:rp0 \
+    --pcie-root-port rc0:rp1 \
+    --vfio rp0:0000:01:00.0 \
+    --vfio rp1:334c:00:00.0
+```
+
+## Step 6: Verify in the guest
+
+If the guest boots with PCI support, the assigned device should be visible:
+
+```bash
+lspci
+```
+
+The device will appear with its real vendor and device ID from the physical hardware.
+
+## Troubleshooting
+
+### "No such file or directory" for `/dev/vfio/*`
+
+Make sure you completed Step 2 (allow unsafe interrupts) and that the device is bound to `vfio-pci`. An IOMMU must be available — NoIommu mode is not supported.
+
+### "No interrupt remapping" / ENODEV on IOMMU setup
+
+Run Step 2 to enable `allow_unsafe_interrupts`. This is needed when the platform's IOMMU does not support interrupt remapping.
+
+### "failed to open VFIO device" / permission denied
+
+Run OpenVMM with `sudo`, or add your user to the `vfio` group and set appropriate permissions on `/dev/vfio/` devices.
+
+### Device not visible in guest `lspci`
+
+- Verify the device is bound to `vfio-pci` (Step 3)
+- Verify the VFIO group exists in `/dev/vfio/` (Step 4)
+- Verify the `--vfio` port name matches a `--pcie-root-port` name
+- Check OpenVMM log output for errors during VFIO device initialization
+
+## Current Limitations
+
+- **No save/restore** — VMs with VFIO devices cannot be saved or migrated.
+- **No hot-plug** — VFIO devices must be specified at VM creation time and cannot be added or removed while the VM is running.
+- **Linux only** — the `--vfio` flag is only available when OpenVMM is built and run on Linux. On Windows, use `--device` with WHP for device assignment.

openvmm/openvmm_core/src/partition.rs

@@ -92,6 +92,9 @@ pub trait HvlitePartition: Inspect + Send + Sync + RequestYield {
     /// Gets the [`SignalMsi`] interface for a particular VTL.
     fn as_signal_msi(&self, minimum_vtl: Vtl) -> Option<Arc<dyn SignalMsi>>;
 
+    /// Gets the irqfd routing interface, if supported.
+    fn irqfd(&self) -> Option<Arc<dyn virt::irqfd::IrqFd>>;
+
     /// Returns whether virtual devices are supported.
     fn supports_virtual_devices(&self) -> bool;
 
@@ -213,6 +216,10 @@ where
         self.as_signal_msi(minimum_vtl)
     }
 
+    fn irqfd(&self) -> Option<Arc<dyn virt::irqfd::IrqFd>> {
+        Partition::irqfd(self)
+    }
+
     fn supports_virtual_devices(&self) -> bool {
         self.new_virtual_device().is_some()
     }

openvmm/openvmm_core/src/worker/dispatch.rs

@@ -1744,6 +1744,7 @@ impl InitializedVm {
                 partition.clone().into_doorbell_registration(Vtl::Vtl0),
                 Some(&mapper),
                 partition.as_signal_msi(Vtl::Vtl0),
+                partition.irqfd(),
             )
             .await?;
         }
@@ -2767,6 +2768,7 @@ impl LoadedVm {
                                                 guest_memory: &self.inner.gm,
                                                 doorbell_registration: self.inner.partition.clone().into_doorbell_registration(Vtl::Vtl0),
                                                 shared_mem_mapper: None,
+                                                irqfd: self.inner.partition.irqfd(),
                                             },
                                         )
                                         .await

openvmm/openvmm_entry/Cargo.toml

@@ -49,6 +49,7 @@ net_backend_resources.workspace = true
 netvsp_resources.workspace = true
 nvme_resources.workspace = true
 scsidisk_resources.workspace = true
+vfio_assigned_device_resources.workspace = true
 serial_core.workspace = true
 serial_16550_resources.workspace = true
 serial_socket.workspace = true

openvmm/openvmm_entry/src/cli_args.rs

@@ -811,6 +811,26 @@ Options:
 "#)]
     #[clap(long, conflicts_with("pcat"))]
     pub pcie_remote: Vec<PcieRemoteCli>,
+
+    /// Assign a host PCI device to the guest via VFIO (Linux only)
+    #[clap(long_help = r#"
+Assign a host PCI device to the guest via Linux VFIO.
+
+The device must be bound to vfio-pci on the host before starting the VM.
+
+Examples:
+    # Assign NVMe controller to root port rp0
+    --vfio rp0:0000:01:00.0
+
+Syntax: <port_name>:<pci_bdf>
+
+    port_name    Root port or downstream switch port name
+    pci_bdf      PCI domain:bus:device.function of the VFIO device on
+                 the host (use lspci -D to find it)
+"#)]
+    #[cfg(target_os = "linux")]
+    #[clap(long, conflicts_with("pcat"))]
+    pub vfio: Vec<VfioDeviceCli>,
 }
 
 #[derive(Clone, Debug, PartialEq)]
@@ -1943,6 +1963,45 @@ impl FromStr for PcieRemoteCli {
     }
 }
 
+/// CLI configuration for a VFIO-assigned PCI device.
+#[cfg(target_os = "linux")]
+#[derive(Clone, Debug)]
+pub struct VfioDeviceCli {
+    /// Name of the PCIe downstream port to attach to.
+    pub port_name: String,
+    /// PCI BDF address of the device on the host (e.g., "0000:01:00.0").
+    pub pci_id: String,
+}
+
+#[cfg(target_os = "linux")]
+impl FromStr for VfioDeviceCli {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let (port_name, pci_id) = s
+            .split_once(':')
+            .context("expected <port_name>:<pci_bdf> (e.g., rp0:0000:01:00.0)")?;
+
+        if port_name.is_empty() {
+            anyhow::bail!("port name cannot be empty");
+        }
+
+        if pci_id.is_empty() {
+            anyhow::bail!("PCI address cannot be empty");
+        }
+
+        // Reject path separators to prevent sysfs path traversal via Path::join.
+        if pci_id.contains('/') || pci_id.contains("..") {
+            anyhow::bail!("PCI address must not contain path separators");
+        }
+
+        Ok(VfioDeviceCli {
+            port_name: port_name.to_string(),
+            pci_id: pci_id.to_string(),
+        })
+    }
+}
+
 /// Read a environment variable that may / may-not have a target-specific
 /// prefix. e.g: `default_value_from_arch_env("FOO")` would first try and read
 /// from `FOO`, and if that's not found, it will try `X86_64_FOO`.