Virtual file support

crates/test-programs/build.rs

@@ -11,11 +11,19 @@ fn main() {
 #[cfg(feature = "test_programs")]
 mod wasi_tests {
     use std::env;
-    use std::fs::{read_dir, DirEntry, File};
+    use std::fs::{read_dir, File};
     use std::io::{self, Write};
     use std::path::{Path, PathBuf};
     use std::process::{Command, Stdio};
 
+    #[derive(Clone, Copy, Debug)]
+    enum PreopenType {
+        /// Preopens should be satisfied with real OS files.
+        OS,
+        /// Preopens should be satisfied with virtual files.
+        Virtual,
+    }
+
     pub(super) fn build_and_generate_tests() {
         // Validate if any of test sources are present and if they changed
         // This should always work since there is no submodule to init anymore
@@ -103,34 +111,52 @@ mod wasi_tests {
                 .replace("-", "_")
         )?;
         writeln!(out, "    use super::{{runtime, utils, setup_log}};")?;
+        writeln!(out, "    use runtime::PreopenType;")?;
         for dir_entry in dir_entries {
-            write_testsuite_tests(out, dir_entry, testsuite)?;
+            let test_path = dir_entry.path();
+            let stemstr = test_path
+                .file_stem()
+                .expect("file_stem")
+                .to_str()
+                .expect("to_str");
+
+            if no_preopens(testsuite, stemstr) {
+                write_testsuite_tests(out, &test_path, testsuite, PreopenType::OS)?;
+            } else {
+                write_testsuite_tests(out, &test_path, testsuite, PreopenType::OS)?;
+                write_testsuite_tests(out, &test_path, testsuite, PreopenType::Virtual)?;
+            }
         }
         writeln!(out, "}}")?;
         Ok(())
     }
 
     fn write_testsuite_tests(
         out: &mut File,
-        dir_entry: DirEntry,
+        path: &Path,
         testsuite: &str,
+        preopen_type: PreopenType,
     ) -> io::Result<()> {
-        let path = dir_entry.path();
         let stemstr = path
             .file_stem()
             .expect("file_stem")
             .to_str()
             .expect("to_str");
 
         writeln!(out, "    #[test]")?;
-        if ignore(testsuite, stemstr) {
+        let test_fn_name = format!(
+            "{}{}",
+            &stemstr.replace("-", "_"),
+            if let PreopenType::Virtual = preopen_type {
+                "_virtualfs"
+            } else {
+                ""
+            }
+        );
+        if ignore(testsuite, &test_fn_name) {
             writeln!(out, "    #[ignore]")?;
         }
-        writeln!(
-            out,
-            "    fn r#{}() -> anyhow::Result<()> {{",
-            &stemstr.replace("-", "_")
-        )?;
+        writeln!(out, "    fn r#{}() -> anyhow::Result<()> {{", test_fn_name,)?;
         writeln!(out, "        setup_log();")?;
         writeln!(
             out,
@@ -145,16 +171,25 @@ mod wasi_tests {
         let workspace = if no_preopens(testsuite, stemstr) {
             "None"
         } else {
-            writeln!(
-                out,
-                "        let workspace = utils::prepare_workspace(&bin_name)?;"
-            )?;
-            "Some(workspace.path())"
+            match preopen_type {
+                PreopenType::OS => {
+                    writeln!(
+                        out,
+                        "        let workspace = utils::prepare_workspace(&bin_name)?;"
+                    )?;
+                    "Some(workspace.path())"
+                }
+                PreopenType::Virtual => "Some(std::path::Path::new(&bin_name))",
+            }
         };
         writeln!(
             out,
-            "        runtime::instantiate(&data, &bin_name, {})",
-            workspace
+            "        runtime::instantiate(&data, &bin_name, {}, {})",
+            workspace,
+            match preopen_type {
+                PreopenType::OS => "PreopenType::OS",
+                PreopenType::Virtual => "PreopenType::Virtual",
+            }
         )?;
         writeln!(out, "    }}")?;
         writeln!(out)?;
@@ -164,8 +199,30 @@ mod wasi_tests {
     cfg_if::cfg_if! {
         if #[cfg(not(windows))] {
             /// Ignore tests that aren't supported yet.
-            fn ignore(_testsuite: &str, _name: &str) -> bool {
-                false
+            fn ignore(testsuite: &str, name: &str) -> bool {
+                if testsuite == "wasi-tests" {
+                    match name {
+                        // TODO: virtfs files cannot be poll_oneoff'd yet
+                        "poll_oneoff_virtualfs" => true,
+                        // TODO: virtfs does not support filetimes yet.
+                        "path_filestat_virtualfs" |
+                        "fd_filestat_set_virtualfs" => true,
+                        // TODO: virtfs does not support symlinks yet.
+                        "nofollow_errors_virtualfs" |
+                        "path_link_virtualfs" |
+                        "readlink_virtualfs" |
+                        "readlink_no_buffer_virtualfs" |
+                        "dangling_symlink_virtualfs" |
+                        "symlink_loop_virtualfs" |
+                        "path_symlink_trailing_slashes_virtualfs" => true,
+                        // TODO: virtfs does not support rename yet.
+                        "path_rename_trailing_slashes_virtualfs" |
+                        "path_rename_virtualfs" => true,
+                        _ => false,
+                    }
+                } else {
+                    unreachable!()
+                }
             }
         } else {
             /// Ignore tests that aren't supported yet.
@@ -178,6 +235,22 @@ mod wasi_tests {
                         "truncation_rights" => true,
                         "path_link" => true,
                         "dangling_fd" => true,
+                        // TODO: virtfs files cannot be poll_oneoff'd yet
+                        "poll_oneoff_virtualfs" => true,
+                        // TODO: virtfs does not support filetimes yet.
+                        "path_filestat_virtualfs" |
+                        "fd_filestat_set_virtualfs" => true,
+                        // TODO: virtfs does not support symlinks yet.
+                        "nofollow_errors_virtualfs" |
+                        "path_link_virtualfs" |
+                        "readlink_virtualfs" |
+                        "readlink_no_buffer_virtualfs" |
+                        "dangling_symlink_virtualfs" |
+                        "symlink_loop_virtualfs" |
+                        "path_symlink_trailing_slashes_virtualfs" => true,
+                        // TODO: virtfs does not support rename yet.
+                        "path_rename_trailing_slashes_virtualfs" |
+                        "path_rename_virtualfs" => true,
                         _ => false,
                     }
                 } else {

crates/test-programs/tests/wasm_tests/runtime.rs

@@ -1,30 +1,44 @@
 use anyhow::{bail, Context};
 use std::fs::File;
 use std::path::Path;
+use wasi_common::VirtualDirEntry;
 use wasmtime::{Instance, Module, Store};
 
-pub fn instantiate(data: &[u8], bin_name: &str, workspace: Option<&Path>) -> anyhow::Result<()> {
-    let store = Store::default();
-
-    let get_preopens = |workspace: Option<&Path>| -> anyhow::Result<Vec<_>> {
-        if let Some(workspace) = workspace {
-            let preopen_dir = wasi_common::preopen_dir(workspace)
-                .context(format!("error while preopening {:?}", workspace))?;
+#[derive(Clone, Copy, Debug)]
+pub enum PreopenType {
+    /// Preopens should be satisfied with real OS files.
+    OS,
+    /// Preopens should be satisfied with virtual files.
+    Virtual,
+}
 
-            Ok(vec![(".".to_owned(), preopen_dir)])
-        } else {
-            Ok(vec![])
-        }
-    };
+pub fn instantiate(
+    data: &[u8],
+    bin_name: &str,
+    workspace: Option<&Path>,
+    preopen_type: PreopenType,
+) -> anyhow::Result<()> {
+    let store = Store::default();
 
     // Create our wasi context with pretty standard arguments/inheritance/etc.
-    // Additionally register andy preopened directories if we have them.
+    // Additionally register any preopened directories if we have them.
     let mut builder = wasi_common::WasiCtxBuilder::new();
 
     builder.arg(bin_name).arg(".").inherit_stdio();
 
-    for (dir, file) in get_preopens(workspace)? {
-        builder.preopened_dir(file, dir);
+    if let Some(workspace) = workspace {
+        match preopen_type {
+            PreopenType::OS => {
+                let preopen_dir = wasi_common::preopen_dir(workspace)
+                    .context(format!("error while preopening {:?}", workspace))?;
+                builder.preopened_dir(preopen_dir, ".");
+            }
+            PreopenType::Virtual => {
+                // we can ignore the workspace path for virtual preopens because virtual preopens
+                // don't exist in the filesystem anyway - no name conflict concerns.
+                builder.preopened_virt(VirtualDirEntry::empty_directory(), ".");
+            }
+        }
     }
 
     // The nonstandard thing we do with `WasiCtxBuilder` is to ensure that

crates/test-programs/wasi-tests/src/bin/fd_flags_set.rs

@@ -50,7 +50,7 @@ unsafe fn test_fd_fdstat_set_flags(dir_fd: wasi::Fd) {
         )
         .expect("reading file"),
         buffer.len(),
-        "shoudl read {} bytes",
+        "should read {} bytes",
         buffer.len()
     );
 
@@ -87,7 +87,7 @@ unsafe fn test_fd_fdstat_set_flags(dir_fd: wasi::Fd) {
         )
         .expect("reading file"),
         buffer.len(),
-        "shoudl read {} bytes",
+        "should read {} bytes",
         buffer.len()
     );
 
@@ -126,7 +126,7 @@ unsafe fn test_fd_fdstat_set_flags(dir_fd: wasi::Fd) {
         )
         .expect("reading file"),
         buffer.len(),
-        "shoudl read {} bytes",
+        "should read {} bytes",
         buffer.len()
     );
 

crates/test-programs/wasi-tests/src/bin/remove_directory_trailing_slashes.rs

@@ -11,14 +11,14 @@ unsafe fn test_remove_directory_trailing_slashes(dir_fd: wasi::Fd) {
 
     wasi::path_create_directory(dir_fd, "dir").expect("creating a directory");
 
-    // Test that removing it with a trailing flash succeeds.
+    // Test that removing it with a trailing slash succeeds.
     wasi::path_remove_directory(dir_fd, "dir/")
         .expect("remove_directory with a trailing slash on a directory should succeed");
 
     // Create a temporary file.
     create_file(dir_fd, "file");
 
-    // Test that removing it with no trailing flash fails.
+    // Test that removing it with no trailing slash fails.
     assert_eq!(
         wasi::path_remove_directory(dir_fd, "file")
             .expect_err("remove_directory without a trailing slash on a file should fail")
@@ -27,7 +27,7 @@ unsafe fn test_remove_directory_trailing_slashes(dir_fd: wasi::Fd) {
         "errno should be ERRNO_NOTDIR"
     );
 
-    // Test that removing it with a trailing flash fails.
+    // Test that removing it with a trailing slash fails.
     assert_eq!(
         wasi::path_remove_directory(dir_fd, "file/")
             .expect_err("remove_directory with a trailing slash on a file should fail")

crates/wasi-common/src/ctx.rs

@@ -1,4 +1,6 @@
-use crate::fdentry::FdEntry;
+use crate::fdentry::{Descriptor, FdEntry};
+use crate::sys::fdentry_impl::OsHandle;
+use crate::virtfs::{VirtualDir, VirtualDirEntry};
 use crate::{wasi, Error, Result};
 use std::borrow::Borrow;
 use std::collections::HashMap;
@@ -61,7 +63,7 @@ impl PendingCString {
 /// A builder allowing customizable construction of `WasiCtx` instances.
 pub struct WasiCtxBuilder {
     fds: Option<HashMap<wasi::__wasi_fd_t, PendingFdEntry>>,
-    preopens: Option<Vec<(PathBuf, File)>>,
+    preopens: Option<Vec<(PathBuf, Descriptor)>>,
     args: Option<Vec<PendingCString>>,
     env: Option<HashMap<PendingCString, PendingCString>>,
 }
@@ -222,10 +224,46 @@ impl WasiCtxBuilder {
 
     /// Add a preopened directory.
     pub fn preopened_dir<P: AsRef<Path>>(&mut self, dir: File, guest_path: P) -> &mut Self {
+        self.preopens.as_mut().unwrap().push((
+            guest_path.as_ref().to_owned(),
+            Descriptor::OsHandle(OsHandle::from(dir)),
+        ));
+        self
+    }
+
+    /// Add a preopened virtual directory.
+    pub fn preopened_virt<P: AsRef<Path>>(
+        &mut self,
+        dir: VirtualDirEntry,
+        guest_path: P,
+    ) -> &mut Self {
+        fn populate_directory(virtentry: HashMap<String, VirtualDirEntry>, dir: &mut VirtualDir) {
+            for (path, entry) in virtentry.into_iter() {
+                match entry {
+                    VirtualDirEntry::Directory(dir_entries) => {
+                        let mut subdir = VirtualDir::new(true);
+                        populate_directory(dir_entries, &mut subdir);
+                        dir.add_dir(subdir, path);
+                    }
+                    VirtualDirEntry::File(content) => {
+                        dir.add_file(content, path);
+                    }
+                }
+            }
+        }
+
+        let dir = if let VirtualDirEntry::Directory(entries) = dir {
+            let mut dir = VirtualDir::new(true);
+            populate_directory(entries, &mut dir);
+            Box::new(dir)
+        } else {
+            panic!("the root of a VirtualDirEntry tree must be a VirtualDirEntry::Directory");
+        };
+
         self.preopens
             .as_mut()
             .unwrap()
-            .push((guest_path.as_ref().to_owned(), dir));
+            .push((guest_path.as_ref().to_owned(), Descriptor::VirtualFile(dir)));
         self
     }
 
@@ -271,7 +309,7 @@ impl WasiCtxBuilder {
                     fds.insert(fd, f()?);
                 }
                 PendingFdEntry::File(f) => {
-                    fds.insert(fd, FdEntry::from(f)?);
+                    fds.insert(fd, FdEntry::from(Descriptor::OsHandle(OsHandle::from(f)))?);
                 }
             }
         }
@@ -284,8 +322,20 @@ impl WasiCtxBuilder {
             // unnecessarily if we have exactly the maximum number of file descriptors.
             preopen_fd = preopen_fd.checked_add(1).ok_or(Error::ENFILE)?;
 
-            if !dir.metadata()?.is_dir() {
-                return Err(Error::EBADF);
+            match &dir {
+                Descriptor::OsHandle(handle) => {
+                    if !handle.metadata()?.is_dir() {
+                        return Err(Error::EBADF);
+                    }
+                }
+                Descriptor::VirtualFile(virt) => {
+                    if virt.get_file_type() != wasi::__WASI_FILETYPE_DIRECTORY {
+                        return Err(Error::EBADF);
+                    }
+                }
+                Descriptor::Stdin | Descriptor::Stdout | Descriptor::Stderr => {
+                    panic!("implementation error, stdin/stdout/stderr shouldn't be in the list of preopens");
+                }
             }
 
             // We don't currently allow setting file descriptors other than 0-2, but this will avoid