Add shared memories

crates/c-api/src/extern.rs

@@ -20,6 +20,7 @@ pub extern "C" fn wasm_extern_kind(e: &wasm_extern_t) -> wasm_externkind_t {
         Extern::Global(_) => crate::WASM_EXTERN_GLOBAL,
         Extern::Table(_) => crate::WASM_EXTERN_TABLE,
         Extern::Memory(_) => crate::WASM_EXTERN_MEMORY,
+        Extern::SharedMemory(_) => todo!(),
     }
 }
 
@@ -119,6 +120,7 @@ impl From<Extern> for wasmtime_extern_t {
                 kind: WASMTIME_EXTERN_MEMORY,
                 of: wasmtime_extern_union { memory },
             },
+            Extern::SharedMemory(_memory) => todo!(),
         }
     }
 }

crates/cranelift/src/compiler.rs

@@ -16,7 +16,7 @@ use cranelift_codegen::{MachSrcLoc, MachStackMap};
 use cranelift_entity::{EntityRef, PrimaryMap};
 use cranelift_frontend::FunctionBuilder;
 use cranelift_wasm::{
-    DefinedFuncIndex, DefinedMemoryIndex, FuncIndex, FuncTranslator, MemoryIndex, SignatureIndex,
+    DefinedFuncIndex, FuncIndex, FuncTranslator, MemoryIndex, OwnedMemoryIndex, SignatureIndex,
     WasmFuncType,
 };
 use object::write::{Object, StandardSegment, SymbolId};
@@ -711,8 +711,18 @@ impl Compiler {
         let memory_offset = if ofs.num_imported_memories > 0 {
             ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0)))
         } else if ofs.num_defined_memories > 0 {
+            // The addition of shared memory makes the following assumption,
+            // "owned memory index = 0", possibly false. If the first memory
+            // is a shared memory, the base pointer will not be stored in
+            // the `owned_memories` array. The following code should
+            // eventually be fixed to not only handle shared memories but
+            // also multiple memories.
+            assert_eq!(
+                ofs.num_defined_memories, ofs.num_owned_memories,
+                "the memory base pointer may be incorrect due to sharing memory"
+            );
             ModuleMemoryOffset::Defined(
-                ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)),
+                ofs.vmctx_vmmemory_definition_base(OwnedMemoryIndex::new(0)),
             )
         } else {
             ModuleMemoryOffset::None

crates/cranelift/src/func_environ.rs

@@ -1368,18 +1368,37 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
 
     fn make_heap(&mut self, func: &mut ir::Function, index: MemoryIndex) -> WasmResult<ir::Heap> {
         let pointer_type = self.pointer_type();
-
+        let is_shared = self.module.memory_plans[index].memory.shared;
         let (ptr, base_offset, current_length_offset) = {
             let vmctx = self.vmctx(func);
             if let Some(def_index) = self.module.defined_memory_index(index) {
-                let base_offset =
-                    i32::try_from(self.offsets.vmctx_vmmemory_definition_base(def_index)).unwrap();
-                let current_length_offset = i32::try_from(
-                    self.offsets
-                        .vmctx_vmmemory_definition_current_length(def_index),
-                )
-                .unwrap();
-                (vmctx, base_offset, current_length_offset)
+                if is_shared {
+                    // As with imported memory, the `VMMemoryDefinition` for a
+                    // shared memory is stored elsewhere. We store a `*mut
+                    // VMMemoryDefinition` to it and dereference that when
+                    // atomically growing it.
+                    let from_offset = self.offsets.vmctx_vmmemory_pointer(def_index);
+                    let memory = func.create_global_value(ir::GlobalValueData::Load {
+                        base: vmctx,
+                        offset: Offset32::new(i32::try_from(from_offset).unwrap()),
+                        global_type: pointer_type,
+                        readonly: true,
+                    });
+                    let base_offset = i32::from(self.offsets.vmmemory_definition_base());
+                    let current_length_offset =
+                        i32::from(self.offsets.vmmemory_definition_current_length());
+                    (memory, base_offset, current_length_offset)
+                } else {
+                    let owned_index = self.module.owned_memory_index(def_index);
+                    let owned_base_offset =
+                        self.offsets.vmctx_vmmemory_definition_base(owned_index);
+                    let owned_length_offset = self
+                        .offsets
+                        .vmctx_vmmemory_definition_current_length(owned_index);
+                    let current_base_offset = i32::try_from(owned_base_offset).unwrap();
+                    let current_length_offset = i32::try_from(owned_length_offset).unwrap();
+                    (vmctx, current_base_offset, current_length_offset)
+                }
             } else {
                 let from_offset = self.offsets.vmctx_vmmemory_import_from(index);
                 let memory = func.create_global_value(ir::GlobalValueData::Load {
@@ -1693,28 +1712,65 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
     ) -> WasmResult<ir::Value> {
         let pointer_type = self.pointer_type();
         let vmctx = self.vmctx(&mut pos.func);
+        let is_shared = self.module.memory_plans[index].memory.shared;
         let base = pos.ins().global_value(pointer_type, vmctx);
         let current_length_in_bytes = match self.module.defined_memory_index(index) {
             Some(def_index) => {
-                let offset = i32::try_from(
-                    self.offsets
-                        .vmctx_vmmemory_definition_current_length(def_index),
-                )
-                .unwrap();
-                pos.ins()
-                    .load(pointer_type, ir::MemFlags::trusted(), base, offset)
+                if is_shared {
+                    let offset =
+                        i32::try_from(self.offsets.vmctx_vmmemory_pointer(def_index)).unwrap();
+                    let vmmemory_ptr =
+                        pos.ins()
+                            .load(pointer_type, ir::MemFlags::trusted(), base, offset);
+                    let vmmemory_definition_offset =
+                        i64::from(self.offsets.vmmemory_definition_current_length());
+                    let vmmemory_definition_ptr =
+                        pos.ins().iadd_imm(vmmemory_ptr, vmmemory_definition_offset);
+                    // This atomic access of the
+                    // `VMMemoryDefinition::current_length` is direct; no bounds
+                    // check is needed. This is possible because shared memory
+                    // has a static size (the maximum is always known). Shared
+                    // memory is thus built with a static memory plan and no
+                    // bounds-checked version of this is implemented.
+                    pos.ins().atomic_load(
+                        pointer_type,
+                        ir::MemFlags::trusted(),
+                        vmmemory_definition_ptr,
+                    )
+                } else {
+                    let owned_index = self.module.owned_memory_index(def_index);
+                    let offset = i32::try_from(
+                        self.offsets
+                            .vmctx_vmmemory_definition_current_length(owned_index),
+                    )
+                    .unwrap();
+                    pos.ins()
+                        .load(pointer_type, ir::MemFlags::trusted(), base, offset)
+                }
             }
             None => {
                 let offset = i32::try_from(self.offsets.vmctx_vmmemory_import_from(index)).unwrap();
                 let vmmemory_ptr =
                     pos.ins()
                         .load(pointer_type, ir::MemFlags::trusted(), base, offset);
-                pos.ins().load(
-                    pointer_type,
-                    ir::MemFlags::trusted(),
-                    vmmemory_ptr,
-                    i32::from(self.offsets.vmmemory_definition_current_length()),
-                )
+                if is_shared {
+                    let vmmemory_definition_offset =
+                        i64::from(self.offsets.vmmemory_definition_current_length());
+                    let vmmemory_definition_ptr =
+                        pos.ins().iadd_imm(vmmemory_ptr, vmmemory_definition_offset);
+                    pos.ins().atomic_load(
+                        pointer_type,
+                        ir::MemFlags::trusted(),
+                        vmmemory_definition_ptr,
+                    )
+                } else {
+                    pos.ins().load(
+                        pointer_type,
+                        ir::MemFlags::trusted(),
+                        vmmemory_ptr,
+                        i32::from(self.offsets.vmmemory_definition_current_length()),
+                    )
+                }
             }
         };
         let current_length_in_pages = pos

crates/environ/src/module.rs

@@ -10,15 +10,15 @@ use std::mem;
 use std::ops::Range;
 use wasmtime_types::*;
 
-/// Implemenation styles for WebAssembly linear memory.
+/// Implementation styles for WebAssembly linear memory.
 #[derive(Debug, Clone, Hash, Serialize, Deserialize)]
 pub enum MemoryStyle {
     /// The actual memory can be resized and moved.
     Dynamic {
         /// Extra space to reserve when a memory must be moved due to growth.
         reserve: u64,
     },
-    /// Addresss space is allocated up front.
+    /// Address space is allocated up front.
     Static {
         /// The number of mapped and unmapped pages.
         bound: u64,
@@ -160,7 +160,7 @@ pub enum MemoryInitialization {
     ///   which might reside in a compiled module on disk, available immediately
     ///   in a linear memory's address space.
     ///
-    /// To facilitate the latter fo these techniques the `try_static_init`
+    /// To facilitate the latter of these techniques the `try_static_init`
     /// function below, which creates this variant, takes a host page size
     /// argument which can page-align everything to make mmap-ing possible.
     Static {
@@ -919,6 +919,28 @@ impl Module {
         }
     }
 
+    /// Convert a `DefinedMemoryIndex` into an `OwnedMemoryIndex`. Returns None
+    /// if the index is an imported memory.
+    #[inline]
+    pub fn owned_memory_index(&self, memory: DefinedMemoryIndex) -> OwnedMemoryIndex {
+        assert!(
+            memory.index() < self.memory_plans.len(),
+            "non-shared memory must have an owned index"
+        );
+
+        // Once we know that the memory index is not greater than the number of
+        // plans, we can iterate through the plans up to the memory index and
+        // count how many are not shared (i.e., owned).
+        let owned_memory_index = self
+            .memory_plans
+            .iter()
+            .skip(self.num_imported_memories)
+            .take(memory.index())
+            .filter(|(_, mp)| !mp.memory.shared)
+            .count();
+        OwnedMemoryIndex::new(owned_memory_index)
+    }
+
     /// Test whether the given memory index is for an imported memory.
     #[inline]
     pub fn is_imported_memory(&self, index: MemoryIndex) -> bool {

crates/environ/src/module_environ.rs

@@ -240,9 +240,6 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
                             EntityType::Function(sig_index)
                         }
                         TypeRef::Memory(ty) => {
-                            if ty.shared {
-                                return Err(WasmError::Unsupported("shared memories".to_owned()));
-                            }
                             self.result.module.num_imported_memories += 1;
                             EntityType::Memory(ty.into())
                         }
@@ -296,9 +293,6 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
 
                 for entry in memories {
                     let memory = entry?;
-                    if memory.shared {
-                        return Err(WasmError::Unsupported("shared memories".to_owned()));
-                    }
                     let plan = MemoryPlan::for_memory(memory.into(), &self.tunables);
                     self.result.module.memory_plans.push(plan);
                 }