diff --git a/crates/debug/src/lib.rs b/crates/debug/src/lib.rs index 8b0a2b0bdb02..31915d2281cd 100644 --- a/crates/debug/src/lib.rs +++ b/crates/debug/src/lib.rs @@ -2,10 +2,6 @@ #![allow(clippy::cast_ptr_alignment)] -use anyhow::{bail, ensure, Error}; -use object::{RelocationEncoding, RelocationKind}; -use std::collections::HashMap; - pub use crate::read_debuginfo::{read_debuginfo, DebugInfoData, WasmFileInfo}; pub use crate::write_debuginfo::{emit_dwarf, DwarfSection, DwarfSectionRelocTarget}; @@ -13,188 +9,3 @@ mod gc; mod read_debuginfo; mod transform; mod write_debuginfo; - -pub fn create_gdbjit_image( - mut bytes: Vec, - code_region: (*const u8, usize), - defined_funcs_offset: usize, - funcs: &[*const u8], -) -> Result, Error> { - ensure_supported_elf_format(&mut bytes)?; - - // patch relocs - relocate_dwarf_sections(&mut bytes, defined_funcs_offset, funcs)?; - - // elf is still missing details... - convert_object_elf_to_loadable_file(&mut bytes, code_region); - - // let mut file = ::std::fs::File::create(::std::path::Path::new("test.o")).expect("file"); - // ::std::io::Write::write_all(&mut file, &bytes).expect("write"); - - Ok(bytes) -} - -fn relocate_dwarf_sections( - bytes: &mut [u8], - defined_funcs_offset: usize, - funcs: &[*const u8], -) -> Result<(), Error> { - use object::read::{File, Object, ObjectSection, RelocationTarget}; - - let obj = File::parse(bytes)?; - let mut func_symbols = HashMap::new(); - for (id, sym) in obj.symbols() { - match (sym.name(), sym.section_index()) { - (Some(name), Some(_section_index)) if name.starts_with("_wasm_function_") => { - let index = name["_wasm_function_".len()..].parse::()?; - let data = funcs[index - defined_funcs_offset]; - func_symbols.insert(id, data); - } - _ => (), - } - } - - for section in obj.sections() { - for (off, r) in section.relocations() { - if r.kind() != RelocationKind::Absolute - || r.encoding() != RelocationEncoding::Generic - || r.size() != 64 - { - continue; - } - - let data = match r.target() { - RelocationTarget::Symbol(ref index) => func_symbols.get(index), - _ => None, - }; - let data: *const u8 = match data { - Some(data) => *data, - None => { - continue; - } - }; - - let target = (data as u64).wrapping_add(r.addend() as u64); - - let entry_ptr = section.data_range(off, 8).unwrap().unwrap().as_ptr(); - unsafe { - std::ptr::write(entry_ptr as *mut u64, target); - } - } - } - Ok(()) -} - -fn ensure_supported_elf_format(bytes: &mut Vec) -> Result<(), Error> { - use object::elf::*; - use object::endian::LittleEndian; - use std::mem::size_of; - - let e = LittleEndian; - let header: &FileHeader64 = - unsafe { &*(bytes.as_mut_ptr() as *const FileHeader64<_>) }; - ensure!( - header.e_ident.class == ELFCLASS64 && header.e_ident.data == ELFDATA2LSB, - "bits and endianess in .ELF", - ); - match header.e_machine.get(e) { - EM_X86_64 => (), - machine => { - bail!("Unsupported ELF target machine: {:x}", machine); - } - } - ensure!( - header.e_phoff.get(e) == 0 && header.e_phnum.get(e) == 0, - "program header table is empty" - ); - let e_shentsize = header.e_shentsize.get(e); - ensure!( - e_shentsize as usize == size_of::>(), - "size of sh" - ); - Ok(()) -} - -fn convert_object_elf_to_loadable_file(bytes: &mut Vec, code_region: (*const u8, usize)) { - use object::elf::*; - use object::endian::LittleEndian; - use std::ffi::CStr; - use std::mem::size_of; - use std::os::raw::c_char; - - let e = LittleEndian; - let header: &FileHeader64 = - unsafe { &*(bytes.as_mut_ptr() as *const FileHeader64<_>) }; - - let e_shentsize = header.e_shentsize.get(e); - let e_shoff = header.e_shoff.get(e); - let e_shnum = header.e_shnum.get(e); - let mut shstrtab_off = 0; - for i in 0..e_shnum { - let off = e_shoff as isize + i as isize * e_shentsize as isize; - let section: &SectionHeader64 = - unsafe { &*(bytes.as_ptr().offset(off) as *const SectionHeader64<_>) }; - if section.sh_type.get(e) != SHT_STRTAB { - continue; - } - shstrtab_off = section.sh_offset.get(e); - } - let mut segment: Option<_> = None; - for i in 0..e_shnum { - let off = e_shoff as isize + i as isize * e_shentsize as isize; - let section: &mut SectionHeader64 = - unsafe { &mut *(bytes.as_mut_ptr().offset(off) as *mut SectionHeader64<_>) }; - if section.sh_type.get(e) != SHT_PROGBITS { - continue; - } - // It is a SHT_PROGBITS, but we need to check sh_name to ensure it is our function - let sh_name_off = section.sh_name.get(e); - let sh_name = unsafe { - CStr::from_ptr( - bytes - .as_ptr() - .offset((shstrtab_off + sh_name_off as u64) as isize) - as *const c_char, - ) - .to_str() - .expect("name") - }; - if sh_name != ".text" { - continue; - } - - assert!(segment.is_none()); - // Patch vaddr, and save file location and its size. - section.sh_addr.set(e, code_region.0 as u64); - let sh_offset = section.sh_offset.get(e); - let sh_size = section.sh_size.get(e); - segment = Some((sh_offset, sh_size)); - } - - // LLDB wants segment with virtual address set, placing them at the end of ELF. - let ph_off = bytes.len(); - let e_phentsize = size_of::>(); - let e_phnum = 1; - bytes.resize(ph_off + e_phentsize * e_phnum, 0); - if let Some((sh_offset, sh_size)) = segment { - let (v_offset, size) = code_region; - let program: &mut ProgramHeader64 = - unsafe { &mut *(bytes.as_ptr().add(ph_off) as *mut ProgramHeader64<_>) }; - program.p_type.set(e, PT_LOAD); - program.p_offset.set(e, sh_offset); - program.p_vaddr.set(e, v_offset as u64); - program.p_paddr.set(e, v_offset as u64); - program.p_filesz.set(e, sh_size as u64); - program.p_memsz.set(e, size as u64); - } else { - unreachable!(); - } - - // It is somewhat loadable ELF file at this moment. - let header: &mut FileHeader64 = - unsafe { &mut *(bytes.as_mut_ptr() as *mut FileHeader64<_>) }; - header.e_type.set(e, ET_DYN); - header.e_phoff.set(e, ph_off as u64); - header.e_phentsize.set(e, e_phentsize as u16); - header.e_phnum.set(e, e_phnum as u16); -} diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs index 854564e5c7fd..b642a79e5cca 100644 --- a/crates/jit/src/code_memory.rs +++ b/crates/jit/src/code_memory.rs @@ -9,7 +9,7 @@ use object::read::{File as ObjectFile, Object, ObjectSection}; use region; use std::collections::BTreeMap; use std::mem::ManuallyDrop; -use std::{cmp, mem}; +use std::{cmp, mem, ops::Range}; use wasmtime_environ::{ isa::{unwind::UnwindInfo, TargetIsa}, wasm::{FuncIndex, SignatureIndex}, @@ -53,13 +53,17 @@ impl Drop for CodeMemoryEntry { pub(crate) struct CodeMemoryObjectAllocation<'a> { buf: &'a mut [u8], + code_range: Range, funcs: BTreeMap, trampolines: BTreeMap, } impl<'a> CodeMemoryObjectAllocation<'a> { - pub fn code_range(self) -> &'a mut [u8] { - self.buf + pub fn as_mut_slice(&mut self) -> &mut [u8] { + &mut self.buf + } + pub fn code_range(&self) -> &[u8] { + &self.buf[self.code_range.clone()] } pub fn funcs(&'a self) -> impl Iterator + 'a { let buf = self.buf as *const _ as *mut [u8]; @@ -292,27 +296,30 @@ impl CodeMemory { /// Returns references to functions and trampolines defined there. pub(crate) fn allocate_for_object<'a>( &'a mut self, - obj: &ObjectFile, + obj_bytes: &[u8], unwind_info: &[ObjectUnwindInfo], ) -> Result, String> { - let text_section = obj.section_by_name(".text").unwrap(); + // Allocate chunk memory that spans entire object file. + let (buf, registry, start) = self.allocate(obj_bytes.len())?; + buf.copy_from_slice(obj_bytes); + + let obj = ObjectFile::parse(obj_bytes).map_err(|_| "Unable to read obj".to_string())?; + let (text_start, text_size) = obj + .section_by_name(".text") + .and_then(|section| section.file_range()) + .unwrap_or_else(|| (0, 0)); - if text_section.size() == 0 { + if text_size == 0 { // No code in the image. return Ok(CodeMemoryObjectAllocation { - buf: &mut [], + buf: &mut buf[..obj_bytes.len()], + code_range: 0..0, funcs: BTreeMap::new(), trampolines: BTreeMap::new(), }); } - // Allocate chunk memory that spans entire code section. - let (buf, registry, start) = self.allocate(text_section.size() as usize)?; - buf.copy_from_slice( - text_section - .data() - .map_err(|_| "cannot read section data".to_string())?, - ); + let start = start + text_start as usize; // Track locations of all defined functions and trampolines. let mut funcs = BTreeMap::new(); @@ -357,7 +364,8 @@ impl CodeMemory { } Ok(CodeMemoryObjectAllocation { - buf: &mut buf[..text_section.size() as usize], + buf: &mut buf[..obj_bytes.len()], + code_range: start..start + text_size as usize, funcs, trampolines, }) diff --git a/crates/jit/src/compiler.rs b/crates/jit/src/compiler.rs index 2938e5766c9c..bb01aaeb3eb7 100644 --- a/crates/jit/src/compiler.rs +++ b/crates/jit/src/compiler.rs @@ -3,7 +3,6 @@ use crate::instantiate::SetupError; use crate::object::{build_object, ObjectUnwindInfo}; use cranelift_codegen::ir; -use object::write::Object; use std::hash::{Hash, Hasher}; use wasmtime_debug::{emit_dwarf, DebugInfoData, DwarfSection}; use wasmtime_environ::entity::{EntityRef, PrimaryMap}; @@ -97,7 +96,7 @@ fn transform_dwarf_data( #[allow(missing_docs)] pub struct Compilation { - pub obj: Object, + pub obj: Vec, pub unwind_info: Vec, pub traps: Traps, pub stack_maps: StackMaps, diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index 0e50a927d6f8..502f151a6b21 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -6,16 +6,16 @@ use crate::code_memory::CodeMemory; use crate::compiler::{Compilation, Compiler}; use crate::imports::resolve_imports; -use crate::link::link_module; +use crate::link::{link_module, unlink_module}; use crate::object::ObjectUnwindInfo; use crate::resolver::Resolver; -use object::File as ObjectFile; use serde::{Deserialize, Serialize}; use std::any::Any; use std::collections::HashMap; +use std::mem::ManuallyDrop; use std::sync::Arc; use thiserror::Error; -use wasmtime_debug::{create_gdbjit_image, read_debuginfo}; +use wasmtime_debug::read_debuginfo; use wasmtime_environ::entity::{BoxedSlice, PrimaryMap}; use wasmtime_environ::isa::TargetIsa; use wasmtime_environ::wasm::{DefinedFuncIndex, SignatureIndex}; @@ -23,6 +23,7 @@ use wasmtime_environ::{ CompileError, DataInitializer, DataInitializerLocation, Module, ModuleAddressMap, ModuleEnvironment, ModuleTranslation, StackMaps, Traps, }; +use wasmtime_obj::{ensure_supported_elf_format, patch_loadable_file, sanitize_loadable_file}; use wasmtime_profiling::ProfilingAgent; use wasmtime_runtime::VMInterrupts; use wasmtime_runtime::{ @@ -59,7 +60,7 @@ pub struct CompilationArtifacts { module: Module, /// ELF image with functions code. - obj: Box<[u8]>, + obj: Vec, /// Unwind information for function code. unwind_info: Box<[ObjectUnwindInfo]>, @@ -117,15 +118,9 @@ impl CompilationArtifacts { .collect::>() .into_boxed_slice(); - let obj = obj.write().map_err(|_| { - SetupError::Instantiate(InstantiationError::Resource( - "failed to create image memory".to_string(), - )) - })?; - Ok(Self { module, - obj: obj.into_boxed_slice(), + obj, unwind_info: unwind_info.into_boxed_slice(), data_initializers, traps, @@ -143,9 +138,22 @@ unsafe impl Sync for FinishedFunctions {} /// Container for data needed for an Instance function to exist. pub struct ModuleCode { - code_memory: CodeMemory, - #[allow(dead_code)] - dbg_jit_registration: Option, + code_memory: ManuallyDrop, + dbg_jit_registration: ManuallyDrop>, + // Note that this is stored as a `usize` instead of a `*const` or `*mut` + // pointer to allow this structure to be natively `Send` and `Sync` without + // `unsafe impl`. This type is sendable across threads and shareable since + // the coordination all happens at the OS layer. + image_range: std::ops::Range, +} + +impl Drop for ModuleCode { + fn drop(&mut self) { + unsafe { + ManuallyDrop::drop(&mut self.dbg_jit_registration); + ManuallyDrop::drop(&mut self.code_memory); + } + } } /// A compiled wasm module, ready to be instantiated. @@ -158,7 +166,6 @@ pub struct CompiledModule { traps: Traps, stack_maps: StackMaps, address_transform: ModuleAddressMap, - obj: Box<[u8]>, unwind_info: Box<[ObjectUnwindInfo]>, } @@ -192,8 +199,8 @@ impl CompiledModule { // Allocate all of the compiled functions into executable memory, // copying over their contents. - let (code_memory, code_range, finished_functions, trampolines) = - build_code_memory(isa, &obj, &module, &unwind_info).map_err(|message| { + let (code_memory, image_range, finished_functions, trampolines) = + build_code_memory(isa, obj, &module, &unwind_info).map_err(|message| { SetupError::Instantiate(InstantiationError::Resource(format!( "failed to build code memory for functions: {}", message @@ -202,11 +209,12 @@ impl CompiledModule { // Register GDB JIT images; initialize profiler and load the wasm module. let dbg_jit_registration = if debug_info { - let bytes = create_dbg_image(obj.to_vec(), code_range, &module, &finished_functions)?; + let bytes = unsafe { std::slice::from_raw_parts(image_range.0, image_range.1) }; + ensure_supported_elf_format(bytes).map_err(SetupError::DebugInfo)?; - profiler.module_load(&module, &finished_functions, Some(&bytes)); + profiler.module_load(&module, &finished_functions, Some(bytes)); - let reg = GdbJitImageRegistration::register(bytes); + let reg = GdbJitImageRegistration::register(image_range.0, image_range.1); Some(reg) } else { profiler.module_load(&module, &finished_functions, None); @@ -218,8 +226,9 @@ impl CompiledModule { Ok(Self { module: Arc::new(module), code: Arc::new(ModuleCode { - code_memory, - dbg_jit_registration, + code_memory: ManuallyDrop::new(code_memory), + dbg_jit_registration: ManuallyDrop::new(dbg_jit_registration), + image_range: image_range.0 as usize..image_range.0 as usize + image_range.1, }), finished_functions, trampolines, @@ -227,16 +236,23 @@ impl CompiledModule { traps, stack_maps, address_transform, - obj, unwind_info, }) } /// Extracts `CompilationArtifacts` from the compiled module. pub fn to_compilation_artifacts(&self) -> CompilationArtifacts { + // Get ELF image bytes and sanitize that. + let mut obj = Vec::from(unsafe { + let range = &self.code.image_range; + std::slice::from_raw_parts(range.start as *const u8, range.len()) + }); + drop(unlink_module(&mut obj)); + drop(sanitize_loadable_file(&mut obj)); + CompilationArtifacts { module: (*self.module).clone(), - obj: self.obj.clone(), + obj, unwind_info: self.unwind_info.clone(), data_initializers: self.data_initializers.clone(), traps: self.traps.clone(), @@ -372,23 +388,9 @@ impl OwnedDataInitializer { } } -fn create_dbg_image( - obj: Vec, - code_range: (*const u8, usize), - module: &Module, - finished_functions: &PrimaryMap, -) -> Result, SetupError> { - let funcs = finished_functions - .values() - .map(|allocated: &*mut [VMFunctionBody]| (*allocated) as *const u8) - .collect::>(); - create_gdbjit_image(obj, code_range, module.local.num_imported_funcs, &funcs) - .map_err(SetupError::DebugInfo) -} - fn build_code_memory( isa: &dyn TargetIsa, - obj: &[u8], + obj: Vec, module: &Module, unwind_info: &Box<[ObjectUnwindInfo]>, ) -> Result< @@ -400,11 +402,9 @@ fn build_code_memory( ), String, > { - let obj = ObjectFile::parse(obj).map_err(|_| "Unable to read obj".to_string())?; - let mut code_memory = CodeMemory::new(); - let allocation = code_memory.allocate_for_object(&obj, unwind_info)?; + let mut allocation = code_memory.allocate_for_object(&obj, &unwind_info)?; // Second, create a PrimaryMap from result vector of pointers. let mut finished_functions = PrimaryMap::new(); @@ -424,13 +424,18 @@ fn build_code_memory( } let code_range = allocation.code_range(); + let code_range = (code_range.as_ptr(), code_range.len()); - link_module(&obj, &module, code_range, &finished_functions); + let obj = allocation.as_mut_slice(); + link_module(obj, &module, &finished_functions)?; - let code_range = (code_range.as_ptr(), code_range.len()); + patch_loadable_file(obj, code_range).map_err(|e| e.to_string())?; + + let obj = (obj.as_ptr(), obj.len()); // Make all code compiled thus far executable. + // TODO publish only .text section of ELF object. code_memory.publish(isa); - Ok((code_memory, code_range, finished_functions, trampolines)) + Ok((code_memory, obj, finished_functions, trampolines)) } diff --git a/crates/jit/src/link.rs b/crates/jit/src/link.rs index 71284e875586..d74f969d04d4 100644 --- a/crates/jit/src/link.rs +++ b/crates/jit/src/link.rs @@ -2,7 +2,7 @@ use crate::object::utils::try_parse_func_name; use object::read::{Object, ObjectSection, Relocation, RelocationTarget}; -use object::{elf, File, RelocationEncoding, RelocationKind}; +use object::{elf, File, RelocationEncoding, RelocationKind, SymbolKind}; use std::ptr::{read_unaligned, write_unaligned}; use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::wasm::DefinedFuncIndex; @@ -19,18 +19,27 @@ use wasmtime_runtime::VMFunctionBody; /// TODO refactor logic to remove panics and add defensive code the image data /// becomes untrusted. pub fn link_module( - obj: &File, + obj_data: &mut [u8], module: &Module, - code_range: &mut [u8], finished_functions: &PrimaryMap, -) { - // Read the ".text" section and process its relocations. - let text_section = obj.section_by_name(".text").unwrap(); - let body = code_range.as_ptr() as *const VMFunctionBody; +) -> Result<(), String> { + let obj = File::parse(obj_data).map_err(|_| "Unable to read obj".to_string())?; + + for section in obj.sections() { + let body_offset = match section.file_range() { + Some((start, _)) => start as usize, + None => { + continue; + } + }; + let body = unsafe { obj_data.as_ptr().add(body_offset) } as *const VMFunctionBody; - for (offset, r) in text_section.relocations() { - apply_reloc(module, obj, finished_functions, body, offset, r); + for (offset, r) in section.relocations() { + apply_reloc(module, &obj, finished_functions, body, offset, r); + } } + + Ok(()) } fn apply_reloc( @@ -44,8 +53,11 @@ fn apply_reloc( let target_func_address: usize = match r.target() { RelocationTarget::Symbol(i) => { // Processing relocation target is a named symbols that is compiled - // wasm function or runtime libcall. + // wasm function or runtime libcall. These are `SymbolKind::Text`. let sym = obj.symbol_by_index(i).unwrap(); + if sym.kind() != SymbolKind::Text { + return; + } match sym.name() { Some(name) => { if let Some(index) = try_parse_func_name(name) { @@ -67,7 +79,15 @@ fn apply_reloc( } _ => panic!("unexpected relocation target"), }; + patch_reloc_entry(body, offset, target_func_address, r) +} +fn patch_reloc_entry( + body: *const VMFunctionBody, + offset: u64, + target_func_address: usize, + r: Relocation, +) { match (r.kind(), r.encoding(), r.size()) { #[cfg(target_pointer_width = "64")] (RelocationKind::Absolute, RelocationEncoding::Generic, 64) => unsafe { @@ -132,6 +152,25 @@ fn apply_reloc( } } +/// Sanitizes relocation information (they may point to real addresses). +pub fn unlink_module(obj_data: &mut [u8]) -> Result<(), String> { + let obj = File::parse(obj_data).map_err(|_| "Unable to read obj".to_string())?; + for section in obj.sections() { + let body_offset = match section.file_range() { + Some((start, _)) => start as usize, + None => { + continue; + } + }; + let body = unsafe { obj_data.as_ptr().add(body_offset) } as *const VMFunctionBody; + const NON_EXISTENT_ADDRESS: usize = 0; + for (offset, r) in section.relocations() { + patch_reloc_entry(body, offset, NON_EXISTENT_ADDRESS, r); + } + } + Ok(()) +} + fn to_libcall_address(name: &str) -> Option { use self::libcalls::*; use wasmtime_environ::for_each_libcall; diff --git a/crates/jit/src/object.rs b/crates/jit/src/object.rs index 66e3a3520943..b7b155acca80 100644 --- a/crates/jit/src/object.rs +++ b/crates/jit/src/object.rs @@ -1,15 +1,15 @@ //! Object file generation. use super::trampoline::build_trampoline; +use anyhow::Error; use cranelift_frontend::FunctionBuilderContext; -use object::write::Object; use serde::{Deserialize, Serialize}; use wasmtime_debug::DwarfSection; use wasmtime_environ::entity::{EntityRef, PrimaryMap}; use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; use wasmtime_environ::wasm::{FuncIndex, SignatureIndex}; use wasmtime_environ::{Compilation, Module, Relocations}; -use wasmtime_obj::{ObjectBuilder, ObjectBuilderTarget}; +use wasmtime_obj::{convert_object_elf_to_loadable_file, ObjectBuilder, ObjectBuilderTarget}; pub use wasmtime_obj::utils; @@ -27,7 +27,7 @@ pub(crate) fn build_object( compilation: Compilation, relocations: Relocations, dwarf_sections: Vec, -) -> Result<(Object, Vec), anyhow::Error> { +) -> Result<(Vec, Vec), Error> { const CODE_SECTION_ALIGNMENT: u64 = 0x1000; assert_eq!( isa.triple().architecture.endianness(), @@ -73,5 +73,9 @@ pub(crate) fn build_object( .set_dwarf_sections(dwarf_sections); let obj = builder.build()?; - Ok((obj, unwind_info)) + let mut bytes = obj.write()?; + + convert_object_elf_to_loadable_file(&mut bytes); + + Ok((bytes, unwind_info)) } diff --git a/crates/obj/src/lib.rs b/crates/obj/src/lib.rs index 5db5bb071ddf..8d1efa373417 100644 --- a/crates/obj/src/lib.rs +++ b/crates/obj/src/lib.rs @@ -27,10 +27,15 @@ mod builder; mod context; mod data_segment; mod module; +mod patchable; mod table; pub use crate::builder::{utils, ObjectBuilder, ObjectBuilderTarget}; pub use crate::module::emit_module; +pub use crate::patchable::{ + convert_object_elf_to_loadable_file, ensure_supported_elf_format, patch_loadable_file, + sanitize_loadable_file, +}; /// Version number of this crate. pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/crates/obj/src/patchable.rs b/crates/obj/src/patchable.rs new file mode 100644 index 000000000000..3def8d0832fa --- /dev/null +++ b/crates/obj/src/patchable.rs @@ -0,0 +1,342 @@ +use anyhow::{bail, ensure, Error}; +use object::elf::*; +use object::endian::LittleEndian; +use std::ffi::CStr; +use std::mem::size_of; +use std::os::raw::c_char; + +/// Checks if ELF is supported. +pub fn ensure_supported_elf_format(bytes: &[u8]) -> Result<(), Error> { + parse_elf_object_mut(bytes)?; + Ok(()) +} + +/// Converts ELF into loadable file. +pub fn convert_object_elf_to_loadable_file(bytes: &mut Vec) { + // TODO support all platforms, but now don't fix unsupported. + let mut file = match parse_elf_object_mut(bytes) { + Ok(file) => file, + Err(_) => { + return; + } + }; + + let shstrtab_off = shstrtab_off(file.as_mut()); + + let segment = ElfSectionIterator::new(file.as_mut()) + .find(|s| is_text_section(bytes, shstrtab_off, s.as_ref())) + .map(|s| { + let sh_offset = s.sh_offset(); + let sh_size = s.sh_size(); + (sh_offset, sh_size) + }); + + // LLDB wants segment with virtual address set, placing them at the end of ELF. + let ph_off = bytes.len(); + let e_phentsize = file.target_phentsize(); + let e_phnum = 1; + bytes.resize(ph_off + e_phentsize * e_phnum, 0); + let mut file = parse_elf_object_mut(bytes).unwrap(); + + if let Some((sh_offset, sh_size)) = segment { + let mut program = file.program_at(ph_off as u64); + program.p_type_set(PT_LOAD); + program.p_offset_set(sh_offset); + program.p_filesz_set(sh_size); + program.p_memsz_set(sh_size); + } else { + unreachable!(); + } + + // It is somewhat loadable ELF file at this moment. + file.e_type_set(ET_DYN); + file.e_ph_set(e_phentsize as u16, ph_off as u64, e_phnum as u16); + + // A linker needs to patch section's `sh_addr` and + // program's `p_vaddr`, `p_paddr`, and maybe `p_memsz`. +} + +/// Patches loadable file fields. +pub fn patch_loadable_file(bytes: &mut [u8], code_region: (*const u8, usize)) -> Result<(), Error> { + // TODO support all platforms, but now don't fix unsupported. + let mut file = match parse_elf_object_mut(bytes) { + Ok(file) => file, + Err(_) => { + return Ok(()); + } + }; + + ensure!( + file.e_phoff() != 0 && file.e_phnum() == 1, + "program header table must created" + ); + + let shstrtab_off = shstrtab_off(file.as_mut()); + + if let Some(mut section) = ElfSectionIterator::new(file.as_mut()) + .find(|s| is_text_section(bytes, shstrtab_off, s.as_ref())) + { + // Patch vaddr, and save file location and its size. + section.sh_addr_set(code_region.0 as u64); + } + + // LLDB wants segment with virtual address set, placing them at the end of ELF. + let ph_off = file.e_phoff(); + let (v_offset, size) = code_region; + let mut program = file.program_at(ph_off); + program.p_vaddr_set(v_offset as u64); + program.p_paddr_set(v_offset as u64); + program.p_memsz_set(size as u64); + + Ok(()) +} + +/// Removes all patched information from loadable file fields. +/// Inverse of `patch_loadable_file`. +pub fn sanitize_loadable_file(bytes: &mut [u8]) -> Result<(), Error> { + const NON_EXISTENT_RANGE: (*const u8, usize) = (std::ptr::null(), 0); + patch_loadable_file(bytes, NON_EXISTENT_RANGE) +} + +fn is_text_section(bytes: &mut [u8], shstrtab_off: u64, section: &dyn ElfSection) -> bool { + if section.sh_type() != SHT_PROGBITS { + return false; + } + // It is a SHT_PROGBITS, but we need to check sh_name to ensure it is our function + let sh_name_off = section.sh_name(); + let sh_name = unsafe { + CStr::from_ptr( + bytes + .as_ptr() + .offset((shstrtab_off + sh_name_off as u64) as isize) as *const c_char, + ) + .to_str() + .expect("name") + }; + sh_name == ".text" +} + +fn shstrtab_off(file: &mut dyn ElfObject) -> u64 { + ElfSectionIterator::new(file) + .rev() + .find(|s| s.sh_type() == SHT_STRTAB) + .map_or(0, |s| s.sh_offset()) +} + +fn parse_elf_object_mut<'data>(obj: &'data [u8]) -> Result, Error> { + let ident: &Ident = unsafe { &*(obj.as_ptr() as *const Ident) }; + let elf = match (ident.class, ident.data) { + (ELFCLASS64, ELFDATA2LSB) => Box::new(ElfObject64LE { + header: unsafe { &mut *(obj.as_ptr() as *mut FileHeader64<_>) }, + }), + (c, d) => { + bail!("Unsupported elf format (class: {}, data: {})", c, d); + } + }; + match elf.e_machine() { + EM_X86_64 | EM_ARM => (), + machine => { + bail!("Unsupported ELF target machine: {:x}", machine); + } + } + let e_shentsize = elf.e_shentsize(); + ensure!(e_shentsize as usize == elf.target_shentsize(), "size of sh"); + Ok(elf) +} + +trait ElfObject { + fn e_ident(&self) -> &Ident; + fn e_machine(&self) -> u16; + fn e_type(&self) -> u16; + fn e_type_set(&mut self, ty: u16); + fn e_shentsize(&self) -> u16; + fn e_shoff(&self) -> u64; + fn e_shnum(&self) -> u16; + fn e_phentsize(&self) -> u16; + fn e_phoff(&self) -> u64; + fn e_phnum(&self) -> u16; + fn e_ph_set(&mut self, entsize: u16, off: u64, num: u16); + fn target_shentsize(&self) -> usize; + fn target_phentsize(&self) -> usize; + fn section_at<'file, 'a>(&'file mut self, off: u64) -> Box + where + 'a: 'file; + fn program_at<'file, 'a>(&'file mut self, off: u64) -> Box + where + 'a: 'file; +} + +struct ElfObject64LE<'data> { + header: &'data mut FileHeader64, +} + +impl<'data> ElfObject for ElfObject64LE<'data> { + fn e_ident(&self) -> &Ident { + &self.header.e_ident + } + fn e_machine(&self) -> u16 { + self.header.e_machine.get(LittleEndian) + } + fn e_type(&self) -> u16 { + self.header.e_type.get(LittleEndian) + } + fn e_type_set(&mut self, ty: u16) { + self.header.e_type.set(LittleEndian, ty); + } + fn e_shentsize(&self) -> u16 { + self.header.e_shentsize.get(LittleEndian) + } + fn e_shoff(&self) -> u64 { + self.header.e_shoff.get(LittleEndian) + } + fn e_shnum(&self) -> u16 { + self.header.e_shnum.get(LittleEndian) + } + fn e_phentsize(&self) -> u16 { + self.header.e_phentsize.get(LittleEndian) + } + fn e_phoff(&self) -> u64 { + self.header.e_phoff.get(LittleEndian) + } + fn e_phnum(&self) -> u16 { + self.header.e_phnum.get(LittleEndian) + } + fn e_ph_set(&mut self, entsize: u16, off: u64, num: u16) { + self.header.e_phentsize.set(LittleEndian, entsize); + self.header.e_phoff.set(LittleEndian, off); + self.header.e_phnum.set(LittleEndian, num); + } + fn target_shentsize(&self) -> usize { + size_of::>() + } + fn target_phentsize(&self) -> usize { + size_of::>() + } + fn section_at<'file, 'a>(&'file mut self, off: u64) -> Box + where + 'a: 'file, + { + let header: &mut SectionHeader64 = unsafe { + &mut *((self.header as *const _ as *const u8).offset(off as isize) + as *mut SectionHeader64<_>) + }; + Box::new(ElfSection64LE { header }) + } + fn program_at<'file, 'a>(&'file mut self, off: u64) -> Box + where + 'a: 'file, + { + let header: &mut ProgramHeader64 = unsafe { + &mut *((self.header as *const _ as *const u8).offset(off as isize) + as *mut ProgramHeader64<_>) + }; + Box::new(ElfProgram64LE { header }) + } +} + +trait ElfSection { + fn sh_type(&self) -> u32; + fn sh_name(&self) -> u32; + fn sh_offset(&self) -> u64; + fn sh_size(&self) -> u64; + fn sh_addr_set(&mut self, addr: u64); +} + +struct ElfSection64LE<'data> { + header: &'data mut SectionHeader64, +} + +impl<'data> ElfSection for ElfSection64LE<'data> { + fn sh_type(&self) -> u32 { + self.header.sh_type.get(LittleEndian) + } + fn sh_name(&self) -> u32 { + self.header.sh_name.get(LittleEndian) + } + fn sh_offset(&self) -> u64 { + self.header.sh_offset.get(LittleEndian) + } + fn sh_size(&self) -> u64 { + self.header.sh_size.get(LittleEndian) + } + fn sh_addr_set(&mut self, addr: u64) { + self.header.sh_addr.set(LittleEndian, addr); + } +} + +struct ElfSectionIterator<'b> { + e_shentsize: u64, + e_shoff: u64, + elf: &'b mut dyn ElfObject, + start: u64, + end: u64, +} +impl<'b> ElfSectionIterator<'b> { + pub fn new(elf: &'b mut (dyn ElfObject + 'b)) -> Self { + let e_shentsize = elf.e_shentsize() as u64; + let e_shoff = elf.e_shoff(); + let e_shnum = elf.e_shnum() as u64; + Self { + e_shentsize, + e_shoff, + elf, + start: 0, + end: e_shnum, + } + } +} +impl<'b> Iterator for ElfSectionIterator<'b> { + type Item = Box; + fn next(&mut self) -> Option { + if self.start >= self.end { + return None; + } + let off = self.e_shoff + self.start * self.e_shentsize; + self.start += 1; + Some(self.elf.section_at(off)) + } +} +impl<'b> DoubleEndedIterator for ElfSectionIterator<'b> { + fn next_back(&mut self) -> Option { + if self.start >= self.end { + return None; + } + self.end -= 1; + let off = self.e_shoff + self.end * self.e_shentsize; + Some(self.elf.section_at(off)) + } +} + +trait ElfProgram { + fn p_vaddr_set(&mut self, off: u64); + fn p_paddr_set(&mut self, off: u64); + fn p_memsz_set(&mut self, size: u64); + fn p_type_set(&mut self, ty: u32); + fn p_offset_set(&mut self, off: u64); + fn p_filesz_set(&mut self, size: u64); +} + +struct ElfProgram64LE<'data> { + header: &'data mut ProgramHeader64, +} + +impl<'data> ElfProgram for ElfProgram64LE<'data> { + fn p_vaddr_set(&mut self, off: u64) { + self.header.p_vaddr.set(LittleEndian, off); + } + fn p_paddr_set(&mut self, off: u64) { + self.header.p_paddr.set(LittleEndian, off); + } + fn p_memsz_set(&mut self, size: u64) { + self.header.p_memsz.set(LittleEndian, size); + } + fn p_type_set(&mut self, ty: u32) { + self.header.p_type.set(LittleEndian, ty); + } + fn p_offset_set(&mut self, off: u64) { + self.header.p_offset.set(LittleEndian, off); + } + fn p_filesz_set(&mut self, size: u64) { + self.header.p_filesz.set(LittleEndian, size); + } +} diff --git a/crates/runtime/src/jit_int.rs b/crates/runtime/src/jit_int.rs index 4fd1e7f88e41..60b3c5f8a9ea 100644 --- a/crates/runtime/src/jit_int.rs +++ b/crates/runtime/src/jit_int.rs @@ -59,21 +59,19 @@ lazy_static! { /// Registeration for JIT image pub struct GdbJitImageRegistration { entry: Pin>, - file: Pin>, + file: *const u8, } impl GdbJitImageRegistration { /// Registers JIT image using __jit_debug_register_code - pub fn register(file: Vec) -> Self { - let file = Pin::new(file.into_boxed_slice()); - + pub fn register(file: *const u8, len: usize) -> Self { // Create a code entry for the file, which gives the start and size // of the symbol file. let mut entry = Pin::new(Box::new(JITCodeEntry { next_entry: ptr::null_mut(), prev_entry: ptr::null_mut(), - symfile_addr: file.as_ptr(), - symfile_size: file.len() as u64, + symfile_addr: file, + symfile_size: len as u64, })); unsafe { @@ -84,8 +82,8 @@ impl GdbJitImageRegistration { } /// JIT image used in registration - pub fn file(&self) -> &[u8] { - &self.file + pub fn file(&self) -> *const u8 { + self.file } }