Change VMMemoryDefinition::current_length to usize

cranelift/codegen/src/ir/dfg.rs

@@ -7,8 +7,8 @@ use crate::ir::extfunc::ExtFuncData;
 use crate::ir::instructions::{BranchInfo, CallInfo, InstructionData};
 use crate::ir::{types, ConstantData, ConstantPool, Immediate};
 use crate::ir::{
-    Block, FuncRef, Inst, SigRef, Signature, Type, Value, ValueLabelAssignments, ValueList,
-    ValueListPool,
+    Block, FuncRef, Inst, SigRef, Signature, SourceLoc, Type, Value, ValueLabelAssignments,
+    ValueList, ValueListPool,
 };
 use crate::isa::TargetIsa;
 use crate::packed_option::ReservedValue;
@@ -153,6 +153,14 @@ impl DataFlowGraph {
             self.values_labels = Some(HashMap::new());
         }
     }
+
+    /// Inserts a `ValueLabelAssignments::Alias` for `to_alias` if debug info
+    /// collection is enabled.
+    pub fn add_value_label_alias(&mut self, to_alias: Value, from: SourceLoc, value: Value) {
+        if let Some(values_labels) = self.values_labels.as_mut() {
+            values_labels.insert(to_alias, ir::ValueLabelAssignments::Alias { from, value });
+        }
+    }
 }
 
 /// Resolve value aliases.

cranelift/codegen/src/legalizer/heap.rs

@@ -64,8 +64,10 @@ fn dynamic_addr(
     let mut pos = FuncCursor::new(func).at_inst(inst);
     pos.use_srcloc(inst);
 
+    let offset = cast_offset_to_pointer_ty(offset, offset_ty, addr_ty, &mut pos);
+
     // Start with the bounds check. Trap if `offset + access_size > bound`.
-    let bound = pos.ins().global_value(offset_ty, bound_gv);
+    let bound = pos.ins().global_value(addr_ty, bound_gv);
     let (cc, lhs, bound) = if access_size == 1 {
         // `offset > bound - 1` is the same as `offset >= bound`.
         (IntCC::UnsignedGreaterThanOrEqual, offset, bound)
@@ -76,7 +78,7 @@ fn dynamic_addr(
         (IntCC::UnsignedGreaterThan, offset, adj_bound)
     } else {
         // We need an overflow check for the adjusted offset.
-        let access_size_val = pos.ins().iconst(offset_ty, access_size as i64);
+        let access_size_val = pos.ins().iconst(addr_ty, access_size as i64);
         let (adj_offset, overflow) = pos.ins().iadd_ifcout(offset, access_size_val);
         pos.ins().trapif(
             isa.unsigned_add_overflow_condition(),
@@ -100,7 +102,6 @@ fn dynamic_addr(
         heap,
         addr_ty,
         offset,
-        offset_ty,
         pos.func,
         spectre_oob_comparison,
     );
@@ -111,7 +112,7 @@ fn static_addr(
     isa: &dyn TargetIsa,
     inst: ir::Inst,
     heap: ir::Heap,
-    offset: ir::Value,
+    mut offset: ir::Value,
     access_size: u32,
     bound: u64,
     func: &mut ir::Function,
@@ -156,6 +157,7 @@ fn static_addr(
     // `bound - access_size >= 4GB` we can omit a bounds check.
     let limit = bound - access_size;
     let mut spectre_oob_comparison = None;
+    offset = cast_offset_to_pointer_ty(offset, offset_ty, addr_ty, &mut pos);
     if offset_ty != ir::types::I32 || limit < 0xffff_ffff {
         let (cc, lhs, limit_imm) = if limit & 1 == 1 {
             // Prefer testing `offset >= limit - 1` when limit is odd because an even number is
@@ -169,7 +171,7 @@ fn static_addr(
         let oob = pos.ins().icmp_imm(cc, lhs, limit_imm);
         pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
         if isa.flags().enable_heap_access_spectre_mitigation() {
-            let limit = pos.ins().iconst(offset_ty, limit_imm);
+            let limit = pos.ins().iconst(addr_ty, limit_imm);
             spectre_oob_comparison = Some((cc, lhs, limit));
         }
     }
@@ -180,45 +182,57 @@ fn static_addr(
         heap,
         addr_ty,
         offset,
-        offset_ty,
         pos.func,
         spectre_oob_comparison,
     );
 }
 
+fn cast_offset_to_pointer_ty(
+    offset: ir::Value,
+    offset_ty: ir::Type,
+    addr_ty: ir::Type,
+    pos: &mut FuncCursor,
+) -> ir::Value {
+    if offset_ty == addr_ty {
+        return offset;
+    }
+    // Note that using 64-bit heaps on a 32-bit host is not currently supported,
+    // would require at least a bounds check here to ensure that the truncation
+    // from 64-to-32 bits doesn't lose any upper bits. For now though we're
+    // mostly interested in the 32-bit-heaps-on-64-bit-hosts cast.
+    assert!(offset_ty.bits() < addr_ty.bits());
+
+    // Convert `offset` to `addr_ty`.
+    let extended_offset = pos.ins().uextend(addr_ty, offset);
+
+    // Add debug value-label alias so that debuginfo can name the extended
+    // value as the address
+    let loc = pos.srcloc();
+    pos.func
+        .dfg
+        .add_value_label_alias(extended_offset, loc, offset);
+
+    extended_offset
+}
+
 /// Emit code for the base address computation of a `heap_addr` instruction.
 fn compute_addr(
     isa: &dyn TargetIsa,
     inst: ir::Inst,
     heap: ir::Heap,
     addr_ty: ir::Type,
-    mut offset: ir::Value,
-    offset_ty: ir::Type,
+    offset: ir::Value,
     func: &mut ir::Function,
     // If we are performing Spectre mitigation with conditional selects, the
     // values to compare and the condition code that indicates an out-of bounds
     // condition; on this condition, the conditional move will choose a
     // speculatively safe address (a zero / null pointer) instead.
     spectre_oob_comparison: Option<(IntCC, ir::Value, ir::Value)>,
 ) {
+    debug_assert_eq!(func.dfg.value_type(offset), addr_ty);
     let mut pos = FuncCursor::new(func).at_inst(inst);
     pos.use_srcloc(inst);
 
-    // Convert `offset` to `addr_ty`.
-    if offset_ty != addr_ty {
-        let labels_value = offset;
-        offset = pos.ins().uextend(addr_ty, offset);
-        if let Some(values_labels) = pos.func.dfg.values_labels.as_mut() {
-            values_labels.insert(
-                offset,
-                ir::ValueLabelAssignments::Alias {
-                    from: pos.func.srclocs[inst],
-                    value: labels_value,
-                },
-            );
-        }
-    }
-
     // Add the heap base address base
     let base = if isa.flags().enable_pinned_reg() && isa.flags().use_pinned_reg_as_heap_base() {
         pos.ins().get_pinned_reg(isa.pointer_type())

cranelift/codegen/src/verifier/mod.rs

@@ -439,14 +439,13 @@ impl<'a> Verifier<'a> {
                             .nonfatal((heap, format!("invalid bound global value {}", bound_gv)));
                     }
 
-                    let index_type = heap_data.index_type;
                     let bound_type = self.func.global_values[bound_gv].global_type(isa);
-                    if index_type != bound_type {
+                    if pointer_type != bound_type {
                         errors.report((
                             heap,
                             format!(
-                                "heap index type {} differs from the type of its bound, {}",
-                                index_type, bound_type
+                                "heap pointer type {} differs from the type of its bound, {}",
+                                pointer_type, bound_type
                             ),
                         ));
                     }

cranelift/filetests/filetests/isa/aarch64/heap_addr.clif

@@ -5,7 +5,7 @@ target aarch64
 
 function %dynamic_heap_check(i64 vmctx, i32) -> i64 {
     gv0 = vmctx
-    gv1 = load.i32 notrap aligned gv0
+    gv1 = load.i64 notrap aligned gv0
     heap0 = dynamic gv0, bound gv1, offset_guard 0x1000, index_type i32
 
 block0(v0: i64, v1: i32):
@@ -16,13 +16,14 @@ block0(v0: i64, v1: i32):
 ; check: Block 0:
 ; check: stp fp, lr, [sp, #-16]!
 ; nextln: mov fp, sp
-; nextln: ldr w2, [x0]
-; nextln: add w2, w2, #0
-; nextln: subs wzr, w1, w2
+; nextln: mov w2, w1
+; nextln: ldr x3, [x0]
+; nextln: mov x3, x3
+; nextln: subs xzr, x2, x3
 ; nextln: b.ls label1 ; b label2
 ; check: Block 1:
 ; check: add x0, x0, x1, UXTW
-; nextln: subs wzr, w1, w2
+; nextln: subs xzr, x2, x3
 ; nextln: movz x1, #0
 ; nextln: csel x0, x1, x0, hi
 ; nextln: ldp fp, lr, [sp], #16
@@ -42,11 +43,12 @@ block0(v0: i64, v1: i32):
 ; check: Block 0:
 ; check: stp fp, lr, [sp, #-16]!
 ; nextln: mov fp, sp
-; nextln: subs wzr, w1, #65536
+; nextln: mov w2, w1
+; nextln: subs xzr, x2, #65536
 ; nextln: b.ls label1 ; b label2
 ; check: Block 1:
 ; check: add x0, x0, x1, UXTW
-; nextln: subs wzr, w1, #65536
+; nextln: subs xzr, x2, #65536
 ; nextln: movz x1, #0
 ; nextln: csel x0, x1, x0, hi
 ; nextln: ldp fp, lr, [sp], #16

cranelift/filetests/filetests/isa/s390x/heap_addr.clif

@@ -3,7 +3,7 @@ target s390x
 
 function %dynamic_heap_check(i64 vmctx, i32) -> i64 {
     gv0 = vmctx
-    gv1 = load.i32 notrap aligned gv0
+    gv1 = load.i64 notrap aligned gv0
     heap0 = dynamic gv0, bound gv1, offset_guard 0x1000, index_type i32
 
 block0(v0: i64, v1: i32):
@@ -12,15 +12,15 @@ block0(v0: i64, v1: i32):
 }
 
 ; check: Block 0:
-; check:  l %r4, 0(%r2)
-; nextln: ahi %r4, 0
-; nextln: clr %r3, %r4
+; check:  llgfr %r3, %r3
+; nextln: lg %r4, 0(%r2)
+; nextln: aghi %r4, 0
+; nextln: clgr %r3, %r4
 ; nextln: jgnh label1 ; jg label2
 ; check: Block 1:
-; check:  llgfr %r5, %r3
-; nextln: agr %r2, %r5
+; check: agr %r2, %r3
 ; nextln: lghi %r5, 0
-; nextln: clr %r3, %r4
+; nextln: clgr %r3, %r4
 ; nextln: locgrh %r2, %r5
 ; nextln: br %r14
 ; check: Block 2:
@@ -36,13 +36,13 @@ block0(v0: i64, v1: i32):
 }
 
 ; check: Block 0:
-; check:  clfi %r3, 65536
+; check:  llgfr %r3, %r3
+; nextln: clgfi %r3, 65536
 ; nextln: jgnh label1 ; jg label2
 ; check: Block 1:
-; check:  llgfr %r4, %r3
-; nextln: agr %r2, %r4
+; check: agr %r2, %r3
 ; nextln: lghi %r4, 0
-; nextln: clfi %r3, 65536
+; nextln: clgfi %r3, 65536
 ; nextln: locgrh %r2, %r4
 ; nextln: br %r14
 ; check: Block 2:

cranelift/filetests/filetests/isa/x64/heap.clif

@@ -4,19 +4,20 @@ target x86_64 machinst
 function %f(i32, i64 vmctx) -> i64 {
     gv0 = vmctx
     gv1 = load.i64 notrap aligned gv0+0
-    gv2 = load.i32 notrap aligned gv0+8
+    gv2 = load.i64 notrap aligned gv0+8
     heap0 = dynamic gv1, bound gv2, offset_guard 0x1000, index_type i32
 
 block0(v0: i32, v1: i64):
 
     v2 = heap_addr.i64 heap0, v0, 0x8000
-    ; check:  movl    8(%rsi), %ecx
-    ; nextln: movq    %rdi, %rax
-    ; nextln: addl    $$32768, %eax
+    ; check:  movl    %edi, %ecx
+    ; nextln: movq    8(%rsi), %rdi
+    ; nextln: movq    %rcx, %rax
+    ; nextln: addq    $$32768, %rax
     ; nextln: jnb ; ud2 heap_oob ;
-    ; nextln: cmpl    %ecx, %eax
+    ; nextln: cmpq    %rdi, %rax
     ; nextln: jbe     label1; j label2
     ; check:  Block 1:
- 
+
     return v2
 }

cranelift/filetests/filetests/isa/x86/legalize-heaps.clif

@@ -10,7 +10,7 @@ function %heap_addrs(i32, i64, i64 vmctx) {
     gv0 = iadd_imm.i64 gv4, 64
     gv1 = iadd_imm.i64 gv4, 72
     gv2 = iadd_imm.i64 gv4, 80
-    gv3 = load.i32 notrap aligned gv4+88
+    gv3 = load.i64 notrap aligned gv4+88
 
     heap0 = static gv0, min 0x1_0000, bound 0x1_0000_0000, offset_guard 0x8000_0000, index_type i32
     heap1 = static gv0, offset_guard 0x1000, bound 0x1_0000, index_type i32
@@ -38,15 +38,15 @@ block0(v0: i32, v1: i64, v3: i64):
     ; check:         v4 = iadd v13, v12
 
     v5 = heap_addr.i64 heap1, v0, 0
-    ; check:         v14 = icmp_imm ugt v0, 0x0001_0000
-    ; check:         brz v14, $(resume_1=$BB)
+    ; check:         v14 = uextend.i64 v0
+    ; check:         v15 = icmp_imm ugt v14, 0x0001_0000
+    ; check:         brz v15, $(resume_1=$BB)
     ; nextln:        jump $(trap_1=$BB)
     ; check:     $trap_1:
     ; nextln:        trap heap_oob
     ; check:     $resume_1:
-    ; check:         v15 = uextend.i64 v0
     ; check:         v16 = iadd_imm.i64 v3, 64
-    ; check:         v5 = iadd v16, v15
+    ; check:         v5 = iadd v16, v14
 
     v6 = heap_addr.i64 heap2, v1, 0
     ; check:         v19 = iconst.i64 0x0001_0000_0000
@@ -70,30 +70,30 @@ block0(v0: i32, v1: i64, v3: i64):
     ; check:         v7 = iadd v21, v1
 
     v8 = heap_addr.i64 heap4, v0, 0
-    ; check:         v22 = load.i32 notrap aligned v3+88
-    ; check:         v23 = iadd_imm v22, 0
-    ; check:         v24 = icmp.i32 ugt v0, v23
-    ; check:         brz v24, $(resume_4=$BB)
+    ; check:         v22 = uextend.i64 v0
+    ; check:         v23 = load.i64 notrap aligned v3+88
+    ; check:         v24 = iadd_imm v23, 0
+    ; check:         v25 = icmp ugt v22, v24
+    ; check:         brz v25, $(resume_4=$BB)
     ; nextln:        jump $(trap_4=$BB)
     ; check:     $trap_4:
     ; nextln:        trap heap_oob
     ; check:     $resume_4:
-    ; check:         v25 = uextend.i64 v0
     ; check:         v26 = iadd_imm.i64 v3, 72
-    ; check:         v8 = iadd v26, v25
+    ; check:         v8 = iadd v26, v22
 
     v9 = heap_addr.i64 heap5, v0, 0
-    ; check:         v27 = load.i32 notrap aligned v3+88
-    ; check:         v28 = iadd_imm v27, 0
-    ; check:         v29 = icmp.i32 ugt v0, v28
-    ; check:         brz v29, $(resume_5=$BB)
+    ; check:         v27 = uextend.i64 v0
+    ; check:         v28 = load.i64 notrap aligned v3+88
+    ; check:         v29 = iadd_imm v28, 0
+    ; check:         v30 = icmp ugt v27, v29
+    ; check:         brz v30, $(resume_5=$BB)
     ; nextln:        jump $(trap_5=$BB)
     ; check:     $trap_5:
     ; nextln:        trap heap_oob
     ; check:     $resume_5:
-    ; check:         v30 = uextend.i64 v0
     ; check:         v31 = iadd_imm.i64 v3, 72
-    ; check:         v9 = iadd v31, v30
+    ; check:         v9 = iadd v31, v27
 
     v10 = heap_addr.i64 heap6, v1, 0
     ; check:         v32 = iadd_imm.i64 v3, 80

cranelift/filetests/filetests/isa/x86/legalize-memory.clif

@@ -56,7 +56,7 @@ block0(v0: i32, v999: i64):
     ; Boundscheck should be eliminated.
     ; Checks here are assuming that no pipehole opts fold the load offsets.
     ; nextln: $(xoff=$V) = uextend.i64 v0
-    ; nextln: $(hbase=$V) = iadd_imm v999, 64
+    ; check: $(hbase=$V) = iadd_imm v999, 64
     ; nextln: v1 = iadd $hbase, $xoff
     v2 = load.f32 v1+16
     ; nextln: v2 = load.f32 v1+16
@@ -99,15 +99,15 @@ block0(v0: i32, v999: i64):
     ; check: block0(
     v1 = heap_addr.i64 heap0, v0, 0x8000_0000
     ; Boundscheck code
+    ; check: $(xoff=$V) = uextend.i64 v0
     ; check: $(oob=$V) = icmp
     ; nextln: brz $oob, $(ok=$BB)
     ; nextln: jump $(trap_oob=$BB)
     ; check: $trap_oob:
     ; nextln: trap heap_oob
     ; check: $ok:
     ; Checks here are assuming that no pipehole opts fold the load offsets.
-    ; nextln: $(xoff=$V) = uextend.i64 v0
-    ; nextln: $(hbase=$V) = iadd_imm.i64 v999, 64
+    ; check: $(hbase=$V) = iadd_imm.i64 v999, 64
     ; nextln: v1 = iadd $hbase, $xoff
     v2 = load.f32 v1+0x7fff_ffff
     ; nextln: v2 = load.f32 v1+0x7fff_ffff