diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs index 1dc182a9ac94..72194f7e899f 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.rs +++ b/cranelift/codegen/src/isa/aarch64/lower.rs @@ -81,23 +81,20 @@ impl ResultRSEImm12 { // Lowering: convert instruction inputs to forms that we can use. /// Lower an instruction input to a 64-bit constant, if possible. -pub(crate) fn input_to_const>(ctx: &mut C, input: InsnInput) -> Option { +pub(crate) fn input_to_const(ctx: &mut Lower, input: InsnInput) -> Option { let input = ctx.get_input_as_source_or_const(input.insn, input.input); input.constant } /// Lower an instruction input to a constant register-shift amount, if possible. -pub(crate) fn input_to_shiftimm>( - ctx: &mut C, +pub(crate) fn input_to_shiftimm( + ctx: &mut Lower, input: InsnInput, ) -> Option { input_to_const(ctx, input).and_then(ShiftOpShiftImm::maybe_from_shift) } -pub(crate) fn const_param_to_u128>( - ctx: &mut C, - inst: IRInst, -) -> Option { +pub(crate) fn const_param_to_u128(ctx: &mut Lower, inst: IRInst) -> Option { match ctx.get_immediate(inst) { Some(DataValue::V128(bytes)) => Some(u128::from_le_bytes(bytes)), _ => None, @@ -139,7 +136,7 @@ impl NarrowValueMode { /// Emits instruction(s) to generate the given constant value into newly-allocated /// temporary registers, returning these registers. -fn generate_constant>(ctx: &mut C, ty: Type, c: u128) -> ValueRegs { +fn generate_constant(ctx: &mut Lower, ty: Type, c: u128) -> ValueRegs { let from_bits = ty_bits(ty); let masked = if from_bits < 128 { c & ((1u128 << from_bits) - 1) @@ -160,8 +157,8 @@ fn generate_constant>(ctx: &mut C, ty: Type, c: u128) -> V /// Extends a register according to `narrow_mode`. /// If extended, the value is always extended to 64 bits, for simplicity. -fn extend_reg>( - ctx: &mut C, +fn extend_reg( + ctx: &mut Lower, ty: Type, in_reg: Reg, is_const: bool, @@ -232,10 +229,7 @@ fn extend_reg>( } /// Lowers an instruction input to multiple regs -fn lower_value_to_regs>( - ctx: &mut C, - value: Value, -) -> (ValueRegs, Type, bool) { +fn lower_value_to_regs(ctx: &mut Lower, value: Value) -> (ValueRegs, Type, bool) { trace!("lower_value_to_regs: value {:?}", value); let ty = ctx.value_ty(value); let inputs = ctx.get_value_as_source_or_const(value); @@ -256,8 +250,8 @@ fn lower_value_to_regs>( /// The given register will be extended appropriately, according to /// `narrow_mode` and the input's type. If extended, the value is /// always extended to 64 bits, for simplicity. -pub(crate) fn put_input_in_reg>( - ctx: &mut C, +pub(crate) fn put_input_in_reg( + ctx: &mut Lower, input: InsnInput, narrow_mode: NarrowValueMode, ) -> Reg { @@ -266,11 +260,7 @@ pub(crate) fn put_input_in_reg>( } /// Like above, only for values -fn put_value_in_reg>( - ctx: &mut C, - value: Value, - narrow_mode: NarrowValueMode, -) -> Reg { +fn put_value_in_reg(ctx: &mut Lower, value: Value, narrow_mode: NarrowValueMode) -> Reg { let (in_regs, ty, is_const) = lower_value_to_regs(ctx, value); let reg = in_regs .only_reg() @@ -280,10 +270,7 @@ fn put_value_in_reg>( } /// Lower an instruction input to multiple regs -pub(crate) fn put_input_in_regs>( - ctx: &mut C, - input: InsnInput, -) -> ValueRegs { +pub(crate) fn put_input_in_regs(ctx: &mut Lower, input: InsnInput) -> ValueRegs { let value = ctx.input_as_value(input.insn, input.input); let (in_regs, _, _) = lower_value_to_regs(ctx, value); in_regs @@ -300,8 +287,8 @@ pub(crate) fn put_input_in_regs>( /// divide or a right-shift or a compare-to-zero), `narrow_mode` should be /// set to `ZeroExtend` or `SignExtend` as appropriate, and the resulting /// register will be provided the extended value. -fn put_input_in_rs>( - ctx: &mut C, +fn put_input_in_rs( + ctx: &mut Lower, input: InsnInput, narrow_mode: NarrowValueMode, ) -> ResultRS { @@ -334,8 +321,8 @@ fn put_input_in_rs>( /// vreg into which the source instruction will generate its value. /// /// See note on `put_input_in_rs` for a description of `narrow_mode`. -fn put_input_in_rse>( - ctx: &mut C, +fn put_input_in_rse( + ctx: &mut Lower, input: InsnInput, narrow_mode: NarrowValueMode, ) -> ResultRSE { @@ -348,8 +335,8 @@ fn put_input_in_rse>( ResultRSE::from_rs(put_input_in_rs(ctx, input, narrow_mode)) } -fn get_as_extended_value>( - ctx: &mut C, +fn get_as_extended_value( + ctx: &mut Lower, val: Value, narrow_mode: NarrowValueMode, ) -> Option<(Value, ExtendOp)> { @@ -427,8 +414,8 @@ fn get_as_extended_value>( None } -pub(crate) fn put_input_in_rse_imm12>( - ctx: &mut C, +pub(crate) fn put_input_in_rse_imm12( + ctx: &mut Lower, input: InsnInput, narrow_mode: NarrowValueMode, ) -> ResultRSEImm12 { @@ -526,8 +513,8 @@ type AddressAddend64List = SmallVec<[Reg; 4]>; /// additional masking of high-order bits, which is too complex. So, in essence, we /// descend any number of adds from the roots, collecting all 64-bit address addends; /// then possibly support extensions at these leaves. -fn collect_address_addends>( - ctx: &mut C, +fn collect_address_addends( + ctx: &mut Lower, roots: &[InsnInput], ) -> (AddressAddend64List, AddressAddend32List, i64) { let mut result32: AddressAddend32List = SmallVec::new(); @@ -597,8 +584,8 @@ fn collect_address_addends>( } /// Lower the address of a pair load or store. -pub(crate) fn lower_pair_address>( - ctx: &mut C, +pub(crate) fn lower_pair_address( + ctx: &mut Lower, roots: &[InsnInput], offset: i32, ) -> PairAMode { @@ -654,8 +641,8 @@ pub(crate) fn lower_pair_address>( } /// Lower the address of a load or store. -pub(crate) fn lower_address>( - ctx: &mut C, +pub(crate) fn lower_address( + ctx: &mut Lower, elem_ty: Type, roots: &[InsnInput], offset: i32, @@ -764,8 +751,8 @@ pub(crate) fn lower_address>( memarg } -fn lower_add_addends>( - ctx: &mut C, +fn lower_add_addends( + ctx: &mut Lower, rd: Writable, addends64: AddressAddend64List, addends32: AddressAddend32List, @@ -803,7 +790,7 @@ fn lower_add_addends>( /// Adds into `rd` a signed imm pattern matching the best instruction for it. // TODO: This function is duplicated in ctx.gen_add_imm -fn lower_add_immediate>(ctx: &mut C, dst: Writable, src: Reg, imm: i64) { +fn lower_add_immediate(ctx: &mut Lower, dst: Writable, src: Reg, imm: i64) { // If we can fit offset or -offset in an imm12, use an add-imm // Otherwise, lower the constant first then add. if let Some(imm12) = Imm12::maybe_from_u64(imm as u64) { @@ -834,21 +821,13 @@ fn lower_add_immediate>(ctx: &mut C, dst: Writable, s } } -pub(crate) fn lower_constant_u64>( - ctx: &mut C, - rd: Writable, - value: u64, -) { +pub(crate) fn lower_constant_u64(ctx: &mut Lower, rd: Writable, value: u64) { for inst in Inst::load_constant(rd, value) { ctx.emit(inst); } } -pub(crate) fn lower_constant_f32>( - ctx: &mut C, - rd: Writable, - value: f32, -) { +pub(crate) fn lower_constant_f32(ctx: &mut Lower, rd: Writable, value: f32) { let alloc_tmp = |ty| ctx.alloc_tmp(ty).only_reg().unwrap(); for inst in Inst::load_fp_constant32(rd, value.to_bits(), alloc_tmp) { @@ -856,11 +835,7 @@ pub(crate) fn lower_constant_f32>( } } -pub(crate) fn lower_constant_f64>( - ctx: &mut C, - rd: Writable, - value: f64, -) { +pub(crate) fn lower_constant_f64(ctx: &mut Lower, rd: Writable, value: f64) { let alloc_tmp = |ty| ctx.alloc_tmp(ty).only_reg().unwrap(); for inst in Inst::load_fp_constant64(rd, value.to_bits(), alloc_tmp) { @@ -868,11 +843,7 @@ pub(crate) fn lower_constant_f64>( } } -pub(crate) fn lower_constant_f128>( - ctx: &mut C, - rd: Writable, - value: u128, -) { +pub(crate) fn lower_constant_f128(ctx: &mut Lower, rd: Writable, value: u128) { if value == 0 { // Fast-track a common case. The general case, viz, calling `Inst::load_fp_constant128`, // is potentially expensive. @@ -890,8 +861,8 @@ pub(crate) fn lower_constant_f128>( } } -pub(crate) fn lower_splat_const>( - ctx: &mut C, +pub(crate) fn lower_splat_const( + ctx: &mut Lower, rd: Writable, value: u64, size: VectorSize, @@ -974,8 +945,8 @@ pub(crate) fn lower_fp_condcode(cc: FloatCC) -> Cond { } } -pub(crate) fn lower_vector_compare>( - ctx: &mut C, +pub(crate) fn lower_vector_compare( + ctx: &mut Lower, rd: Writable, mut rn: Reg, mut rm: Reg, @@ -1114,8 +1085,8 @@ pub(crate) fn choose_32_64(ty: Type, op32: T, op64: T) -> T { } /// Checks for an instance of `op` feeding the given input. -pub(crate) fn maybe_input_insn>( - c: &mut C, +pub(crate) fn maybe_input_insn( + c: &mut Lower, input: InsnInput, op: Opcode, ) -> Option { @@ -1137,8 +1108,8 @@ pub(crate) fn maybe_input_insn>( } /// Checks for an instance of any one of `ops` feeding the given input. -pub(crate) fn maybe_input_insn_multi>( - c: &mut C, +pub(crate) fn maybe_input_insn_multi( + c: &mut Lower, input: InsnInput, ops: &[Opcode], ) -> Option<(Opcode, IRInst)> { @@ -1155,8 +1126,8 @@ pub(crate) fn maybe_input_insn_multi>( /// /// FIXME cfallin 2020-03-30: this is really ugly. Factor out tree-matching stuff and make it /// a bit more generic. -pub(crate) fn maybe_input_insn_via_conv>( - c: &mut C, +pub(crate) fn maybe_input_insn_via_conv( + c: &mut Lower, input: InsnInput, op: Opcode, conv: Opcode, @@ -1222,8 +1193,8 @@ impl IcmpResult { /// /// We can lower into the status flags, or materialize the result into a register /// This is controlled by the `output` parameter. -pub(crate) fn lower_icmp>( - ctx: &mut C, +pub(crate) fn lower_icmp( + ctx: &mut Lower, insn: IRInst, condcode: IntCC, output: IcmpOutput, @@ -1478,7 +1449,7 @@ pub(crate) fn lower_icmp>( }) } -pub(crate) fn lower_fcmp_or_ffcmp_to_flags>(ctx: &mut C, insn: IRInst) { +pub(crate) fn lower_fcmp_or_ffcmp_to_flags(ctx: &mut Lower, insn: IRInst) { let ty = ctx.input_ty(insn, 0); let inputs = [InsnInput { insn, input: 0 }, InsnInput { insn, input: 1 }]; let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); @@ -1493,8 +1464,8 @@ pub(crate) fn lower_fcmp_or_ffcmp_to_flags>(ctx: &mut C, i /// Materialize a boolean value into a register from the flags /// (e.g set by a comparison). /// A 0 / -1 (all-ones) result as expected for bool operations. -pub(crate) fn materialize_bool_result>( - ctx: &mut C, +pub(crate) fn materialize_bool_result( + ctx: &mut Lower, insn: IRInst, rd: Writable, cond: Cond, @@ -1524,10 +1495,9 @@ fn load_op_to_ty(op: Opcode) -> Option { /// Helper to lower a load instruction; this is used in several places, because /// a load can sometimes be merged into another operation. pub(crate) fn lower_load< - C: LowerCtx, - F: FnMut(&mut C, ValueRegs>, Type, AMode) -> CodegenResult<()>, + F: FnMut(&mut Lower, ValueRegs>, Type, AMode) -> CodegenResult<()>, >( - ctx: &mut C, + ctx: &mut Lower, ir_inst: IRInst, inputs: &[InsnInput], output: InsnOutput, @@ -1550,13 +1520,13 @@ pub(crate) fn lower_load< impl LowerBackend for AArch64Backend { type MInst = Inst; - fn lower>(&self, ctx: &mut C, ir_inst: IRInst) -> CodegenResult<()> { + fn lower(&self, ctx: &mut Lower, ir_inst: IRInst) -> CodegenResult<()> { lower_inst::lower_insn_to_regs(ctx, ir_inst, &self.triple, &self.flags, &self.isa_flags) } - fn lower_branch_group>( + fn lower_branch_group( &self, - ctx: &mut C, + ctx: &mut Lower, branches: &[IRInst], targets: &[MachLabel], ) -> CodegenResult<()> { diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index 504ce8ef04a8..49ea91e1081c 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -24,7 +24,7 @@ use crate::{ isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm}, isa::aarch64::lower::{writable_vreg, writable_xreg, xreg}, isa::unwind::UnwindInst, - machinst::{ty_bits, InsnOutput, LowerCtx, VCodeConstant, VCodeConstantData}, + machinst::{ty_bits, InsnOutput, Lower, VCodeConstant, VCodeConstantData}, }; use regalloc2::PReg; use std::boxed::Box; @@ -39,17 +39,14 @@ type BoxJTSequenceInfo = Box; type BoxExternalName = Box; /// The main entry point for lowering with ISLE. -pub(crate) fn lower( - lower_ctx: &mut C, +pub(crate) fn lower( + lower_ctx: &mut Lower, triple: &Triple, flags: &Flags, isa_flags: &IsaFlags, outputs: &[InsnOutput], inst: Inst, -) -> Result<(), ()> -where - C: LowerCtx, -{ +) -> Result<(), ()> { lower_common( lower_ctx, triple, @@ -71,10 +68,7 @@ pub struct SinkableAtomicLoad { atomic_addr: Value, } -impl generated_code::Context for IsleContext<'_, C, Flags, IsaFlags, 6> -where - C: LowerCtx, -{ +impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { isle_prelude_methods!(); fn use_lse(&mut self, _: Inst) -> Option<()> { diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 24d45d0c2706..4b841da050e5 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -19,8 +19,8 @@ use core::convert::TryFrom; use target_lexicon::Triple; /// Actually codegen an instruction's results into registers. -pub(crate) fn lower_insn_to_regs>( - ctx: &mut C, +pub(crate) fn lower_insn_to_regs( + ctx: &mut Lower, insn: IRInst, triple: &Triple, flags: &Flags, @@ -39,7 +39,7 @@ pub(crate) fn lower_insn_to_regs>( return Ok(()); } - let implemented_in_isle = |ctx: &mut C| -> ! { + let implemented_in_isle = |ctx: &mut Lower| -> ! { unreachable!( "implemented in ISLE: inst = `{}`, type = `{:?}`", ctx.dfg().display_inst(insn), @@ -1621,8 +1621,8 @@ pub(crate) fn lower_insn_to_regs>( Ok(()) } -pub(crate) fn lower_branch>( - ctx: &mut C, +pub(crate) fn lower_branch( + ctx: &mut Lower, branches: &[IRInst], targets: &[MachLabel], ) -> CodegenResult<()> { diff --git a/cranelift/codegen/src/isa/s390x/lower.rs b/cranelift/codegen/src/isa/s390x/lower.rs index c5567c5460fe..1894c366cfe4 100644 --- a/cranelift/codegen/src/isa/s390x/lower.rs +++ b/cranelift/codegen/src/isa/s390x/lower.rs @@ -4,7 +4,7 @@ use crate::ir::Inst as IRInst; use crate::ir::Opcode; use crate::isa::s390x::inst::Inst; use crate::isa::s390x::S390xBackend; -use crate::machinst::{InsnOutput, LowerBackend, LowerCtx, MachLabel}; +use crate::machinst::{InsnOutput, Lower, LowerBackend, MachLabel}; use crate::CodegenResult; use smallvec::SmallVec; @@ -16,7 +16,7 @@ pub mod isle; impl LowerBackend for S390xBackend { type MInst = Inst; - fn lower>(&self, ctx: &mut C, ir_inst: IRInst) -> CodegenResult<()> { + fn lower(&self, ctx: &mut Lower, ir_inst: IRInst) -> CodegenResult<()> { let op = ctx.data(ir_inst).opcode(); let outputs: SmallVec<[InsnOutput; 2]> = (0..ctx.num_outputs(ir_inst)) .map(|i| InsnOutput { @@ -278,9 +278,9 @@ impl LowerBackend for S390xBackend { } } - fn lower_branch_group>( + fn lower_branch_group( &self, - ctx: &mut C, + ctx: &mut Lower, branches: &[IRInst], targets: &[MachLabel], ) -> CodegenResult<()> { diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs index 99b6fbdce670..0ffba418d1ad 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs @@ -21,7 +21,7 @@ use crate::{ isa::unwind::UnwindInst, isa::CallConv, machinst::abi_impl::ABIMachineSpec, - machinst::{InsnOutput, LowerCtx, VCodeConstant, VCodeConstantData}, + machinst::{InsnOutput, Lower, VCodeConstant, VCodeConstantData}, }; use regalloc2::PReg; use smallvec::{smallvec, SmallVec}; @@ -46,17 +46,14 @@ type VecMInst = Vec; type VecMInstBuilder = Cell>; /// The main entry point for lowering with ISLE. -pub(crate) fn lower( - lower_ctx: &mut C, +pub(crate) fn lower( + lower_ctx: &mut Lower, triple: &Triple, flags: &Flags, isa_flags: &IsaFlags, outputs: &[InsnOutput], inst: Inst, -) -> Result<(), ()> -where - C: LowerCtx, -{ +) -> Result<(), ()> { lower_common( lower_ctx, triple, @@ -69,17 +66,14 @@ where } /// The main entry point for branch lowering with ISLE. -pub(crate) fn lower_branch( - lower_ctx: &mut C, +pub(crate) fn lower_branch( + lower_ctx: &mut Lower, triple: &Triple, flags: &Flags, isa_flags: &IsaFlags, branch: Inst, targets: &[MachLabel], -) -> Result<(), ()> -where - C: LowerCtx, -{ +) -> Result<(), ()> { lower_common( lower_ctx, triple, @@ -91,10 +85,7 @@ where ) } -impl generated_code::Context for IsleContext<'_, C, Flags, IsaFlags, 6> -where - C: LowerCtx, -{ +impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { isle_prelude_methods!(); fn abi_sig(&mut self, sig_ref: SigRef) -> ABISig { diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 382bb085548d..a80a2695175f 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -34,11 +34,7 @@ fn is_int_or_ref_ty(ty: Type) -> bool { /// Returns whether the given specified `input` is a result produced by an instruction with Opcode /// `op`. // TODO investigate failures with checking against the result index. -fn matches_input>( - ctx: &mut C, - input: InsnInput, - op: Opcode, -) -> Option { +fn matches_input(ctx: &mut Lower, input: InsnInput, op: Opcode) -> Option { let inputs = ctx.get_input_as_source_or_const(input.insn, input.input); inputs.inst.as_inst().and_then(|(src_inst, _)| { let data = ctx.data(src_inst); @@ -51,7 +47,7 @@ fn matches_input>( /// Emits instruction(s) to generate the given 64-bit constant value into a newly-allocated /// temporary register, returning that register. -fn generate_constant>(ctx: &mut C, ty: Type, c: u64) -> ValueRegs { +fn generate_constant(ctx: &mut Lower, ty: Type, c: u64) -> ValueRegs { let from_bits = ty_bits(ty); let masked = if from_bits < 64 { c & ((1u64 << from_bits) - 1) @@ -71,7 +67,7 @@ fn generate_constant>(ctx: &mut C, ty: Type, c: u64) -> Va } /// Put the given input into possibly multiple registers, and mark it as used (side-effect). -fn put_input_in_regs>(ctx: &mut C, spec: InsnInput) -> ValueRegs { +fn put_input_in_regs(ctx: &mut Lower, spec: InsnInput) -> ValueRegs { let ty = ctx.input_ty(spec.insn, spec.input); let input = ctx.get_input_as_source_or_const(spec.insn, spec.input); @@ -84,7 +80,7 @@ fn put_input_in_regs>(ctx: &mut C, spec: InsnInput) -> Val } /// Put the given input into a register, and mark it as used (side-effect). -fn put_input_in_reg>(ctx: &mut C, spec: InsnInput) -> Reg { +fn put_input_in_reg(ctx: &mut Lower, spec: InsnInput) -> Reg { put_input_in_regs(ctx, spec) .only_reg() .expect("Multi-register value not expected") @@ -94,10 +90,7 @@ fn put_input_in_reg>(ctx: &mut C, spec: InsnInput) -> Reg /// into the current lowering point. If so, returns the address-base source (as /// an `InsnInput`) and an offset from that address from which to perform the /// load. -fn is_mergeable_load>( - ctx: &mut C, - src_insn: IRInst, -) -> Option<(InsnInput, i32)> { +fn is_mergeable_load(ctx: &mut Lower, src_insn: IRInst) -> Option<(InsnInput, i32)> { let insn_data = ctx.data(src_insn); let inputs = ctx.num_inputs(src_insn); if inputs != 1 { @@ -142,7 +135,7 @@ fn is_mergeable_load>( /// Put the given input into a register or a memory operand. /// Effectful: may mark the given input as used, when returning the register form. -fn input_to_reg_mem>(ctx: &mut C, spec: InsnInput) -> RegMem { +fn input_to_reg_mem(ctx: &mut Lower, spec: InsnInput) -> RegMem { let inputs = ctx.get_input_as_source_or_const(spec.insn, spec.input); if let Some(c) = inputs.constant { @@ -166,19 +159,13 @@ fn input_to_reg_mem>(ctx: &mut C, spec: InsnInput) -> RegM ) } -fn input_to_imm>(ctx: &mut C, spec: InsnInput) -> Option { +fn input_to_imm(ctx: &mut Lower, spec: InsnInput) -> Option { ctx.get_input_as_source_or_const(spec.insn, spec.input) .constant } /// Emit an instruction to insert a value `src` into a lane of `dst`. -fn emit_insert_lane>( - ctx: &mut C, - src: RegMem, - dst: Writable, - lane: u8, - ty: Type, -) { +fn emit_insert_lane(ctx: &mut Lower, src: RegMem, dst: Writable, lane: u8, ty: Type) { if !ty.is_float() { let (sse_op, size) = match ty.lane_bits() { 8 => (SseOpcode::Pinsrb, OperandSize::Size32), @@ -220,13 +207,7 @@ fn emit_insert_lane>( } /// Emit an instruction to extract a lane of `src` into `dst`. -fn emit_extract_lane>( - ctx: &mut C, - src: Reg, - dst: Writable, - lane: u8, - ty: Type, -) { +fn emit_extract_lane(ctx: &mut Lower, src: Reg, dst: Writable, lane: u8, ty: Type) { if !ty.is_float() { let (sse_op, size) = match ty.lane_bits() { 8 => (SseOpcode::Pextrb, OperandSize::Size32), @@ -277,8 +258,8 @@ fn emit_extract_lane>( } } -fn emit_vm_call>( - ctx: &mut C, +fn emit_vm_call( + ctx: &mut Lower, flags: &Flags, triple: &Triple, libcall: LibCall, @@ -319,10 +300,7 @@ fn emit_vm_call>( /// Returns whether the given input is a shift by a constant value less or equal than 3. /// The goal is to embed it within an address mode. -fn matches_small_constant_shift>( - ctx: &mut C, - spec: InsnInput, -) -> Option<(InsnInput, u8)> { +fn matches_small_constant_shift(ctx: &mut Lower, spec: InsnInput) -> Option<(InsnInput, u8)> { matches_input(ctx, spec, Opcode::Ishl).and_then(|shift| { match input_to_imm( ctx, @@ -346,7 +324,7 @@ fn matches_small_constant_shift>( /// Lowers an instruction to one of the x86 addressing modes. /// /// Note: the 32-bit offset in Cranelift has to be sign-extended, which maps x86's behavior. -fn lower_to_amode>(ctx: &mut C, spec: InsnInput, offset: i32) -> Amode { +fn lower_to_amode(ctx: &mut Lower, spec: InsnInput, offset: i32) -> Amode { let flags = ctx .memflags(spec.insn) .expect("Instruction with amode should have memflags"); @@ -443,8 +421,8 @@ fn lower_to_amode>(ctx: &mut C, spec: InsnInput, offset: i // Top-level instruction lowering entry point, for one instruction. /// Actually codegen an instruction's results into registers. -fn lower_insn_to_regs>( - ctx: &mut C, +fn lower_insn_to_regs( + ctx: &mut Lower, insn: IRInst, flags: &Flags, isa_flags: &x64_settings::Flags, @@ -469,7 +447,7 @@ fn lower_insn_to_regs>( return Ok(()); } - let implemented_in_isle = |ctx: &mut C| { + let implemented_in_isle = |ctx: &mut Lower| { unreachable!( "implemented in ISLE: inst = `{}`, type = `{:?}`", ctx.dfg().display_inst(insn), @@ -2227,13 +2205,13 @@ fn lower_insn_to_regs>( impl LowerBackend for X64Backend { type MInst = Inst; - fn lower>(&self, ctx: &mut C, ir_inst: IRInst) -> CodegenResult<()> { + fn lower(&self, ctx: &mut Lower, ir_inst: IRInst) -> CodegenResult<()> { lower_insn_to_regs(ctx, ir_inst, &self.flags, &self.x64_flags, &self.triple) } - fn lower_branch_group>( + fn lower_branch_group( &self, - ctx: &mut C, + ctx: &mut Lower, branches: &[IRInst], targets: &[MachLabel], ) -> CodegenResult<()> { diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 8fffd3857ffe..f39c8e1aaf84 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -30,7 +30,7 @@ use crate::{ }, }, machinst::{ - isle::*, valueregs, ABICaller, InsnInput, InsnOutput, LowerCtx, MachAtomicRmwOp, MachInst, + isle::*, valueregs, ABICaller, InsnInput, InsnOutput, Lower, MachAtomicRmwOp, MachInst, VCodeConstant, VCodeConstantData, }, }; @@ -51,17 +51,14 @@ pub struct SinkableLoad { } /// The main entry point for lowering with ISLE. -pub(crate) fn lower( - lower_ctx: &mut C, +pub(crate) fn lower( + lower_ctx: &mut Lower, triple: &Triple, flags: &Flags, isa_flags: &IsaFlags, outputs: &[InsnOutput], inst: Inst, -) -> Result<(), ()> -where - C: LowerCtx, -{ +) -> Result<(), ()> { lower_common( lower_ctx, triple, @@ -73,17 +70,14 @@ where ) } -pub(crate) fn lower_branch( - lower_ctx: &mut C, +pub(crate) fn lower_branch( + lower_ctx: &mut Lower, triple: &Triple, flags: &Flags, isa_flags: &IsaFlags, branch: Inst, targets: &[MachLabel], -) -> Result<(), ()> -where - C: LowerCtx, -{ +) -> Result<(), ()> { lower_common( lower_ctx, triple, @@ -95,10 +89,7 @@ where ) } -impl Context for IsleContext<'_, C, Flags, IsaFlags, 6> -where - C: LowerCtx, -{ +impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { isle_prelude_methods!(); #[inline] @@ -784,10 +775,7 @@ where } } -impl IsleContext<'_, C, Flags, IsaFlags, 6> -where - C: LowerCtx, -{ +impl IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { fn abi_arg_slot_regs(&mut self, arg: &ABIArg) -> Option { match arg { &ABIArg::Slots { ref slots, .. } => match slots.len() { diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs index dd4b95d2bddd..0fb8ac851143 100644 --- a/cranelift/codegen/src/machinst/abi.rs +++ b/cranelift/codegen/src/machinst/abi.rs @@ -194,45 +194,40 @@ pub trait ABICaller { /// Emit a copy of an argument value from a source register, prior to the call. /// For large arguments with associated stack buffer, this may load the address /// of the buffer into the argument register, if required by the ABI. - fn emit_copy_regs_to_arg>( - &self, - ctx: &mut C, - idx: usize, - from_reg: ValueRegs, - ); + fn emit_copy_regs_to_arg(&self, ctx: &mut Lower, idx: usize, from_reg: ValueRegs); /// Emit a copy of a large argument into its associated stack buffer, if any. /// We must be careful to perform all these copies (as necessary) before setting /// up the argument registers, since we may have to invoke memcpy(), which could /// clobber any registers already set up. The back-end should call this routine /// for all arguments before calling emit_copy_regs_to_arg for all arguments. - fn emit_copy_regs_to_buffer>( + fn emit_copy_regs_to_buffer( &self, - ctx: &mut C, + ctx: &mut Lower, idx: usize, from_reg: ValueRegs, ); /// Emit a copy a return value into a destination register, after the call returns. - fn emit_copy_retval_to_regs>( + fn emit_copy_retval_to_regs( &self, - ctx: &mut C, + ctx: &mut Lower, idx: usize, into_reg: ValueRegs>, ); /// Emit code to pre-adjust the stack, prior to argument copies and call. - fn emit_stack_pre_adjust>(&self, ctx: &mut C); + fn emit_stack_pre_adjust(&self, ctx: &mut Lower); /// Emit code to post-adjust the satck, after call return and return-value copies. - fn emit_stack_post_adjust>(&self, ctx: &mut C); + fn emit_stack_post_adjust(&self, ctx: &mut Lower); /// Accumulate outgoing arguments. This ensures that the caller (as /// identified via the CTX argument) allocates enough space in the /// prologue to hold all arguments and return values for this call. /// There is no code emitted at the call site, everything is done /// in the caller's function prologue. - fn accumulate_outgoing_args_size>(&self, ctx: &mut C); + fn accumulate_outgoing_args_size(&self, ctx: &mut Lower); /// Emit the call itself. /// @@ -247,5 +242,5 @@ pub trait ABICaller { /// /// This function should only be called once, as it is allowed to re-use /// parts of the ABICaller object in emitting instructions. - fn emit_call>(&mut self, ctx: &mut C); + fn emit_call(&mut self, ctx: &mut Lower); } diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs index 6b8dabbab8ab..d248daab3128 100644 --- a/cranelift/codegen/src/machinst/abi_impl.rs +++ b/cranelift/codegen/src/machinst/abi_impl.rs @@ -1681,11 +1681,7 @@ impl ABICallerImpl { } } -fn adjust_stack_and_nominal_sp>( - ctx: &mut C, - off: i32, - is_sub: bool, -) { +fn adjust_stack_and_nominal_sp(ctx: &mut Lower, off: i32, is_sub: bool) { if off == 0 { return; } @@ -1707,24 +1703,24 @@ impl ABICaller for ABICallerImpl { } } - fn accumulate_outgoing_args_size>(&self, ctx: &mut C) { + fn accumulate_outgoing_args_size(&self, ctx: &mut Lower) { let off = self.sig.sized_stack_arg_space + self.sig.sized_stack_ret_space; ctx.abi().accumulate_outgoing_args_size(off as u32); } - fn emit_stack_pre_adjust>(&self, ctx: &mut C) { + fn emit_stack_pre_adjust(&self, ctx: &mut Lower) { let off = self.sig.sized_stack_arg_space + self.sig.sized_stack_ret_space; - adjust_stack_and_nominal_sp::(ctx, off as i32, /* is_sub = */ true) + adjust_stack_and_nominal_sp::(ctx, off as i32, /* is_sub = */ true) } - fn emit_stack_post_adjust>(&self, ctx: &mut C) { + fn emit_stack_post_adjust(&self, ctx: &mut Lower) { let off = self.sig.sized_stack_arg_space + self.sig.sized_stack_ret_space; - adjust_stack_and_nominal_sp::(ctx, off as i32, /* is_sub = */ false) + adjust_stack_and_nominal_sp::(ctx, off as i32, /* is_sub = */ false) } - fn emit_copy_regs_to_buffer>( + fn emit_copy_regs_to_buffer( &self, - ctx: &mut C, + ctx: &mut Lower, idx: usize, from_regs: ValueRegs, ) { @@ -1754,9 +1750,9 @@ impl ABICaller for ABICallerImpl { } } - fn emit_copy_regs_to_arg>( + fn emit_copy_regs_to_arg( &self, - ctx: &mut C, + ctx: &mut Lower, idx: usize, from_regs: ValueRegs, ) { @@ -1837,9 +1833,9 @@ impl ABICaller for ABICallerImpl { } } - fn emit_copy_retval_to_regs>( + fn emit_copy_retval_to_regs( &self, - ctx: &mut C, + ctx: &mut Lower, idx: usize, into_regs: ValueRegs>, ) { @@ -1873,7 +1869,7 @@ impl ABICaller for ABICallerImpl { } } - fn emit_call>(&mut self, ctx: &mut C) { + fn emit_call(&mut self, ctx: &mut Lower) { let (uses, defs) = ( mem::replace(&mut self.uses, Default::default()), mem::replace(&mut self.defs, Default::default()), diff --git a/cranelift/codegen/src/machinst/helpers.rs b/cranelift/codegen/src/machinst/helpers.rs index 5fb1484cfc74..8d4e4e23f07a 100644 --- a/cranelift/codegen/src/machinst/helpers.rs +++ b/cranelift/codegen/src/machinst/helpers.rs @@ -1,6 +1,6 @@ //! Miscellaneous helpers for machine backends. -use super::{InsnOutput, LowerCtx, VCodeInst, ValueRegs}; +use super::{InsnOutput, Lower, VCodeInst, ValueRegs}; use super::{Reg, Writable}; use crate::ir::Type; use std::ops::{Add, BitAnd, Not, Sub}; @@ -21,8 +21,8 @@ pub(crate) fn ty_has_float_or_vec_representation(ty: Type) -> bool { } /// Allocate a register for an instruction output and return it. -pub(crate) fn get_output_reg>( - ctx: &mut C, +pub(crate) fn get_output_reg( + ctx: &mut Lower, spec: InsnOutput, ) -> ValueRegs> { ctx.get_output(spec.insn, spec.output) diff --git a/cranelift/codegen/src/machinst/inst_common.rs b/cranelift/codegen/src/machinst/inst_common.rs index 740a0346cc9b..0f13a912fd74 100644 --- a/cranelift/codegen/src/machinst/inst_common.rs +++ b/cranelift/codegen/src/machinst/inst_common.rs @@ -1,6 +1,6 @@ //! A place to park MachInst::Inst fragments which are common across multiple architectures. -use super::{LowerCtx, VCodeInst}; +use super::{Lower, VCodeInst}; use crate::ir::{self, Inst as IRInst}; use smallvec::SmallVec; @@ -24,17 +24,14 @@ pub(crate) struct InsnOutput { pub(crate) output: usize, } -pub(crate) fn insn_inputs>( - ctx: &C, - insn: IRInst, -) -> SmallVec<[InsnInput; 4]> { +pub(crate) fn insn_inputs(ctx: &Lower, insn: IRInst) -> SmallVec<[InsnInput; 4]> { (0..ctx.num_inputs(insn)) .map(|i| InsnInput { insn, input: i }) .collect() } -pub(crate) fn insn_outputs>( - ctx: &C, +pub(crate) fn insn_outputs( + ctx: &Lower, insn: IRInst, ) -> SmallVec<[InsnOutput; 4]> { (0..ctx.num_outputs(insn)) diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index d137a0206120..67e8df831c4b 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -1,5 +1,5 @@ use crate::ir::{types, Inst, Value, ValueList}; -use crate::machinst::{get_output_reg, InsnOutput, LowerCtx}; +use crate::machinst::{get_output_reg, InsnOutput}; use alloc::boxed::Box; use alloc::vec::Vec; use smallvec::SmallVec; @@ -13,7 +13,8 @@ pub use crate::ir::{ }; pub use crate::isa::unwind::UnwindInst; pub use crate::machinst::{ - ABIArg, ABIArgSlot, ABISig, InputSourceInst, RealReg, Reg, RelocDistance, Writable, + ABIArg, ABIArgSlot, ABISig, InputSourceInst, Lower, RealReg, Reg, RelocDistance, VCodeInst, + Writable, }; pub use crate::settings::TlsModel; @@ -948,14 +949,15 @@ macro_rules! isle_prelude_methods { /// This structure is used to implement the ISLE-generated `Context` trait and /// internally has a temporary reference to a machinst `LowerCtx`. -pub(crate) struct IsleContext<'a, C: LowerCtx, F, I, const N: usize> +pub(crate) struct IsleContext<'a, 'b, I, Flags, IsaFlags, const N: usize> where - [(C::I, bool); N]: smallvec::Array, + I: VCodeInst, + [(I, bool); N]: smallvec::Array, { - pub lower_ctx: &'a mut C, + pub lower_ctx: &'a mut Lower<'b, I>, pub triple: &'a Triple, - pub flags: &'a F, - pub isa_flags: &'a I, + pub flags: &'a Flags, + pub isa_flags: &'a IsaFlags, } /// Shared lowering code amongst all backends for doing ISLE-based lowering. @@ -963,19 +965,19 @@ where /// The `isle_lower` argument here is an ISLE-generated function for `lower` and /// then this function otherwise handles register mapping and such around the /// lowering. -pub(crate) fn lower_common( - lower_ctx: &mut C, +pub(crate) fn lower_common( + lower_ctx: &mut Lower, triple: &Triple, - flags: &F, - isa_flags: &I, + flags: &Flags, + isa_flags: &IsaFlags, outputs: &[InsnOutput], inst: Inst, - isle_lower: IF, + isle_lower: IsleFunction, ) -> Result<(), ()> where - C: LowerCtx, - [(C::I, bool); N]: smallvec::Array, - IF: Fn(&mut IsleContext<'_, C, F, I, N>, Inst) -> Option, + I: VCodeInst, + [(I, bool); N]: smallvec::Array, + IsleFunction: Fn(&mut IsleContext<'_, '_, I, Flags, IsaFlags, N>, Inst) -> Option, { // TODO: reuse the ISLE context across lowerings so we can reuse its // internal heap allocations. diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index 02eba77bcfd3..eab008925f5f 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -2,8 +2,8 @@ //! to machine instructions with virtual registers. This is *almost* the final //! machine code, except for register allocation. -// TODO: separate the IR-query core of `LowerCtx` from the lowering logic built -// on top of it, e.g. the side-effect/coloring analysis and the scan support. +// TODO: separate the IR-query core of `Lower` from the lowering logic built on +// top of it, e.g. the side-effect/coloring analysis and the scan support. use crate::data_value::DataValue; use crate::entity::SecondaryMap; @@ -57,141 +57,11 @@ impl InstColor { } } -/// A context that machine-specific lowering code can use to emit lowered -/// instructions. This is the view of the machine-independent per-function -/// lowering context that is seen by the machine backend. -pub trait LowerCtx { - /// The instruction type for which this lowering framework is instantiated. - type I: VCodeInst; - - fn dfg(&self) -> &DataFlowGraph; - - // Function-level queries: - - /// Get the `ABICallee`. - fn abi(&mut self) -> &mut dyn ABICallee; - /// Get the (virtual) register that receives the return value. A return - /// instruction should lower into a sequence that fills this register. (Why - /// not allow the backend to specify its own result register for the return? - /// Because there may be multiple return points.) - fn retval(&self, idx: usize) -> ValueRegs>; - /// Returns the vreg containing the VmContext parameter, if there's one. - fn get_vm_context(&self) -> Option; - - // General instruction queries: - - /// Get the instdata for a given IR instruction. - fn data(&self, ir_inst: Inst) -> &InstructionData; - /// Get the controlling type for a polymorphic IR instruction. - fn ty(&self, ir_inst: Inst) -> Type; - /// Get the target for a call instruction, as an `ExternalName`. Returns a tuple - /// providing this name and the "relocation distance", i.e., whether the backend - /// can assume the target will be "nearby" (within some small offset) or an - /// arbitrary address. (This comes from the `colocated` bit in the CLIF.) - fn call_target<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance)>; - /// Get the signature for a call or call-indirect instruction. - fn call_sig<'b>(&'b self, ir_inst: Inst) -> Option<&'b Signature>; - /// Get the symbol name, relocation distance estimate, and offset for a - /// symbol_value instruction. - fn symbol_value<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance, i64)>; - /// Likewise, but starting with a GlobalValue identifier. - fn symbol_value_data<'b>( - &'b self, - global_value: GlobalValue, - ) -> Option<(&'b ExternalName, RelocDistance, i64)>; - /// Returns the memory flags of a given memory access. - fn memflags(&self, ir_inst: Inst) -> Option; - /// Get the source location for a given instruction. - fn srcloc(&self, ir_inst: Inst) -> SourceLoc; - - // Instruction input/output queries: - - /// Get the number of inputs to the given IR instruction. - fn num_inputs(&self, ir_inst: Inst) -> usize; - /// Get the number of outputs to the given IR instruction. - fn num_outputs(&self, ir_inst: Inst) -> usize; - /// Get the type for an instruction's input. - fn input_ty(&self, ir_inst: Inst, idx: usize) -> Type; - /// Get the type for a value. - fn value_ty(&self, val: Value) -> Type; - /// Get the type for an instruction's output. - fn output_ty(&self, ir_inst: Inst, idx: usize) -> Type; - /// Get the value of a constant instruction (`iconst`, etc.) as a 64-bit - /// value, if possible. - fn get_constant(&self, ir_inst: Inst) -> Option; - /// Get the input as one of two options other than a direct register: - /// - /// - An instruction, given that it is effect-free or able to sink its - /// effect to the current instruction being lowered, and given it has only - /// one output, and if effect-ful, given that this is the only use; - /// - A constant, if the value is a constant. - /// - /// The instruction input may be available in either of these forms. It may - /// be available in neither form, if the conditions are not met; if so, use - /// `put_input_in_regs()` instead to get it in a register. - /// - /// If the backend merges the effect of a side-effecting instruction, it - /// must call `sink_inst()`. When this is called, it indicates that the - /// effect has been sunk to the current scan location. The sunk - /// instruction's result(s) must have *no* uses remaining, because it will - /// not be codegen'd (it has been integrated into the current instruction). - fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput; - /// Like `get_input_as_source_or_const` but with a `Value`. - fn get_value_as_source_or_const(&self, value: Value) -> NonRegInput; - /// Resolves a particular input of an instruction to the `Value` that it is - /// represented with. - fn input_as_value(&self, ir_inst: Inst, idx: usize) -> Value; - /// Increment the reference count for the Value, ensuring that it gets lowered. - fn increment_lowered_uses(&mut self, val: Value); - /// Put the `idx`th input into register(s) and return the assigned register. - fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs; - /// Put the given value into register(s) and return the assigned register. - fn put_value_in_regs(&mut self, value: Value) -> ValueRegs; - /// Get the `idx`th output register(s) of the given IR instruction. When - /// `backend.lower_inst_to_regs(ctx, inst)` is called, it is expected that - /// the backend will write results to these output register(s). This - /// register will always be "fresh"; it is guaranteed not to overlap with - /// any of the inputs, and can be freely used as a scratch register within - /// the lowered instruction sequence, as long as its final value is the - /// result of the computation. - fn get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs>; - - // Codegen primitives: allocate temps, emit instructions, set result registers, - // ask for an input to be gen'd into a register. - - /// Get a new temp. - fn alloc_tmp(&mut self, ty: Type) -> ValueRegs>; - /// Emit a machine instruction. - fn emit(&mut self, mach_inst: Self::I); - /// Indicate that the side-effect of an instruction has been sunk to the - /// current scan location. This should only be done with the instruction's - /// original results are not used (i.e., `put_input_in_regs` is not invoked - /// for the input produced by the sunk instruction), otherwise the - /// side-effect will occur twice. - fn sink_inst(&mut self, ir_inst: Inst); - /// Retrieve immediate data given a handle. - fn get_immediate_data(&self, imm: Immediate) -> &ConstantData; - /// Retrieve constant data given a handle. - fn get_constant_data(&self, constant_handle: Constant) -> &ConstantData; - /// Indicate that a constant should be emitted. - fn use_constant(&mut self, constant: VCodeConstantData) -> VCodeConstant; - /// Retrieve the value immediate from an instruction. This will perform necessary lookups on the - /// `DataFlowGraph` to retrieve even large immediates. - fn get_immediate(&self, ir_inst: Inst) -> Option; - /// Cause the value in `reg` to be in a virtual reg, by copying it into a new virtual reg - /// if `reg` is a real reg. `ty` describes the type of the value in `reg`. - fn ensure_in_vreg(&mut self, reg: Reg, ty: Type) -> Reg; - - /// Note that one vreg is to be treated as an alias of another. - fn set_vreg_alias(&mut self, from: Reg, to: Reg); -} - /// A representation of all of the ways in which a value is available, aside /// from as a direct register. /// /// - An instruction, if it would be allowed to occur at the current location -/// instead (see [LowerCtx::get_input_as_source_or_const()] for more -/// details). +/// instead (see [Lower::get_input_as_source_or_const()] for more details). /// /// - A constant, if the value is known to be a constant. #[derive(Clone, Copy, Debug)] @@ -200,8 +70,8 @@ pub struct NonRegInput { /// computation (and side-effect if applicable) could occur at the /// current instruction's location instead. /// - /// If this instruction's operation is merged into the current - /// instruction, the backend must call [LowerCtx::sink_inst()]. + /// If this instruction's operation is merged into the current instruction, + /// the backend must call [Lower::sink_inst()]. /// /// This enum indicates whether this use of the source instruction /// is unique or not. @@ -255,13 +125,13 @@ pub trait LowerBackend { /// edge (block-param actuals) into registers, because the actual branch /// generation (`lower_branch_group()`) happens *after* any possible merged /// out-edge. - fn lower>(&self, ctx: &mut C, inst: Inst) -> CodegenResult<()>; + fn lower(&self, ctx: &mut Lower, inst: Inst) -> CodegenResult<()>; /// Lower a block-terminating group of branches (which together can be seen /// as one N-way branch), given a vcode MachLabel for each target. - fn lower_branch_group>( + fn lower_branch_group( &self, - ctx: &mut C, + ctx: &mut Lower, insts: &[Inst], targets: &[MachLabel], ) -> CodegenResult<()>; @@ -333,10 +203,6 @@ pub struct Lower<'func, I: VCodeInst> { /// The register to use for GetPinnedReg, if any, on this architecture. pinned_reg: Option, - - /// The vreg containing the special VmContext parameter, if it is present in the current - /// function's signature. - vm_context: Option, } /// How is a value used in the IR? @@ -374,12 +240,11 @@ pub struct Lower<'func, I: VCodeInst> { /// can only get a `&T` (one can only get a "I am one of several users /// of this instruction" result). /// -/// We could track these paths, either dynamically as one "looks up -/// the operand tree" or precomputed. But the former requires state -/// and means that the `LowerCtx` API carries that state implicitly, -/// which we'd like to avoid if we can. And the latter implies O(n^2) -/// storage: it is an all-pairs property (is inst `i` unique from the -/// point of view of `j`). +/// We could track these paths, either dynamically as one "looks up the operand +/// tree" or precomputed. But the former requires state and means that the +/// `Lower` API carries that state implicitly, which we'd like to avoid if we +/// can. And the latter implies O(n^2) storage: it is an all-pairs property (is +/// inst `i` unique from the point of view of `j`). /// /// To make matters even a little more complex still, a value that is /// not uniquely used when initially viewing the IR can *become* @@ -525,16 +390,6 @@ impl<'func, I: VCodeInst> Lower<'func, I> { } } - let vm_context = vcode - .abi() - .signature() - .special_param_index(ArgumentPurpose::VMContext) - .map(|vm_context_index| { - let entry_block = f.layout.entry_block().unwrap(); - let param = f.dfg.block_params(entry_block)[vm_context_index]; - value_regs[param].only_reg().unwrap() - }); - // Assign vreg(s) to each return value. let mut retval_regs = vec![]; for ret in &vcode.abi().signature().returns.clone() { @@ -589,7 +444,6 @@ impl<'func, I: VCodeInst> Lower<'func, I> { cur_inst: None, ir_insts: vec![], pinned_reg: None, - vm_context, }) } @@ -801,12 +655,11 @@ impl<'func, I: VCodeInst> Lower<'func, I> { // possible trap), or if used outside of this block, or if // demanded by another inst, then lower. // - // That's it! Lowering of side-effecting ops will force all - // *needed* (live) non-side-effecting ops to be lowered at the - // right places, via the `use_input_reg()` callback on the - // `LowerCtx` (that's us). That's because `use_input_reg()` - // sets the eager/demand bit for any insts whose result - // registers are used. + // That's it! Lowering of side-effecting ops will force all *needed* + // (live) non-side-effecting ops to be lowered at the right places, via + // the `use_input_reg()` callback on the `Lower` (that's us). That's + // because `use_input_reg()` sets the eager/demand bit for any insts + // whose result registers are used. // // We set the VCodeBuilder to "backward" mode, so we emit // blocks in reverse order wrt the BlockIndex sequence, and @@ -1148,34 +1001,38 @@ impl<'func, I: VCodeInst> Lower<'func, I> { } } -impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { - type I = I; - - fn dfg(&self) -> &DataFlowGraph { +/// Function-level queries. +impl<'func, I: VCodeInst> Lower<'func, I> { + pub fn dfg(&self) -> &DataFlowGraph { &self.f.dfg } - fn abi(&mut self) -> &mut dyn ABICallee { + /// Get the `ABICallee`. + pub fn abi(&mut self) -> &mut dyn ABICallee { self.vcode.abi() } - fn retval(&self, idx: usize) -> ValueRegs> { + /// Get the (virtual) register that receives the return value. A return + /// instruction should lower into a sequence that fills this register. (Why + /// not allow the backend to specify its own result register for the return? + /// Because there may be multiple return points.) + pub fn retval(&self, idx: usize) -> ValueRegs> { writable_value_regs(self.retval_regs[idx]) } +} - fn get_vm_context(&self) -> Option { - self.vm_context - } - - fn data(&self, ir_inst: Inst) -> &InstructionData { +/// Instruction input/output queries. +impl<'func, I: VCodeInst> Lower<'func, I> { + /// Get the instdata for a given IR instruction. + pub fn data(&self, ir_inst: Inst) -> &InstructionData { &self.f.dfg[ir_inst] } - fn ty(&self, ir_inst: Inst) -> Type { - self.f.dfg.ctrl_typevar(ir_inst) - } - - fn call_target<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance)> { + /// Get the target for a call instruction, as an `ExternalName`. Returns a tuple + /// providing this name and the "relocation distance", i.e., whether the backend + /// can assume the target will be "nearby" (within some small offset) or an + /// arbitrary address. (This comes from the `colocated` bit in the CLIF.) + pub fn call_target<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance)> { match &self.f.dfg[ir_inst] { &InstructionData::Call { func_ref, .. } | &InstructionData::FuncAddr { func_ref, .. } => { @@ -1187,7 +1044,8 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { } } - fn call_sig<'b>(&'b self, ir_inst: Inst) -> Option<&'b Signature> { + /// Get the signature for a call or call-indirect instruction. + pub fn call_sig<'b>(&'b self, ir_inst: Inst) -> Option<&'b Signature> { match &self.f.dfg[ir_inst] { &InstructionData::Call { func_ref, .. } => { let funcdata = &self.f.dfg.ext_funcs[func_ref]; @@ -1198,7 +1056,12 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { } } - fn symbol_value<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance, i64)> { + /// Get the symbol name, relocation distance estimate, and offset for a + /// symbol_value instruction. + pub fn symbol_value<'b>( + &'b self, + ir_inst: Inst, + ) -> Option<(&'b ExternalName, RelocDistance, i64)> { match &self.f.dfg[ir_inst] { &InstructionData::UnaryGlobalValue { global_value, .. } => { self.symbol_value_data(global_value) @@ -1207,7 +1070,8 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { } } - fn symbol_value_data<'b>( + /// Likewise, but starting with a GlobalValue identifier. + pub fn symbol_value_data<'b>( &'b self, global_value: GlobalValue, ) -> Option<(&'b ExternalName, RelocDistance, i64)> { @@ -1226,7 +1090,8 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { } } - fn memflags(&self, ir_inst: Inst) -> Option { + /// Returns the memory flags of a given memory access. + pub fn memflags(&self, ir_inst: Inst) -> Option { match &self.f.dfg[ir_inst] { &InstructionData::AtomicCas { flags, .. } => Some(flags), &InstructionData::AtomicRmw { flags, .. } => Some(flags), @@ -1238,45 +1103,72 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { } } - fn srcloc(&self, ir_inst: Inst) -> SourceLoc { + /// Get the source location for a given instruction. + pub fn srcloc(&self, ir_inst: Inst) -> SourceLoc { self.f.srclocs[ir_inst] } - fn num_inputs(&self, ir_inst: Inst) -> usize { + /// Get the number of inputs to the given IR instruction. + pub fn num_inputs(&self, ir_inst: Inst) -> usize { self.f.dfg.inst_args(ir_inst).len() } - fn num_outputs(&self, ir_inst: Inst) -> usize { + /// Get the number of outputs to the given IR instruction. + pub fn num_outputs(&self, ir_inst: Inst) -> usize { self.f.dfg.inst_results(ir_inst).len() } - fn input_ty(&self, ir_inst: Inst, idx: usize) -> Type { + /// Get the type for an instruction's input. + pub fn input_ty(&self, ir_inst: Inst, idx: usize) -> Type { self.value_ty(self.input_as_value(ir_inst, idx)) } - fn value_ty(&self, val: Value) -> Type { + /// Get the type for a value. + pub fn value_ty(&self, val: Value) -> Type { self.f.dfg.value_type(val) } - fn output_ty(&self, ir_inst: Inst, idx: usize) -> Type { + /// Get the type for an instruction's output. + pub fn output_ty(&self, ir_inst: Inst, idx: usize) -> Type { self.f.dfg.value_type(self.f.dfg.inst_results(ir_inst)[idx]) } - fn get_constant(&self, ir_inst: Inst) -> Option { + /// Get the value of a constant instruction (`iconst`, etc.) as a 64-bit + /// value, if possible. + pub fn get_constant(&self, ir_inst: Inst) -> Option { self.inst_constants.get(&ir_inst).cloned() } - fn input_as_value(&self, ir_inst: Inst, idx: usize) -> Value { + /// Get the input as one of two options other than a direct register: + /// + /// - An instruction, given that it is effect-free or able to sink its + /// effect to the current instruction being lowered, and given it has only + /// one output, and if effect-ful, given that this is the only use; + /// - A constant, if the value is a constant. + /// + /// The instruction input may be available in either of these forms. It may + /// be available in neither form, if the conditions are not met; if so, use + /// `put_input_in_regs()` instead to get it in a register. + /// + /// If the backend merges the effect of a side-effecting instruction, it + /// must call `sink_inst()`. When this is called, it indicates that the + /// effect has been sunk to the current scan location. The sunk + /// instruction's result(s) must have *no* uses remaining, because it will + /// not be codegen'd (it has been integrated into the current instruction). + pub fn input_as_value(&self, ir_inst: Inst, idx: usize) -> Value { let val = self.f.dfg.inst_args(ir_inst)[idx]; self.f.dfg.resolve_aliases(val) } - fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput { + /// Like `get_input_as_source_or_const` but with a `Value`. + pub fn get_input_as_source_or_const(&self, ir_inst: Inst, idx: usize) -> NonRegInput { let val = self.input_as_value(ir_inst, idx); self.get_value_as_source_or_const(val) } - fn get_value_as_source_or_const(&self, val: Value) -> NonRegInput { + /// Resolves a particular input of an instruction to the `Value` that it is + /// represented with. + pub fn get_value_as_source_or_const(&self, val: Value) -> NonRegInput { trace!( "get_input_for_val: val {} at cur_inst {:?} cur_scan_entry_color {:?}", val, @@ -1352,16 +1244,19 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { NonRegInput { inst, constant } } - fn increment_lowered_uses(&mut self, val: Value) { + /// Increment the reference count for the Value, ensuring that it gets lowered. + pub fn increment_lowered_uses(&mut self, val: Value) { self.value_lowered_uses[val] += 1 } - fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs { + /// Put the `idx`th input into register(s) and return the assigned register. + pub fn put_input_in_regs(&mut self, ir_inst: Inst, idx: usize) -> ValueRegs { let val = self.f.dfg.inst_args(ir_inst)[idx]; self.put_value_in_regs(val) } - fn put_value_in_regs(&mut self, val: Value) -> ValueRegs { + /// Put the given value into register(s) and return the assigned register. + pub fn put_value_in_regs(&mut self, val: Value) -> ValueRegs { let val = self.f.dfg.resolve_aliases(val); trace!("put_value_in_regs: val {}", val); @@ -1415,21 +1310,40 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { regs } - fn get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs> { + /// Get the `idx`th output register(s) of the given IR instruction. + /// + /// When `backend.lower_inst_to_regs(ctx, inst)` is called, it is expected + /// that the backend will write results to these output register(s). This + /// register will always be "fresh"; it is guaranteed not to overlap with + /// any of the inputs, and can be freely used as a scratch register within + /// the lowered instruction sequence, as long as its final value is the + /// result of the computation. + pub fn get_output(&self, ir_inst: Inst, idx: usize) -> ValueRegs> { let val = self.f.dfg.inst_results(ir_inst)[idx]; writable_value_regs(self.value_regs[val]) } +} - fn alloc_tmp(&mut self, ty: Type) -> ValueRegs> { +/// Codegen primitives: allocate temps, emit instructions, set result registers, +/// ask for an input to be gen'd into a register. +impl<'func, I: VCodeInst> Lower<'func, I> { + /// Get a new temp. + pub fn alloc_tmp(&mut self, ty: Type) -> ValueRegs> { writable_value_regs(alloc_vregs(ty, &mut self.next_vreg, &mut self.vcode).unwrap()) } - fn emit(&mut self, mach_inst: I) { + /// Emit a machine instruction. + pub fn emit(&mut self, mach_inst: I) { trace!("emit: {:?}", mach_inst); self.ir_insts.push(mach_inst); } - fn sink_inst(&mut self, ir_inst: Inst) { + /// Indicate that the side-effect of an instruction has been sunk to the + /// current scan location. This should only be done with the instruction's + /// original results are not used (i.e., `put_input_in_regs` is not invoked + /// for the input produced by the sunk instruction), otherwise the + /// side-effect will occur twice. + pub fn sink_inst(&mut self, ir_inst: Inst) { assert!(has_lowering_side_effect(self.f, ir_inst)); assert!(self.cur_scan_entry_color.is_some()); @@ -1444,19 +1358,24 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { self.inst_sunk.insert(ir_inst); } - fn get_immediate_data(&self, imm: Immediate) -> &ConstantData { + /// Retrieve immediate data given a handle. + pub fn get_immediate_data(&self, imm: Immediate) -> &ConstantData { self.f.dfg.immediates.get(imm).unwrap() } - fn get_constant_data(&self, constant_handle: Constant) -> &ConstantData { + /// Retrieve constant data given a handle. + pub fn get_constant_data(&self, constant_handle: Constant) -> &ConstantData { self.f.dfg.constants.get(constant_handle) } - fn use_constant(&mut self, constant: VCodeConstantData) -> VCodeConstant { + /// Indicate that a constant should be emitted. + pub fn use_constant(&mut self, constant: VCodeConstantData) -> VCodeConstant { self.vcode.constants().insert(constant) } - fn get_immediate(&self, ir_inst: Inst) -> Option { + /// Retrieve the value immediate from an instruction. This will perform necessary lookups on the + /// `DataFlowGraph` to retrieve even large immediates. + pub fn get_immediate(&self, ir_inst: Inst) -> Option { let inst_data = self.data(ir_inst); match inst_data { InstructionData::Shuffle { imm, .. } => { @@ -1475,7 +1394,9 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { } } - fn ensure_in_vreg(&mut self, reg: Reg, ty: Type) -> Reg { + /// Cause the value in `reg` to be in a virtual reg, by copying it into a new virtual reg + /// if `reg` is a real reg. `ty` describes the type of the value in `reg`. + pub fn ensure_in_vreg(&mut self, reg: Reg, ty: Type) -> Reg { if reg.to_virtual_reg().is_some() { reg } else { @@ -1485,7 +1406,8 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> { } } - fn set_vreg_alias(&mut self, from: Reg, to: Reg) { + /// Note that one vreg is to be treated as an alias of another. + pub fn set_vreg_alias(&mut self, from: Reg, to: Reg) { trace!("set vreg alias: from {:?} to {:?}", from, to); self.vcode.set_vreg_alias(from, to); }