Skip to content

x64: Improve memory support in {insert,extract}lane #5982

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alexcrichton merged 3 commits into from
Mar 13, 2023
Merged

x64: Improve memory support in {insert,extract}lane #5982

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a533803
x64: Improve memory support in `{insert,extract}lane`
alexcrichton Mar 10, 2023
6093cd3
Add missing commas
alexcrichton Mar 11, 2023
29fc2ea
Fix tests
alexcrichton Mar 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 76 additions & 3 deletions cranelift/codegen/src/isa/x64/inst.isle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,8 +290,18 @@
;; XMM (scalar or vector) unary op (from xmm to reg/mem) using the
;; VEX prefix
(XmmMovRMVex (op AvxOpcode)
(src Reg)
(src Xmm)
(dst SyntheticAmode))
(XmmMovRMImmVex (op AvxOpcode)
(src Xmm)
(dst SyntheticAmode)
(imm u8))

;; XMM (scalar) unary op (from xmm to integer reg): vpextr{w,b,d,q}
(XmmToGprImmVex (op AvxOpcode)
(src Xmm)
(dst WritableGpr)
(imm u8))

;; XMM (scalar or vector) binary op that relies on the EVEX
;; prefix. Takes two inputs.
Expand Down Expand Up @@ -343,8 +353,12 @@
;; XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd,
;; movq
(XmmMovRM (op SseOpcode)
(src Reg)
(src Xmm)
(dst SyntheticAmode))
(XmmMovRMImm (op SseOpcode)
(src Xmm)
(dst SyntheticAmode)
(imm u8))

;; XMM (scalar) unary op (from xmm to integer reg): movd, movq,
;; cvtts{s,d}2si
Expand Down Expand Up @@ -1364,6 +1378,10 @@
Vmovups
Vmovupd
Vmovdqu
Vpextrb
Vpextrw
Vpextrd
Vpextrq
))

(type Avx512Opcode extern
Expand Down Expand Up @@ -2043,10 +2061,18 @@
(rule (xmm_movrm op addr data)
(SideEffectNoResult.Inst (MInst.XmmMovRM op data addr)))

(decl xmm_movrm_imm (SseOpcode SyntheticAmode Xmm u8) SideEffectNoResult)
(rule (xmm_movrm_imm op addr data imm)
(SideEffectNoResult.Inst (MInst.XmmMovRMImm op data addr imm)))

(decl xmm_movrm_vex (AvxOpcode SyntheticAmode Xmm) SideEffectNoResult)
(rule (xmm_movrm_vex op addr data)
(SideEffectNoResult.Inst (MInst.XmmMovRMVex op data addr)))

(decl xmm_movrm_imm_vex (AvxOpcode SyntheticAmode Xmm u8) SideEffectNoResult)
(rule (xmm_movrm_imm_vex op addr data imm)
(SideEffectNoResult.Inst (MInst.XmmMovRMImmVex op data addr imm)))

;; Load a constant into an XMM register.
(decl x64_xmm_load_const (Type VCodeConstant) Xmm)
(rule (x64_xmm_load_const ty const)
Expand Down Expand Up @@ -3603,21 +3629,61 @@
(decl x64_pextrb (Xmm u8) Gpr)
(rule (x64_pextrb src lane)
(xmm_to_gpr_imm (SseOpcode.Pextrb) src lane))
(rule 1 (x64_pextrb src lane)
(if-let $true (has_avx))
(xmm_to_gpr_imm_vex (AvxOpcode.Vpextrb) src lane))

(decl x64_pextrb_store (SyntheticAmode Xmm u8) SideEffectNoResult)
(rule (x64_pextrb_store addr src lane)
(xmm_movrm_imm (SseOpcode.Pextrb) addr src lane))
(rule 1 (x64_pextrb_store addr src lane)
(if-let $true (has_avx))
(xmm_movrm_imm_vex (AvxOpcode.Vpextrb) addr src lane))

;; Helper for creating `pextrw` instructions.
(decl x64_pextrw (Xmm u8) Gpr)
(rule (x64_pextrw src lane)
(xmm_to_gpr_imm (SseOpcode.Pextrw) src lane))
(rule 1 (x64_pextrw src lane)
(if-let $true (has_avx))
(xmm_to_gpr_imm_vex (AvxOpcode.Vpextrw) src lane))

(decl x64_pextrw_store (SyntheticAmode Xmm u8) SideEffectNoResult)
(rule (x64_pextrw_store addr src lane)
(xmm_movrm_imm (SseOpcode.Pextrw) addr src lane))
(rule 1 (x64_pextrw_store addr src lane)
(if-let $true (has_avx))
(xmm_movrm_imm_vex (AvxOpcode.Vpextrw) addr src lane))

;; Helper for creating `pextrd` instructions.
(decl x64_pextrd (Xmm u8) Gpr)
(rule (x64_pextrd src lane)
(xmm_to_gpr_imm (SseOpcode.Pextrd) src lane))
(rule 1 (x64_pextrd src lane)
(if-let $true (has_avx))
(xmm_to_gpr_imm_vex (AvxOpcode.Vpextrd) src lane))

(decl x64_pextrd_store (SyntheticAmode Xmm u8) SideEffectNoResult)
(rule (x64_pextrd_store addr src lane)
(xmm_movrm_imm (SseOpcode.Pextrd) addr src lane))
(rule 1 (x64_pextrd_store addr src lane)
(if-let $true (has_avx))
(xmm_movrm_imm_vex (AvxOpcode.Vpextrd) addr src lane))

;; Helper for creating `pextrq` instructions.
(decl x64_pextrq (Xmm u8) Gpr)
(rule (x64_pextrq src lane)
(xmm_to_gpr_imm (SseOpcode.Pextrq) src lane))
(rule 1 (x64_pextrq src lane)
(if-let $true (has_avx))
(xmm_to_gpr_imm_vex (AvxOpcode.Vpextrq) src lane))

(decl x64_pextrq_store (SyntheticAmode Xmm u8) SideEffectNoResult)
(rule (x64_pextrq_store addr src lane)
(xmm_movrm_imm (SseOpcode.Pextrq) addr src lane))
(rule 1 (x64_pextrq_store addr src lane)
(if-let $true (has_avx))
(xmm_movrm_imm_vex (AvxOpcode.Vpextrq) addr src lane))

;; Helper for creating `MInst.XmmToGpr` instructions.
(decl xmm_to_gpr (SseOpcode Xmm OperandSize) Gpr)
Expand All @@ -3626,13 +3692,20 @@
(_ Unit (emit (MInst.XmmToGpr op src dst size))))
dst))

;; Helper for creating `MInst.XmmToGpr` instructions.
;; Helper for creating `MInst.XmmToGprImm` instructions.
(decl xmm_to_gpr_imm (SseOpcode Xmm u8) Gpr)
(rule (xmm_to_gpr_imm op src imm)
(let ((dst WritableGpr (temp_writable_gpr))
(_ Unit (emit (MInst.XmmToGprImm op src dst imm))))
dst))

;; Helper for creating `MInst.XmmToGprImmVex` instructions.
(decl xmm_to_gpr_imm_vex (AvxOpcode Xmm u8) Gpr)
(rule (xmm_to_gpr_imm_vex op src imm)
(let ((dst WritableGpr (temp_writable_gpr))
(_ Unit (emit (MInst.XmmToGprImmVex op src dst imm))))
dst))

;; Helper for creating `pmovmskb` instructions.
(decl x64_pmovmskb (OperandSize Xmm) Gpr)
(rule (x64_pmovmskb size src)
Expand Down
6 changes: 5 additions & 1 deletion cranelift/codegen/src/isa/x64/inst/args.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1699,7 +1699,11 @@ impl AvxOpcode {
| AvxOpcode::Vmovsd
| AvxOpcode::Vmovups
| AvxOpcode::Vmovupd
| AvxOpcode::Vmovdqu => {
| AvxOpcode::Vmovdqu
| AvxOpcode::Vpextrb
| AvxOpcode::Vpextrw
| AvxOpcode::Vpextrd
| AvxOpcode::Vpextrq => {
smallvec![InstructionSet::AVX]
}
}
Expand Down
71 changes: 69 additions & 2 deletions cranelift/codegen/src/isa/x64/inst/emit.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2430,7 +2430,7 @@ pub(crate) fn emit(
}

Inst::XmmMovRMVex { op, src, dst } => {
let src = allocs.next(*src);
let src = allocs.next(src.to_reg());
let dst = dst.with_allocs(allocs).finalize(state, sink);

let (prefix, map, opcode) = match op {
Expand All @@ -2451,6 +2451,52 @@ pub(crate) fn emit(
.encode(sink);
}

Inst::XmmMovRMImmVex { op, src, dst, imm } => {
let src = allocs.next(src.to_reg());
let dst = dst.with_allocs(allocs).finalize(state, sink);

let (w, prefix, map, opcode) = match op {
AvxOpcode::Vpextrb => (false, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x14),
AvxOpcode::Vpextrw => (false, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x15),
AvxOpcode::Vpextrd => (false, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x16),
AvxOpcode::Vpextrq => (true, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x16),
_ => unimplemented!("Opcode {:?} not implemented", op),
};
VexInstruction::new()
.length(VexVectorLength::V128)
.w(w)
.prefix(prefix)
.map(map)
.opcode(opcode)
.rm(dst)
.reg(src.to_real_reg().unwrap().hw_enc())
.imm(*imm)
.encode(sink);
}

Inst::XmmToGprImmVex { op, src, dst, imm } => {
let src = allocs.next(src.to_reg());
let dst = allocs.next(dst.to_reg().to_reg());

let (w, prefix, map, opcode) = match op {
AvxOpcode::Vpextrb => (false, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x14),
AvxOpcode::Vpextrw => (false, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x15),
AvxOpcode::Vpextrd => (false, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x16),
AvxOpcode::Vpextrq => (true, LegacyPrefixes::_66, OpcodeMap::_0F3A, 0x16),
_ => unimplemented!("Opcode {:?} not implemented", op),
};
VexInstruction::new()
.length(VexVectorLength::V128)
.w(w)
.prefix(prefix)
.map(map)
.opcode(opcode)
.rm(dst.to_real_reg().unwrap().hw_enc())
.reg(src.to_real_reg().unwrap().hw_enc())
.imm(*imm)
.encode(sink);
}

Inst::XmmRmREvex {
op,
src1,
Expand Down Expand Up @@ -2649,7 +2695,7 @@ pub(crate) fn emit(
}

Inst::XmmMovRM { op, src, dst } => {
let src = allocs.next(*src);
let src = allocs.next(src.to_reg());
let dst = dst.with_allocs(allocs);

let (prefix, opcode) = match op {
Expand All @@ -2666,6 +2712,27 @@ pub(crate) fn emit(
emit_std_reg_mem(sink, prefix, opcode, 2, src, dst, RexFlags::clear_w(), 0);
}

Inst::XmmMovRMImm { op, src, dst, imm } => {
let src = allocs.next(src.to_reg());
let dst = dst.with_allocs(allocs);

let (w, prefix, opcode) = match op {
SseOpcode::Pextrb => (false, LegacyPrefixes::_66, 0x0F3A14),
SseOpcode::Pextrw => (false, LegacyPrefixes::_66, 0x0F3A15),
SseOpcode::Pextrd => (false, LegacyPrefixes::_66, 0x0F3A16),
SseOpcode::Pextrq => (true, LegacyPrefixes::_66, 0x0F3A16),
_ => unimplemented!("Opcode {:?} not implemented", op),
};
let rex = if w {
RexFlags::set_w()
} else {
RexFlags::clear_w()
};
let dst = &dst.finalize(state, sink);
emit_std_reg_mem(sink, prefix, opcode, 3, src, dst, rex, 1);
sink.put1(*imm);
}

Inst::XmmToGpr {
op,
src,
Expand Down
2 changes: 1 addition & 1 deletion cranelift/codegen/src/isa/x64/inst/emit_tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4871,7 +4871,7 @@ fn test_x64_emit() {
imm: 2,
},
"C4430920EF02",
"vpinsrb $2 %xmm14, %r15, %xmm13",
"vpinsrb $2, %xmm14, %r15, %xmm13",
));

// ========================================================
Expand Down
48 changes: 39 additions & 9 deletions cranelift/codegen/src/isa/x64/inst/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ impl Inst {
// These use dynamic SSE opcodes.
Inst::GprToXmm { op, .. }
| Inst::XmmMovRM { op, .. }
| Inst::XmmMovRMImm { op, .. }
| Inst::XmmRmiReg { opcode: op, .. }
| Inst::XmmRmR { op, .. }
| Inst::XmmRmRUnaligned { op, .. }
Expand All @@ -153,7 +154,9 @@ impl Inst {
| Inst::XmmVexPinsr { op, .. }
| Inst::XmmUnaryRmRVex { op, .. }
| Inst::XmmUnaryRmRImmVex { op, .. }
| Inst::XmmMovRMVex { op, .. } => op.available_from(),
| Inst::XmmMovRMVex { op, .. }
| Inst::XmmMovRMImmVex { op, .. }
| Inst::XmmToGprImmVex { op, .. } => op.available_from(),
}
}
}
Expand Down Expand Up @@ -331,7 +334,7 @@ impl Inst {
debug_assert!(src.class() == RegClass::Float);
Inst::XmmMovRM {
op,
src,
src: Xmm::new(src).unwrap(),
dst: dst.into(),
}
}
Expand Down Expand Up @@ -933,17 +936,33 @@ impl PrettyPrint for Inst {
}

Inst::XmmMovRM { op, src, dst, .. } => {
let src = pretty_print_reg(*src, 8, allocs);
let src = pretty_print_reg(src.to_reg(), 8, allocs);
let dst = dst.pretty_print(8, allocs);
format!("{} {}, {}", ljustify(op.to_string()), src, dst)
}

Inst::XmmMovRMVex { op, src, dst, .. } => {
let src = pretty_print_reg(*src, 8, allocs);
let src = pretty_print_reg(src.to_reg(), 8, allocs);
let dst = dst.pretty_print(8, allocs);
format!("{} {}, {}", ljustify(op.to_string()), src, dst)
}

Inst::XmmMovRMImm {
op, src, dst, imm, ..
} => {
let src = pretty_print_reg(src.to_reg(), 8, allocs);
let dst = dst.pretty_print(8, allocs);
format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
}

Inst::XmmMovRMImmVex {
op, src, dst, imm, ..
} => {
let src = pretty_print_reg(src.to_reg(), 8, allocs);
let dst = dst.pretty_print(8, allocs);
format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
}

Inst::XmmRmR {
op,
src1,
Expand Down Expand Up @@ -1023,7 +1042,7 @@ impl PrettyPrint for Inst {
let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
let src2 = src2.pretty_print(8, allocs);

format!("{} ${imm} {src1}, {src2}, {dst}", ljustify(op.to_string()))
format!("{} ${imm}, {src1}, {src2}, {dst}", ljustify(op.to_string()))
}

Inst::XmmVexPinsr {
Expand All @@ -1038,7 +1057,7 @@ impl PrettyPrint for Inst {
let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
let src2 = src2.pretty_print(8, allocs);

format!("{} ${imm} {src1}, {src2}, {dst}", ljustify(op.to_string()))
format!("{} ${imm}, {src1}, {src2}, {dst}", ljustify(op.to_string()))
}

Inst::XmmRmRVex3 {
Expand Down Expand Up @@ -1190,6 +1209,12 @@ impl PrettyPrint for Inst {
format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
}

Inst::XmmToGprImmVex { op, src, dst, imm } => {
let src = pretty_print_reg(src.to_reg(), 8, allocs);
let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
}

Inst::GprToXmm {
op,
src,
Expand Down Expand Up @@ -2033,8 +2058,11 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
collector.reg_reuse_def(dst.to_writable_reg(), 0); // Reuse RHS.
src2.get_operands(collector);
}
Inst::XmmMovRM { src, dst, .. } | Inst::XmmMovRMVex { src, dst, .. } => {
collector.reg_use(*src);
Inst::XmmMovRM { src, dst, .. }
| Inst::XmmMovRMVex { src, dst, .. }
| Inst::XmmMovRMImm { src, dst, .. }
| Inst::XmmMovRMImmVex { src, dst, .. } => {
collector.reg_use(src.to_reg());
dst.get_operands(collector);
}
Inst::XmmCmpRmR { src, dst, .. } => {
Expand All @@ -2058,7 +2086,9 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
collector.reg_use(src.to_reg());
collector.reg_fixed_nonallocatable(*dst);
}
Inst::XmmToGpr { src, dst, .. } | Inst::XmmToGprImm { src, dst, .. } => {
Inst::XmmToGpr { src, dst, .. }
| Inst::XmmToGprImm { src, dst, .. }
| Inst::XmmToGprImmVex { src, dst, .. } => {
collector.reg_use(src.to_reg());
collector.reg_def(dst.to_writable_reg());
}
Expand Down
Loading