x64: Migrate {s,u}{div,rem} to ISLE

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -959,13 +959,10 @@
             (result Reg (a64_sdiv $I64 valid_x64 y64)))
         result))
 
-;; Helper for extracting an immediate that's not 0 and not -1 from an imm64.
-(decl safe_divisor_from_imm64 (u64) Imm64)
-(extern extractor safe_divisor_from_imm64 safe_divisor_from_imm64)
-
 ;; Special case for `sdiv` where no checks are needed due to division by a
 ;; constant meaning the checks are always passed.
-(rule 1 (lower (has_type (fits_in_64 ty) (sdiv x (iconst (safe_divisor_from_imm64 y)))))
+(rule 1 (lower (has_type (fits_in_64 ty) (sdiv x (iconst imm))))
+      (if-let y (safe_divisor_from_imm64 ty imm))
       (a64_sdiv $I64 (put_in_reg_sext64 x) (imm ty (ImmExtend.Sign) y)))
 
 ;; Helper for placing a `Value` into a `Reg` and validating that it's nonzero.

cranelift/codegen/src/isa/aarch64/lower/isle.rs

@@ -392,13 +392,6 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
         writable_zero_reg()
     }
 
-    fn safe_divisor_from_imm64(&mut self, val: Imm64) -> Option<u64> {
-        match val.bits() {
-            0 | -1 => None,
-            n => Some(n as u64),
-        }
-    }
-
     fn shift_mask(&mut self, ty: Type) -> ImmLogic {
         debug_assert!(ty.lane_bits().is_power_of_two());
 

cranelift/codegen/src/isa/x64/inst.isle

@@ -58,14 +58,23 @@
             (dst WritableGpr))
 
        ;; Integer quotient and remainder: (div idiv) $rax $rdx (reg addr)
-       (Div (size OperandSize) ;; 1, 2, 4, or 8
-            (signed bool)
+       ;;
+       ;; Note that this isn't used for 8-bit division which has its own `Div8`
+       ;; instruction.
+       (Div (size OperandSize) ;; 2, 4, or 8
+            (sign DivSignedness)
             (divisor GprMem)
             (dividend_lo Gpr)
             (dividend_hi Gpr)
             (dst_quotient WritableGpr)
             (dst_remainder WritableGpr))
 
+       ;; Same as `Div`, but for 8-bits where the regalloc behavior is different
+       (Div8 (sign DivSignedness)
+             (divisor GprMem)
+             (dividend Gpr)
+             (dst WritableGpr))
+
        ;; The high (and low) bits of a (un)signed multiply: `RDX:RAX := RAX *
        ;; rhs`.
        (MulHi (size OperandSize)
@@ -75,19 +84,47 @@
               (dst_lo WritableGpr)
               (dst_hi WritableGpr))
 
-       ;; A synthetic sequence to implement the right inline checks for
-       ;; remainder and division, assuming the dividend is in %rax.
+       ;; A synthetic instruction sequence used as part of the lowering of the
+       ;; `srem` instruction which returns 0 if the divisor is -1 and
+       ;; otherwise executes an `idiv` instruction.
        ;;
-       ;; The generated code sequence is described in the emit's function match
-       ;; arm for this instruction.
-       (CheckedDivOrRemSeq (kind DivOrRemKind)
-                           (size OperandSize)
-                           (dividend_lo Gpr)
-                           (dividend_hi Gpr)
-                           (divisor Gpr)
-                           (dst_quotient WritableGpr)
-                           (dst_remainder WritableGpr)
-                           (tmp OptionWritableGpr))
+       ;; Note that this does not check for 0 as that's expected to be done
+       ;; separately. Also note that 8-bit types don't use this and use
+       ;; `CheckedSRemSeq8` instead.
+       (CheckedSRemSeq (size OperandSize)
+                       (dividend_lo Gpr)
+                       (dividend_hi Gpr)
+                       (divisor Gpr)
+                       (dst_quotient WritableGpr)
+                       (dst_remainder WritableGpr))
+
+       ;; Same as above but for 8-bit types.
+       (CheckedSRemSeq8 (dividend Gpr)
+                        (divisor Gpr)
+                        (dst WritableGpr))
+
+       ;; Validates that the `divisor` can be safely divided into the
+       ;; `dividend`.
+       ;;
+       ;; This is a separate pseudo-instruction because it has some jumps in
+       ;; ways that can't be modeled otherwise with instructions right now. This
+       ;; will trap if the `divisor` is zero or if it's -1 and `dividend` is
+       ;; INT_MIN for the associated type.
+       ;;
+       ;; Note that 64-bit types must use `ValidateSdivDivisor64`.
+       (ValidateSdivDivisor (size OperandSize)
+                            (dividend Gpr)
+                            (divisor Gpr))
+
+       ;; Same as `ValidateSdivDivisor` but for 64-bit types.
+       ;;
+       ;; This is a distinct instruction because the emission in `emit.rs`
+       ;; requires a temporary register to load an immediate into, hence the
+       ;; `tmp` field in this instruction not present in the non-64-bit one.
+       (ValidateSdivDivisor64 (dividend Gpr)
+                              (divisor Gpr)
+                              (tmp WritableGpr))
+
 
        ;; Do a sign-extend based on the sign of the value in rax into rdx: (cwd
        ;; cdq cqo) or al into ah: (cbw)
@@ -628,6 +665,10 @@
             Size32
             Size64))
 
+(type DivSignedness
+      (enum Signed
+            Unsigned))
+
 (type FenceKind extern
       (enum MFence
             LFence
@@ -690,12 +731,6 @@
             Tzcnt
             Popcnt))
 
-(type DivOrRemKind extern
-      (enum SignedDiv
-            UnsignedDiv
-            SignedRem
-            UnsignedRem))
-
 (type SseOpcode extern
       (enum Addps
             Addpd
@@ -4521,15 +4556,70 @@
 
 ;;;; Division/Remainders ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(decl emit_div_or_rem (DivOrRemKind Type WritableGpr Gpr Gpr) Unit)
-(extern constructor emit_div_or_rem emit_div_or_rem)
+;; Helper for creating `CheckedSRemSeq` instructions.
+(decl x64_checked_srem_seq (OperandSize Gpr Gpr Gpr) ValueRegs)
+(rule (x64_checked_srem_seq size dividend_lo dividend_hi divisor)
+      (let ((dst_quotient WritableGpr (temp_writable_gpr))
+            (dst_remainder WritableGpr (temp_writable_gpr))
+            (_ Unit (emit (MInst.CheckedSRemSeq size dividend_lo dividend_hi divisor dst_quotient dst_remainder))))
+        (value_regs dst_quotient dst_remainder)))
+
+(decl x64_checked_srem_seq8 (Gpr Gpr) Gpr)
+(rule (x64_checked_srem_seq8 dividend divisor)
+      (let ((dst WritableGpr (temp_writable_gpr))
+            (_ Unit (emit (MInst.CheckedSRemSeq8 dividend divisor dst))))
+        dst))
+
+;; Helper for creating `Div8` instructions
+(decl x64_div8 (Gpr GprMem DivSignedness) Gpr)
+(rule (x64_div8 dividend divisor sign)
+      (let ((dst WritableGpr (temp_writable_gpr))
+            (_ Unit (emit (MInst.Div8 sign divisor dividend dst))))
+        dst))
 
-(decl div_or_rem (DivOrRemKind Value Value) Gpr)
-(rule (div_or_rem kind a @ (value_type ty) b)
+;; Helper for creating `Div` instructions
+;;
+;; Two registers are returned through `ValueRegs` where the first is the
+;; quotient and the second is the remainder.
+(decl x64_div (Gpr Gpr GprMem OperandSize DivSignedness) ValueRegs)
+(rule (x64_div dividend_lo dividend_hi divisor size sign)
+      (let ((dst_quotient WritableGpr (temp_writable_gpr))
+            (dst_remainder WritableGpr (temp_writable_gpr))
+            (_ Unit (emit (MInst.Div size sign divisor dividend_lo dividend_hi dst_quotient dst_remainder))))
+        (value_regs dst_quotient dst_remainder)))
+
+;; Helper for `Div`, returning the quotient and discarding the remainder.
+(decl x64_div_quotient (Gpr Gpr GprMem OperandSize DivSignedness) ValueRegs)
+(rule (x64_div_quotient dividend_lo dividend_hi divisor size sign)
+      (value_regs_get (x64_div dividend_lo dividend_hi divisor size sign) 0))
+
+;; Helper for `Div`, returning the remainder and discarding the quotient.
+(decl x64_div_remainder (Gpr Gpr GprMem OperandSize DivSignedness) ValueRegs)
+(rule (x64_div_remainder dividend_lo dividend_hi divisor size sign)
+      (value_regs_get (x64_div dividend_lo dividend_hi divisor size sign) 1))
+
+;; Helper for creating `SignExtendData` instructions
+(decl x64_sign_extend_data (Gpr OperandSize) Gpr)
+(rule (x64_sign_extend_data src size)
       (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit_div_or_rem kind ty dst a b)))
+            (_ Unit (emit (MInst.SignExtendData size src dst))))
         dst))
 
+;; Helper for creating `ValidateSdivDivisor` instructions.
+(decl validate_sdiv_divisor (OperandSize Gpr Gpr) Gpr)
+(rule (validate_sdiv_divisor size dividend divisor)
+      (let ((_ Unit (emit (MInst.ValidateSdivDivisor size dividend divisor))))
+        divisor))
+
+;; Helper for creating `ValidateSdivDivisor64` instructions.
+(decl validate_sdiv_divisor64 (Gpr Gpr) Gpr)
+(rule (validate_sdiv_divisor64 dividend divisor)
+      (let (
+          (tmp WritableGpr (temp_writable_gpr))
+          (_ Unit (emit (MInst.ValidateSdivDivisor64 dividend divisor tmp)))
+        )
+        divisor))
+
 ;;;; Pinned Register ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (decl read_pinned_gpr () Gpr)

cranelift/codegen/src/isa/x64/inst/args.rs

@@ -12,6 +12,8 @@ use smallvec::{smallvec, SmallVec};
 use std::fmt;
 use std::string::String;
 
+pub use crate::isa::x64::lower::isle::generated_code::DivSignedness;
+
 /// An extenstion trait for converting `Writable{Xmm,Gpr}` to `Writable<Reg>`.
 pub trait ToWritableReg {
     /// Convert `Writable{Xmm,Gpr}` to `Writable<Reg>`.
@@ -1878,35 +1880,6 @@ impl fmt::Display for ShiftKind {
     }
 }
 
-/// What kind of division or remainder instruction this is?
-#[derive(Clone, Eq, PartialEq)]
-pub enum DivOrRemKind {
-    /// Signed division.
-    SignedDiv,
-    /// Unsigned division.
-    UnsignedDiv,
-    /// Signed remainder.
-    SignedRem,
-    /// Unsigned remainder.
-    UnsignedRem,
-}
-
-impl DivOrRemKind {
-    pub(crate) fn is_signed(&self) -> bool {
-        match self {
-            DivOrRemKind::SignedDiv | DivOrRemKind::SignedRem => true,
-            _ => false,
-        }
-    }
-
-    pub(crate) fn is_div(&self) -> bool {
-        match self {
-            DivOrRemKind::SignedDiv | DivOrRemKind::UnsignedDiv => true,
-            _ => false,
-        }
-    }
-}
-
 /// These indicate condition code tests.  Not all are represented since not all are useful in
 /// compiler-generated code.
 #[derive(Copy, Clone)]