Guard recursion in will_simplify_with_ireduce (#7882)

Add a test to expose issues with unbounded recursion through `iadd`
during egraph rewrites, and bound the recursion of
`will_simplify_with_ireduce`.

Fixes #7874

Co-authored-by: Nick Fitzgerald <fitzgen@gmail.com>

Author: elliottt

cranelift/codegen/src/opts/extends.isle

@@ -77,18 +77,26 @@
 
 ;; Matches values where `ireducing` them will not actually introduce another
 ;; instruction, since other rules will collapse them with the reduction.
-(decl pure multi will_simplify_with_ireduce (Value) Value)
-(rule (will_simplify_with_ireduce x@(uextend _ _)) x)
-(rule (will_simplify_with_ireduce x@(sextend _ _)) x)
-(rule (will_simplify_with_ireduce x@(iconst _ _)) x)
-(rule (will_simplify_with_ireduce x@(unary_op _ _ a))
-      (if-let _ (will_simplify_with_ireduce a))
+(decl pure multi will_simplify_with_ireduce_rec (u8 Value) Value)
+(rule (will_simplify_with_ireduce_rec _ x@(uextend _ _)) x)
+(rule (will_simplify_with_ireduce_rec _ x@(sextend _ _)) x)
+(rule (will_simplify_with_ireduce_rec _ x@(iconst _ _)) x)
+(rule (will_simplify_with_ireduce_rec depth x@(unary_op _ _ a))
+      (if-let _ (u8_lt 0 depth))
+      (if-let _ (reducible_modular_op x))
+      (if-let _ (will_simplify_with_ireduce_rec (u8_sub depth 1) a))
       x)
-(rule (will_simplify_with_ireduce x@(binary_op _ _ a b))
-      (if-let _ (will_simplify_with_ireduce a))
-      (if-let _ (will_simplify_with_ireduce b))
+(rule (will_simplify_with_ireduce_rec depth x@(binary_op _ _ a b))
+      (if-let _ (u8_lt 0 depth))
+      (if-let _ (reducible_modular_op x))
+      (if-let _ (will_simplify_with_ireduce_rec (u8_sub depth 1) a))
+      (if-let _ (will_simplify_with_ireduce_rec (u8_sub depth 1) b))
       x)
 
+(decl pure multi will_simplify_with_ireduce (Value) Value)
+(rule (will_simplify_with_ireduce x)
+      (will_simplify_with_ireduce_rec 2 x))
+
 ;; Matches values where the high bits of the input don't affect lower bits of
 ;; the output, and thus the inputs can be reduced before the operation rather
 ;; than doing the wide operation then reducing afterwards.

cranelift/filetests/filetests/egraph/associative-and-commutative.clif

@@ -79,3 +79,41 @@ block0(v0: i32, v1: i32, v2: i32, v3: i32):
 ; nextln: v8 = bxor v4, v7
 ; check: return v8
 }
+
+;; We don't have any assertions about the result of optimizing this function,
+;; but it's a good canary for unbounded recursion in optimization rulesets. In
+;; particular, because of the shared structure in the dag, it won't be obvious
+;; to rules that are pattern matching on trees that this is actually a chain,
+;; and they will exhibit exponential behavior as a result.
+function %iadd_big_chain(i8) -> i16 {
+block0(v0: i8):
+    v1 = uextend.i32 v0
+    v2 = iconst.i32 42
+    v3 = iadd v1, v2
+    v4 = iadd v3, v3
+    v5 = iadd v4, v4
+    v6 = iadd v5, v5
+    v7 = iadd v6, v6
+    v8 = iadd v7, v7
+    v9 = iadd v8, v8
+    v10 = iadd v9, v9
+    v11 = iadd v10, v10
+    v12 = iadd v11, v11
+    v13 = iadd v12, v12
+    v14 = iadd v13, v13
+    v15 = iadd v14, v14
+    v16 = iadd v15, v15
+    v17 = iadd v16, v16
+    v18 = iadd v17, v17
+    v19 = iadd v18, v18
+    v20 = iadd v19, v19
+    v21 = iadd v20, v20
+    v22 = iadd v21, v21
+    v23 = iadd v22, v22
+    v24 = iadd v23, v23
+    v25 = iadd v24, v24
+    v26 = iadd v25, v25
+    v27 = iadd v26, v26
+    v28 = ireduce.i16 v27
+    return v28
+}