[Clang] Remove 't' from __builtin_amdgcn_flat_atomic_fadd_f32/f64 (#173381)

jmmartinez · web-flow · commit fcd9235d86bb · 2025-12-24T12:07:14.000+01:00
Allows for type checking depending on the built-in signature.

This introduces some subtle changes in code generation: before, since
the signature was meaningless, we would accept any pointer type without
casting. After this change, the pointer of the `atomicrmw` matches the
flat address space.
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -272,14 +272,14 @@ TARGET_BUILTIN(__builtin_amdgcn_global_atomic_fadd_v2f16, "V2hV2h*1V2h", "t", "a
 TARGET_BUILTIN(__builtin_amdgcn_global_atomic_fmin_f64, "dd*1d", "t", "gfx90a-insts")
 TARGET_BUILTIN(__builtin_amdgcn_global_atomic_fmax_f64, "dd*1d", "t", "gfx90a-insts")
 
-TARGET_BUILTIN(__builtin_amdgcn_flat_atomic_fadd_f64, "dd*0d", "t", "gfx90a-insts")
+TARGET_BUILTIN(__builtin_amdgcn_flat_atomic_fadd_f64, "dd*0d", "", "gfx90a-insts")
 TARGET_BUILTIN(__builtin_amdgcn_flat_atomic_fmin_f64, "dd*0d", "t", "gfx90a-insts")
 TARGET_BUILTIN(__builtin_amdgcn_flat_atomic_fmax_f64, "dd*0d", "t", "gfx90a-insts")
 
 TARGET_BUILTIN(__builtin_amdgcn_ds_atomic_fadd_f64, "dd*3d", "", "gfx90a-insts")
 TARGET_BUILTIN(__builtin_amdgcn_ds_atomic_fadd_f32, "ff*3f", "", "gfx8-insts")
 
-TARGET_BUILTIN(__builtin_amdgcn_flat_atomic_fadd_f32, "ff*0f", "t", "gfx940-insts")
+TARGET_BUILTIN(__builtin_amdgcn_flat_atomic_fadd_f32, "ff*0f", "", "gfx940-insts")
 TARGET_BUILTIN(__builtin_amdgcn_flat_atomic_fadd_v2f16, "V2hV2h*0V2h", "t", "atomic-flat-pk-add-16-insts")
 TARGET_BUILTIN(__builtin_amdgcn_flat_atomic_fadd_v2bf16, "V2sV2s*0V2s", "t", "atomic-flat-pk-add-16-insts")
 TARGET_BUILTIN(__builtin_amdgcn_global_atomic_fadd_v2bf16, "V2sV2s*1V2s", "t", "atomic-global-pk-add-bf16-inst")
diff --git a/clang/test/CodeGenHIP/amdgpu-flat-atomic-fadd.hip b/clang/test/CodeGenHIP/amdgpu-flat-atomic-fadd.hip
@@ -0,0 +1,175 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -target-cpu gfx942 -emit-llvm -fcuda-is-device %s -o - | FileCheck %s
+
+#define __device__ __attribute__((device))
+#define __shared__ __attribute__((shared))
+#define __constant__ __attribute__((constant))
+
+__constant__ float const_float;
+__constant__ double const_double;
+__device__ float global_float;
+__device__ double global_double;
+
+// CHECK-LABEL: define dso_local void @_Z30test_flat_atomic_fadd_f32_flatPff(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], float noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[PTR_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[VAL_ADDR:%.*]] = alloca float, align 4, addrspace(5)
+// CHECK-NEXT:    [[RESULT:%.*]] = alloca float, align 4, addrspace(5)
+// CHECK-NEXT:    [[PTR_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[PTR_ADDR]] to ptr
+// CHECK-NEXT:    [[VAL_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[VAL_ADDR]] to ptr
+// CHECK-NEXT:    [[RESULT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RESULT]] to ptr
+// CHECK-NEXT:    store ptr [[PTR]], ptr [[PTR_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store float [[VAL]], ptr [[VAL_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[VAL_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr [[TMP0]], float [[TMP1]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META4:![0-9]+]], !amdgpu.ignore.denormal.mode [[META4]]
+// CHECK-NEXT:    store float [[TMP2]], ptr [[RESULT_ASCAST]], align 4
+// CHECK-NEXT:    ret void
+//
+__device__ void test_flat_atomic_fadd_f32_flat(float *ptr, float val) {
+  float result;
+  result = __builtin_amdgcn_flat_atomic_fadd_f32(ptr, val);
+}
+
+// CHECK-LABEL: define dso_local void @_Z30test_flat_atomic_fadd_f64_flatPdd(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], double noundef [[VAL:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[PTR_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[VAL_ADDR:%.*]] = alloca double, align 8, addrspace(5)
+// CHECK-NEXT:    [[RESULT:%.*]] = alloca double, align 8, addrspace(5)
+// CHECK-NEXT:    [[PTR_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[PTR_ADDR]] to ptr
+// CHECK-NEXT:    [[VAL_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[VAL_ADDR]] to ptr
+// CHECK-NEXT:    [[RESULT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RESULT]] to ptr
+// CHECK-NEXT:    store ptr [[PTR]], ptr [[PTR_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store double [[VAL]], ptr [[VAL_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load double, ptr [[VAL_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr [[TMP0]], double [[TMP1]] syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META4]]
+// CHECK-NEXT:    store double [[TMP2]], ptr [[RESULT_ASCAST]], align 8
+// CHECK-NEXT:    ret void
+//
+__device__ void test_flat_atomic_fadd_f64_flat(double *ptr, double val) {
+  double result;
+  result = __builtin_amdgcn_flat_atomic_fadd_f64(ptr, val);
+}
+
+// CHECK-LABEL: define dso_local void @_Z32test_flat_atomic_fadd_f32_sharedPff(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], float noundef [[VAL:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[PTR_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[VAL_ADDR:%.*]] = alloca float, align 4, addrspace(5)
+// CHECK-NEXT:    [[RESULT:%.*]] = alloca float, align 4, addrspace(5)
+// CHECK-NEXT:    [[PTR_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[PTR_ADDR]] to ptr
+// CHECK-NEXT:    [[VAL_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[VAL_ADDR]] to ptr
+// CHECK-NEXT:    [[RESULT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RESULT]] to ptr
+// CHECK-NEXT:    store ptr [[PTR]], ptr [[PTR_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store float [[VAL]], ptr [[VAL_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[VAL_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr [[TMP0]], float [[TMP1]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.ignore.denormal.mode [[META4]]
+// CHECK-NEXT:    store float [[TMP2]], ptr [[RESULT_ASCAST]], align 4
+// CHECK-NEXT:    ret void
+//
+__device__ void test_flat_atomic_fadd_f32_shared(__shared__ float *ptr, float val) {
+  float result;
+  result = __builtin_amdgcn_flat_atomic_fadd_f32(ptr, val);
+}
+
+// CHECK-LABEL: define dso_local void @_Z32test_flat_atomic_fadd_f64_sharedPdd(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], double noundef [[VAL:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[PTR_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[VAL_ADDR:%.*]] = alloca double, align 8, addrspace(5)
+// CHECK-NEXT:    [[RESULT:%.*]] = alloca double, align 8, addrspace(5)
+// CHECK-NEXT:    [[PTR_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[PTR_ADDR]] to ptr
+// CHECK-NEXT:    [[VAL_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[VAL_ADDR]] to ptr
+// CHECK-NEXT:    [[RESULT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RESULT]] to ptr
+// CHECK-NEXT:    store ptr [[PTR]], ptr [[PTR_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store double [[VAL]], ptr [[VAL_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load double, ptr [[VAL_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw fadd ptr [[TMP0]], double [[TMP1]] syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META4]]
+// CHECK-NEXT:    store double [[TMP2]], ptr [[RESULT_ASCAST]], align 8
+// CHECK-NEXT:    ret void
+//
+__device__ void test_flat_atomic_fadd_f64_shared(__shared__ double *ptr, double val) {
+  double result;
+  result = __builtin_amdgcn_flat_atomic_fadd_f64(ptr, val);
+}
+
+// CHECK-LABEL: define dso_local void @_Z34test_flat_atomic_fadd_f32_constantf(
+// CHECK-SAME: float noundef [[VAL:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VAL_ADDR:%.*]] = alloca float, align 4, addrspace(5)
+// CHECK-NEXT:    [[RESULT:%.*]] = alloca float, align 4, addrspace(5)
+// CHECK-NEXT:    [[VAL_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[VAL_ADDR]] to ptr
+// CHECK-NEXT:    [[RESULT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RESULT]] to ptr
+// CHECK-NEXT:    store float [[VAL]], ptr [[VAL_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[VAL_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw fadd ptr addrspacecast (ptr addrspace(4) @const_float to ptr), float [[TMP0]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.ignore.denormal.mode [[META4]]
+// CHECK-NEXT:    store float [[TMP1]], ptr [[RESULT_ASCAST]], align 4
+// CHECK-NEXT:    ret void
+//
+__device__ void test_flat_atomic_fadd_f32_constant(float val) {
+  float result;
+  result = __builtin_amdgcn_flat_atomic_fadd_f32(&const_float, val);
+}
+
+// CHECK-LABEL: define dso_local void @_Z34test_flat_atomic_fadd_f64_constantd(
+// CHECK-SAME: double noundef [[VAL:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VAL_ADDR:%.*]] = alloca double, align 8, addrspace(5)
+// CHECK-NEXT:    [[RESULT:%.*]] = alloca double, align 8, addrspace(5)
+// CHECK-NEXT:    [[VAL_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[VAL_ADDR]] to ptr
+// CHECK-NEXT:    [[RESULT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RESULT]] to ptr
+// CHECK-NEXT:    store double [[VAL]], ptr [[VAL_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[VAL_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw fadd ptr addrspacecast (ptr addrspace(4) @const_double to ptr), double [[TMP0]] syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META4]]
+// CHECK-NEXT:    store double [[TMP1]], ptr [[RESULT_ASCAST]], align 8
+// CHECK-NEXT:    ret void
+//
+__device__ void test_flat_atomic_fadd_f64_constant(double val) {
+  double result;
+  result = __builtin_amdgcn_flat_atomic_fadd_f64(&const_double, val);
+}
+
+// CHECK-LABEL: define dso_local void @_Z32test_flat_atomic_fadd_f32_globalf(
+// CHECK-SAME: float noundef [[VAL:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VAL_ADDR:%.*]] = alloca float, align 4, addrspace(5)
+// CHECK-NEXT:    [[RESULT:%.*]] = alloca float, align 4, addrspace(5)
+// CHECK-NEXT:    [[VAL_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[VAL_ADDR]] to ptr
+// CHECK-NEXT:    [[RESULT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RESULT]] to ptr
+// CHECK-NEXT:    store float [[VAL]], ptr [[VAL_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[VAL_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw fadd ptr addrspacecast (ptr addrspace(1) @global_float to ptr), float [[TMP0]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META4]], !amdgpu.ignore.denormal.mode [[META4]]
+// CHECK-NEXT:    store float [[TMP1]], ptr [[RESULT_ASCAST]], align 4
+// CHECK-NEXT:    ret void
+//
+__device__ void test_flat_atomic_fadd_f32_global(float val) {
+  float result;
+  result = __builtin_amdgcn_flat_atomic_fadd_f32(&global_float, val);
+}
+
+// CHECK-LABEL: define dso_local void @_Z32test_flat_atomic_fadd_f64_globald(
+// CHECK-SAME: double noundef [[VAL:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VAL_ADDR:%.*]] = alloca double, align 8, addrspace(5)
+// CHECK-NEXT:    [[RESULT:%.*]] = alloca double, align 8, addrspace(5)
+// CHECK-NEXT:    [[VAL_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[VAL_ADDR]] to ptr
+// CHECK-NEXT:    [[RESULT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RESULT]] to ptr
+// CHECK-NEXT:    store double [[VAL]], ptr [[VAL_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[VAL_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw fadd ptr addrspacecast (ptr addrspace(1) @global_double to ptr), double [[TMP0]] syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META4]]
+// CHECK-NEXT:    store double [[TMP1]], ptr [[RESULT_ASCAST]], align 8
+// CHECK-NEXT:    ret void
+//
+__device__ void test_flat_atomic_fadd_f64_global(double val) {
+  double result;
+  result = __builtin_amdgcn_flat_atomic_fadd_f64(&global_double, val);
+}
+//.
+// CHECK: [[META4]] = !{}
+//.
diff --git a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx90a.cl b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx90a.cl
@@ -47,7 +47,7 @@ void test_global_max_f64(__global double *addr, double x){
 }
 
 // CHECK-LABEL: test_flat_add_local_f64
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, double %{{.+}} syncscope("agent") monotonic, align 8{{$}}
+// CHECK: = atomicrmw fadd ptr %{{.+}}, double %{{.+}} syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !{{[0-9]+$}}
 
 // GFX90A-LABEL:  test_flat_add_local_f64$local
 // GFX90A:  ds_add_rtn_f64
@@ -57,7 +57,7 @@ void test_flat_add_local_f64(__local double *addr, double x){
 }
 
 // CHECK-LABEL: test_flat_global_add_f64
-// CHECK: = atomicrmw fadd ptr addrspace(1) {{.+}}, double %{{.+}} syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !{{[0-9]+$}}
+// CHECK: = atomicrmw fadd ptr {{.+}}, double %{{.+}} syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !{{[0-9]+$}}
 
 // GFX90A-LABEL:  test_flat_global_add_f64$local
 // GFX90A:  global_atomic_add_f64
diff --git a/clang/test/SemaHIP/amdgpu-flat-atomic-fadd-err.hip b/clang/test/SemaHIP/amdgpu-flat-atomic-fadd-err.hip
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -fsyntax-only -triple amdgcn -target-cpu gfx942 -verify %s -fcuda-is-device
+// RUN: %clang_cc1 -fsyntax-only -triple x86_64 -aux-triple amdgcn -verify %s
+
+#define __device__ __attribute__((device))
+#define __shared__ __attribute__((shared))
+
+__device__ void test_flat_atomic_fadd_f32_valid(float *ptr, float val) {
+  float result;
+  result = __builtin_amdgcn_flat_atomic_fadd_f32(ptr, val);
+}
+
+__device__ void test_flat_atomic_fadd_f32_errors(float *ptr, float val,
+                                                double *ptr_d) {
+  float result;
+  result = __builtin_amdgcn_flat_atomic_fadd_f32(ptr, val, 0); // expected-error{{too many arguments to function call, expected 2, have 3}}
+  result = __builtin_amdgcn_flat_atomic_fadd_f32(ptr_d, val); // expected-error{{cannot initialize a parameter of type}}
+}
+
+__device__ void test_flat_atomic_fadd_f64_valid(double *ptr, double val) {
+  double result;
+  result = __builtin_amdgcn_flat_atomic_fadd_f64(ptr, val);
+}
+
+__device__ void test_flat_atomic_fadd_f64_errors(double *ptr, double val,
+                                                float *ptr_f) {
+  double result;
+  result = __builtin_amdgcn_flat_atomic_fadd_f64(ptr, val, 0); // expected-error{{too many arguments to function call, expected 2, have 3}}
+  result = __builtin_amdgcn_flat_atomic_fadd_f64(ptr_f, val); // expected-error{{cannot initialize a parameter of type}}
+}

Original file line number	Diff line number	Diff line change
`@@ -47,7 +47,7 @@ void test_global_max_f64(__global double *addr, double x){`
`47`	`47`	`}`
`48`	`48`
`49`	`49`	`// CHECK-LABEL: test_flat_add_local_f64`
`50`		`-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, double %{{.+}} syncscope("agent") monotonic, align 8{{$}}`
	`50`	`+// CHECK: = atomicrmw fadd ptr %{{.+}}, double %{{.+}} syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !{{[0-9]+$}}`
`51`	`51`
`52`	`52`	`// GFX90A-LABEL: test_flat_add_local_f64$local`
`53`	`53`	`// GFX90A: ds_add_rtn_f64`
`@@ -57,7 +57,7 @@ void test_flat_add_local_f64(__local double *addr, double x){`
`57`	`57`	`}`
`58`	`58`
`59`	`59`	`// CHECK-LABEL: test_flat_global_add_f64`
`60`		`-// CHECK: = atomicrmw fadd ptr addrspace(1) {{.+}}, double %{{.+}} syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !{{[0-9]+$}}`
	`60`	`+// CHECK: = atomicrmw fadd ptr {{.+}}, double %{{.+}} syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !{{[0-9]+$}}`
`61`	`61`
`62`	`62`	`// GFX90A-LABEL: test_flat_global_add_f64$local`
`63`	`63`	`// GFX90A: global_atomic_add_f64`