caio/clang-p2996
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
1daf2994de49d1ecba4bee4e6842aa8a564cbc96
clang-p2996/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll
Matt Arsenault b01adc6bed AMDGPU: Strengthen some bfloat tests 
Fix bitcast test, which was splitting apart phis intended to force
bitcasts that survive all the way to selection.

Disable the amdgpu-codegenprepare phi splitting, which defeats the technique
of using a phi to ensure a bitcast reaches all the way to selection. Also
add a variety of bfloat tests. These probably need revisiting to avoid the
cast folding into argument loads. Also round out set of bfloat bitcast and
ABI tests.

Add codegen tests for more bf16 operations The promotion of these works
contrary to the comment.
2023-12-20 19:33:45 +07:00

1881 lines
77 KiB
LLVM
Raw Blame History

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX900 %s
; RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX908 %s
; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX90A %s
; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s
; RUN: llc -march=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX11 %s
define amdgpu_kernel void @global_atomic_fadd_ret_f32(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_f32:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[4:5], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: ; implicit-def: $vgpr1
; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX900-NEXT: s_cbranch_execz .LBB0_4
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX900-NEXT: s_mov_b64 s[4:5], 0
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s6
; GFX900-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v5, v1
; GFX900-NEXT: v_add_f32_e32 v4, v5, v2
; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX900-NEXT: s_cbranch_execnz .LBB0_2
; GFX900-NEXT: ; %bb.3: ; %Flow
; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX900-NEXT: .LBB0_4: ; %Flow1
; GFX900-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX900-NEXT: v_readfirstlane_b32 s0, v1
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX900-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX900-NEXT: global_store_dword v[0:1], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_f32:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[4:5], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: ; implicit-def: $vgpr1
; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX908-NEXT: s_cbranch_execz .LBB0_4
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX908-NEXT: v_mov_b32_e32 v3, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s6
; GFX908-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB0_2
; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: .LBB0_4: ; %Flow1
; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT: v_readfirstlane_b32 s0, v1
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX908-NEXT: global_store_dword v[0:1], v0, off
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_f32:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[4:5], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: ; implicit-def: $vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB0_4
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX90A-NEXT: v_mov_b32_e32 v3, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s6
; GFX90A-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v5, v1
; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB0_2
; GFX90A-NEXT: ; %bb.3: ; %Flow
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: .LBB0_4: ; %Flow1
; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT: v_readfirstlane_b32 s0, v1
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX90A-NEXT: global_store_dword v[0:1], v0, off
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_f32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s4, exec_lo
; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB0_4
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s5
; GFX10-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v1
; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: s_cbranch_execnz .LBB0_2
; GFX10-NEXT: ; %bb.3: ; %Flow
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: .LBB0_4: ; %Flow1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX10-NEXT: global_store_dword v[0:1], v0, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB0_4
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s5, s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mul_f32 v2, 4.0, v1 :: v_dual_mov_b32 v1, s5
; GFX11-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_mov_b32_e32 v5, v1
; GFX11-NEXT: v_add_f32_e32 v4, v5, v2
; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v3, v[4:5], s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: s_cbranch_execnz .LBB0_2
; GFX11-NEXT: ; %bb.3: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: .LBB0_4: ; %Flow1
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
; GFX11-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX11-NEXT: v_add_f32_e32 v0, s0, v0
; GFX11-NEXT: global_store_b32 v[0:1], v0, off
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 seq_cst
store float %result, ptr addrspace(1) undef
ret void
}
define amdgpu_kernel void @global_atomic_fadd_ret_f32_ieee(ptr addrspace(1) %ptr) #2 {
; GFX900-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[4:5], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: ; implicit-def: $vgpr1
; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX900-NEXT: s_cbranch_execz .LBB1_4
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX900-NEXT: s_mov_b64 s[4:5], 0
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s6
; GFX900-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v5, v1
; GFX900-NEXT: v_add_f32_e32 v4, v5, v2
; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX900-NEXT: s_cbranch_execnz .LBB1_2
; GFX900-NEXT: ; %bb.3: ; %Flow
; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX900-NEXT: .LBB1_4: ; %Flow1
; GFX900-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX900-NEXT: v_readfirstlane_b32 s0, v1
; GFX900-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX900-NEXT: v_add_f32_e32 v0, s0, v0
; GFX900-NEXT: global_store_dword v[0:1], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[4:5], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: ; implicit-def: $vgpr1
; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX908-NEXT: s_cbranch_execz .LBB1_4
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX908-NEXT: v_mov_b32_e32 v3, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s6
; GFX908-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB1_2
; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: .LBB1_4: ; %Flow1
; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT: v_readfirstlane_b32 s0, v1
; GFX908-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX908-NEXT: v_add_f32_e32 v0, s0, v0
; GFX908-NEXT: global_store_dword v[0:1], v0, off
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[4:5], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: ; implicit-def: $vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB1_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v2, s4
; GFX90A-NEXT: v_mov_b32_e32 v1, 0
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f32 v1, v1, v2, s[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: .LBB1_2:
; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT: v_readfirstlane_b32 s0, v1
; GFX90A-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX90A-NEXT: v_add_f32_e32 v0, s0, v0
; GFX90A-NEXT: global_store_dword v[0:1], v0, off
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s4, exec_lo
; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB1_4
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s5
; GFX10-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v1
; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: s_cbranch_execnz .LBB1_2
; GFX10-NEXT: ; %bb.3: ; %Flow
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: .LBB1_4: ; %Flow1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
; GFX10-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX10-NEXT: v_add_f32_e32 v0, s0, v0
; GFX10-NEXT: global_store_dword v[0:1], v0, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s3, exec_lo
; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB1_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mul_f32 v1, 4.0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_atomic_add_f32 v1, v2, v1, s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: .LBB1_2:
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
; GFX11-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX11-NEXT: v_add_f32_e32 v0, s0, v0
; GFX11-NEXT: global_store_b32 v[0:1], v0, off
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
store float %result, ptr addrspace(1) undef
ret void
}
define amdgpu_kernel void @global_atomic_fadd_noret_f32(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_noret_f32:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[2:3], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX900-NEXT: s_cbranch_execz .LBB2_3
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX900-NEXT: s_mov_b64 s[2:3], 0
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s4
; GFX900-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_add_f32_e32 v0, v1, v2
; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX900-NEXT: v_mov_b32_e32 v1, v0
; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX900-NEXT: s_cbranch_execnz .LBB2_2
; GFX900-NEXT: .LBB2_3:
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_noret_f32:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[2:3], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_cbranch_execz .LBB2_2
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: v_mul_f32_e32 v1, 4.0, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: .LBB2_2:
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_noret_f32:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[2:3], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB2_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: v_mul_f32_e32 v1, 4.0, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: .LBB2_2:
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_noret_f32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s3, exec_lo
; GFX10-NEXT: s_mov_b32 s2, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB2_3
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s4
; GFX10-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_add_f32_e32 v0, v1, v2
; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v0
; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: s_cbranch_execnz .LBB2_2
; GFX10-NEXT: .LBB2_3:
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_noret_f32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: s_mov_b32 s3, exec_lo
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB2_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: .LBB2_2:
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
ret void
}
define amdgpu_kernel void @global_atomic_fadd_noret_f32_ieee(ptr addrspace(1) %ptr) #2 {
; GFX900-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[2:3], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX900-NEXT: s_cbranch_execz .LBB3_3
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX900-NEXT: s_mov_b64 s[2:3], 0
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s4
; GFX900-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_add_f32_e32 v0, v1, v2
; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX900-NEXT: v_mov_b32_e32 v1, v0
; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX900-NEXT: s_cbranch_execnz .LBB3_2
; GFX900-NEXT: .LBB3_3:
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[2:3], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_cbranch_execz .LBB3_2
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: v_mul_f32_e32 v1, 4.0, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: .LBB3_2:
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[2:3], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB3_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: v_mul_f32_e32 v1, 4.0, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: .LBB3_2:
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s3, exec_lo
; GFX10-NEXT: s_mov_b32 s2, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB3_3
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s4
; GFX10-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_add_f32_e32 v0, v1, v2
; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v0
; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: s_cbranch_execnz .LBB3_2
; GFX10-NEXT: .LBB3_3:
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: s_mov_b32 s3, exec_lo
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB3_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: .LBB3_2:
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
ret void
}
define amdgpu_kernel void @global_atomic_fadd_ret_f32_agent(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[4:5], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: ; implicit-def: $vgpr1
; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX900-NEXT: s_cbranch_execz .LBB4_4
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX900-NEXT: s_mov_b64 s[4:5], 0
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s6
; GFX900-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v5, v1
; GFX900-NEXT: v_add_f32_e32 v4, v5, v2
; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX900-NEXT: s_cbranch_execnz .LBB4_2
; GFX900-NEXT: ; %bb.3: ; %Flow
; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX900-NEXT: .LBB4_4: ; %Flow1
; GFX900-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX900-NEXT: v_readfirstlane_b32 s0, v1
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX900-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX900-NEXT: global_store_dword v[0:1], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[4:5], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: ; implicit-def: $vgpr1
; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX908-NEXT: s_cbranch_execz .LBB4_4
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX908-NEXT: v_mov_b32_e32 v3, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s6
; GFX908-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB4_2
; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: .LBB4_4: ; %Flow1
; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT: v_readfirstlane_b32 s0, v1
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX908-NEXT: global_store_dword v[0:1], v0, off
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[4:5], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: ; implicit-def: $vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB4_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v2, s4
; GFX90A-NEXT: v_mov_b32_e32 v1, 0
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f32 v1, v1, v2, s[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: .LBB4_2:
; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT: v_readfirstlane_b32 s0, v1
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX90A-NEXT: global_store_dword v[0:1], v0, off
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s4, exec_lo
; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB4_4
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s5
; GFX10-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v1
; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: s_cbranch_execnz .LBB4_2
; GFX10-NEXT: ; %bb.3: ; %Flow
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: .LBB4_4: ; %Flow1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX10-NEXT: global_store_dword v[0:1], v0, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s3, exec_lo
; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB4_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mul_f32 v1, 4.0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_atomic_add_f32 v1, v2, v1, s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: .LBB4_2:
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
; GFX11-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX11-NEXT: v_add_f32_e32 v0, s0, v0
; GFX11-NEXT: global_store_b32 v[0:1], v0, off
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
store float %result, ptr addrspace(1) undef
ret void
}
define amdgpu_kernel void @global_atomic_fadd_ret_f32_system(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_f32_system:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[4:5], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: ; implicit-def: $vgpr1
; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX900-NEXT: s_cbranch_execz .LBB5_4
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX900-NEXT: s_mov_b64 s[4:5], 0
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s6
; GFX900-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v5, v1
; GFX900-NEXT: v_add_f32_e32 v4, v5, v2
; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX900-NEXT: s_cbranch_execnz .LBB5_2
; GFX900-NEXT: ; %bb.3: ; %Flow
; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX900-NEXT: .LBB5_4: ; %Flow1
; GFX900-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX900-NEXT: v_readfirstlane_b32 s0, v1
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX900-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX900-NEXT: global_store_dword v[0:1], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_f32_system:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[4:5], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: ; implicit-def: $vgpr1
; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX908-NEXT: s_cbranch_execz .LBB5_4
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX908-NEXT: v_mov_b32_e32 v3, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s6
; GFX908-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB5_2
; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: .LBB5_4: ; %Flow1
; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT: v_readfirstlane_b32 s0, v1
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX908-NEXT: global_store_dword v[0:1], v0, off
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_f32_system:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[4:5], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: ; implicit-def: $vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB5_4
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX90A-NEXT: v_mov_b32_e32 v3, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s6
; GFX90A-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v5, v1
; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB5_2
; GFX90A-NEXT: ; %bb.3: ; %Flow
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: .LBB5_4: ; %Flow1
; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT: v_readfirstlane_b32 s0, v1
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX90A-NEXT: global_store_dword v[0:1], v0, off
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_f32_system:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s4, exec_lo
; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB5_4
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s5
; GFX10-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v1
; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: s_cbranch_execnz .LBB5_2
; GFX10-NEXT: ; %bb.3: ; %Flow
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: .LBB5_4: ; %Flow1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX10-NEXT: global_store_dword v[0:1], v0, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32_system:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB5_4
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s5, s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mul_f32 v2, 4.0, v1 :: v_dual_mov_b32 v1, s5
; GFX11-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_mov_b32_e32 v5, v1
; GFX11-NEXT: v_add_f32_e32 v4, v5, v2
; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v3, v[4:5], s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: s_cbranch_execnz .LBB5_2
; GFX11-NEXT: ; %bb.3: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: .LBB5_4: ; %Flow1
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
; GFX11-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX11-NEXT: v_add_f32_e32 v0, s0, v0
; GFX11-NEXT: global_store_b32 v[0:1], v0, off
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("one-as") seq_cst
store float %result, ptr addrspace(1) undef
ret void
}
define amdgpu_kernel void @global_atomic_fadd_ret_f32_wrong_subtarget(ptr addrspace(1) %ptr) #1 {
; GCN-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget:
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[4:5], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GCN-NEXT: s_cbranch_execz .LBB6_4
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GCN-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GCN-NEXT: s_mov_b64 s[4:5], 0
; GCN-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_load_dword s6, s[0:1], 0x0
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: .LBB6_2: ; %atomicrmw.start
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-NEXT: v_mov_b32_e32 v5, v1
; GCN-NEXT: v_add_f32_e32 v4, v5, v2
; GCN-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_wbinvl1_vol
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GCN-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN-NEXT: s_cbranch_execnz .LBB6_2
; GCN-NEXT: ; %bb.3: ; %Flow
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: .LBB6_4: ; %Flow1
; GCN-NEXT: s_or_b64 exec, exec, s[2:3]
; GCN-NEXT: v_readfirstlane_b32 s0, v1
; GCN-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GCN-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GCN-NEXT: global_store_dword v[0:1], v0, off
; GCN-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b64 s[4:5], exec
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX11-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX11-NEXT: s_cbranch_execz .LBB6_4
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX11-NEXT: s_mov_b64 s[4:5], 0
; GFX11-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_mov_b32_e32 v5, v1
; GFX11-NEXT: v_add_f32_e32 v4, v5, v2
; GFX11-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_wbinvl1_vol
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX11-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX11-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX11-NEXT: s_cbranch_execnz .LBB6_2
; GFX11-NEXT: ; %bb.3: ; %Flow
; GFX11-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX11-NEXT: .LBB6_4: ; %Flow1
; GFX11-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX11-NEXT: global_store_dword v[0:1], v0, off
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
store float %result, ptr addrspace(1) undef
ret void
}
define amdgpu_kernel void @global_atomic_fadd_noret_f32_wrong_subtarget(ptr addrspace(1) %ptr) #1 {
; GCN-LABEL: global_atomic_fadd_noret_f32_wrong_subtarget:
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[2:3], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB7_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GCN-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: v_mul_f32_e32 v1, 4.0, v1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_wbinvl1_vol
; GCN-NEXT: .LBB7_2:
; GCN-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_noret_f32_wrong_subtarget:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b64 s[2:3], exec
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX11-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX11-NEXT: s_cbranch_execz .LBB7_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: v_mul_f32_e32 v1, 4.0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_wbinvl1_vol
; GFX11-NEXT: .LBB7_2:
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
ret void
}
define amdgpu_kernel void @global_atomic_fadd_noret_f32_safe(ptr addrspace(1) %ptr) {
; GFX900-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[2:3], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX900-NEXT: s_cbranch_execz .LBB8_3
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX900-NEXT: s_mov_b64 s[2:3], 0
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s4
; GFX900-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_add_f32_e32 v0, v1, v2
; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX900-NEXT: v_mov_b32_e32 v1, v0
; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX900-NEXT: s_cbranch_execnz .LBB8_2
; GFX900-NEXT: .LBB8_3:
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[2:3], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_cbranch_execz .LBB8_3
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX908-NEXT: s_mov_b64 s[2:3], 0
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX908-NEXT: v_mov_b32_e32 v3, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s4
; GFX908-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_add_f32_e32 v0, v1, v2
; GFX908-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX908-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX908-NEXT: s_cbranch_execnz .LBB8_2
; GFX908-NEXT: .LBB8_3:
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[2:3], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB8_3
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX90A-NEXT: s_mov_b64 s[2:3], 0
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX90A-NEXT: v_mov_b32_e32 v3, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s4
; GFX90A-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_add_f32_e32 v0, v1, v2
; GFX90A-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX90A-NEXT: s_cbranch_execnz .LBB8_2
; GFX90A-NEXT: .LBB8_3:
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s3, exec_lo
; GFX10-NEXT: s_mov_b32 s2, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB8_3
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s4
; GFX10-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_add_f32_e32 v0, v1, v2
; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v0
; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: s_cbranch_execnz .LBB8_2
; GFX10-NEXT: .LBB8_3:
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s3, exec_lo
; GFX11-NEXT: s_mov_b32 s2, 0
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB8_3
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mul_f32 v2, 4.0, v0 :: v_dual_mov_b32 v1, s4
; GFX11-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_add_f32_e32 v0, v1, v2
; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
; GFX11-NEXT: s_cbranch_execnz .LBB8_2
; GFX11-NEXT: .LBB8_3:
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
ret void
}
define amdgpu_kernel void @infer_as_before_atomic(ptr addrspace(4) %arg) #0 {
; GFX900-LABEL: infer_as_before_atomic:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[2:3], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX900-NEXT: s_cbranch_execz .LBB9_3
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX900-NEXT: s_mov_b64 s[2:3], 0
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s4
; GFX900-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_add_f32_e32 v0, v1, v2
; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX900-NEXT: v_mov_b32_e32 v1, v0
; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX900-NEXT: s_cbranch_execnz .LBB9_2
; GFX900-NEXT: .LBB9_3:
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: infer_as_before_atomic:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[2:3], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_cbranch_execz .LBB9_2
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX908-NEXT: .LBB9_2:
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: infer_as_before_atomic:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[2:3], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB9_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX90A-NEXT: .LBB9_2:
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: infer_as_before_atomic:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s3, exec_lo
; GFX10-NEXT: s_mov_b32 s2, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB9_3
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s4
; GFX10-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_add_f32_e32 v0, v1, v2
; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v0
; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: s_cbranch_execnz .LBB9_2
; GFX10-NEXT: .LBB9_3:
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: infer_as_before_atomic:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: s_mov_b32 s3, exec_lo
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB9_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX11-NEXT: .LBB9_2:
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%load = load ptr, ptr addrspace(4) %arg
%v = atomicrmw fadd ptr %load, float 1.0 syncscope("agent-one-as") monotonic, align 4
ret void
}
define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX900-NEXT: s_mov_b64 s[0:1], 0
; GFX900-NEXT: v_mov_b32_e32 v0, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_and_b32 s2, s4, -4
; GFX900-NEXT: s_mov_b32 s3, s5
; GFX900-NEXT: s_load_dword s6, s[2:3], 0x0
; GFX900-NEXT: s_and_b32 s4, s4, 3
; GFX900-NEXT: s_lshl_b32 s4, s4, 3
; GFX900-NEXT: s_lshl_b32 s5, 0xffff, s4
; GFX900-NEXT: s_not_b32 s5, s5
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s6
; GFX900-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v2, v1
; GFX900-NEXT: v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX900-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX900-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX900-NEXT: v_and_or_b32 v1, v2, s5, v1
; GFX900-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX900-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX900-NEXT: s_cbranch_execnz .LBB10_1
; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX900-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX900-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX900-NEXT: global_store_short v[0:1], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX908-NEXT: s_mov_b64 s[0:1], 0
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_and_b32 s2, s4, -4
; GFX908-NEXT: s_mov_b32 s3, s5
; GFX908-NEXT: s_load_dword s6, s[2:3], 0x0
; GFX908-NEXT: s_and_b32 s4, s4, 3
; GFX908-NEXT: s_lshl_b32 s4, s4, 3
; GFX908-NEXT: s_lshl_b32 s5, 0xffff, s4
; GFX908-NEXT: s_not_b32 s5, s5
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s6
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v2, v1
; GFX908-NEXT: v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX908-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX908-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX908-NEXT: v_and_or_b32 v1, v2, s5, v1
; GFX908-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX908-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX908-NEXT: s_cbranch_execnz .LBB10_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX908-NEXT: global_store_short v[0:1], v0, off
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_and_b32 s2, s4, -4
; GFX90A-NEXT: s_mov_b32 s3, s5
; GFX90A-NEXT: s_load_dword s6, s[2:3], 0x0
; GFX90A-NEXT: s_and_b32 s4, s4, 3
; GFX90A-NEXT: s_lshl_b32 s4, s4, 3
; GFX90A-NEXT: s_lshl_b32 s5, 0xffff, s4
; GFX90A-NEXT: s_not_b32 s5, s5
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s6
; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: v_lshrrev_b32_sdwa v1, s4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX90A-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX90A-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX90A-NEXT: v_and_or_b32 v2, v3, s5, v1
; GFX90A-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX90A-NEXT: global_store_short v[0:1], v0, off
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_and_b32 s0, s2, -4
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_and_b32 s2, s2, 3
; GFX10-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX10-NEXT: s_lshl_b32 s2, s2, 3
; GFX10-NEXT: s_lshl_b32 s4, 0xffff, s2
; GFX10-NEXT: s_not_b32 s4, s4
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v2, v1
; GFX10-NEXT: v_lshrrev_b32_sdwa v1, s2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT: v_and_or_b32 v1, v2, s4, v1
; GFX10-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: s_cbranch_execnz .LBB10_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX10-NEXT: global_store_short v[0:1], v0, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s0, s2, -4
; GFX11-NEXT: s_mov_b32 s1, s3
; GFX11-NEXT: s_and_b32 s2, s2, 3
; GFX11-NEXT: s_load_b32 s3, s[0:1], 0x0
; GFX11-NEXT: s_lshl_b32 s2, s2, 3
; GFX11-NEXT: s_lshl_b32 s4, 0xffff, s2
; GFX11-NEXT: s_not_b32 s4, s4
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s3
; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_mov_b32_e32 v2, v1
; GFX11-NEXT: v_lshrrev_b32_e32 v1, s2, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v1, s2, v1
; GFX11-NEXT: v_and_or_b32 v1, v2, s4, v1
; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX11-NEXT: global_store_b16 v[0:1], v0, off
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, bfloat 4.0 syncscope("agent") seq_cst
store bfloat %result, ptr addrspace(1) undef
ret void
}
define amdgpu_kernel void @global_atomic_fadd_ret_bf16_system(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX900-NEXT: s_mov_b64 s[0:1], 0
; GFX900-NEXT: v_mov_b32_e32 v0, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_and_b32 s2, s4, -4
; GFX900-NEXT: s_mov_b32 s3, s5
; GFX900-NEXT: s_load_dword s6, s[2:3], 0x0
; GFX900-NEXT: s_and_b32 s4, s4, 3
; GFX900-NEXT: s_lshl_b32 s4, s4, 3
; GFX900-NEXT: s_lshl_b32 s5, 0xffff, s4
; GFX900-NEXT: s_not_b32 s5, s5
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s6
; GFX900-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v2, v1
; GFX900-NEXT: v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX900-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX900-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX900-NEXT: v_and_or_b32 v1, v2, s5, v1
; GFX900-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX900-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX900-NEXT: s_cbranch_execnz .LBB11_1
; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX900-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX900-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX900-NEXT: global_store_short v[0:1], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX908-NEXT: s_mov_b64 s[0:1], 0
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_and_b32 s2, s4, -4
; GFX908-NEXT: s_mov_b32 s3, s5
; GFX908-NEXT: s_load_dword s6, s[2:3], 0x0
; GFX908-NEXT: s_and_b32 s4, s4, 3
; GFX908-NEXT: s_lshl_b32 s4, s4, 3
; GFX908-NEXT: s_lshl_b32 s5, 0xffff, s4
; GFX908-NEXT: s_not_b32 s5, s5
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s6
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v2, v1
; GFX908-NEXT: v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX908-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX908-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX908-NEXT: v_and_or_b32 v1, v2, s5, v1
; GFX908-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX908-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX908-NEXT: global_store_short v[0:1], v0, off
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_and_b32 s2, s4, -4
; GFX90A-NEXT: s_mov_b32 s3, s5
; GFX90A-NEXT: s_load_dword s6, s[2:3], 0x0
; GFX90A-NEXT: s_and_b32 s4, s4, 3
; GFX90A-NEXT: s_lshl_b32 s4, s4, 3
; GFX90A-NEXT: s_lshl_b32 s5, 0xffff, s4
; GFX90A-NEXT: s_not_b32 s5, s5
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s6
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: v_lshrrev_b32_sdwa v1, s4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX90A-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX90A-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX90A-NEXT: v_and_or_b32 v2, v3, s5, v1
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX90A-NEXT: global_store_short v[0:1], v0, off
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_and_b32 s0, s2, -4
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_and_b32 s2, s2, 3
; GFX10-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX10-NEXT: s_lshl_b32 s2, s2, 3
; GFX10-NEXT: s_lshl_b32 s4, 0xffff, s2
; GFX10-NEXT: s_not_b32 s4, s4
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v2, v1
; GFX10-NEXT: v_lshrrev_b32_sdwa v1, s2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT: v_and_or_b32 v1, v2, s4, v1
; GFX10-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: s_cbranch_execnz .LBB11_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX10-NEXT: global_store_short v[0:1], v0, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s0, s2, -4
; GFX11-NEXT: s_mov_b32 s1, s3
; GFX11-NEXT: s_and_b32 s2, s2, 3
; GFX11-NEXT: s_load_b32 s3, s[0:1], 0x0
; GFX11-NEXT: s_lshl_b32 s2, s2, 3
; GFX11-NEXT: s_lshl_b32 s4, 0xffff, s2
; GFX11-NEXT: s_not_b32 s4, s4
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s3
; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_mov_b32_e32 v2, v1
; GFX11-NEXT: v_lshrrev_b32_e32 v1, s2, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v1, s2, v1
; GFX11-NEXT: v_and_or_b32 v1, v2, s4, v1
; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: s_cbranch_execnz .LBB11_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX11-NEXT: global_store_b16 v[0:1], v0, off
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, bfloat 4.0 syncscope("one-as") seq_cst
store bfloat %result, ptr addrspace(1) undef
ret void
}
attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" }
attributes #1 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "target-cpu"="gfx803" "target-features"="+atomic-fadd-no-rtn-insts" "amdgpu-unsafe-fp-atomics"="true" }
attributes #2 = { "amdgpu-unsafe-fp-atomics"="true" }
Reference in New Issue View Git Blame Copy Permalink