caio/clang-p2996
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
1daf2994de49d1ecba4bee4e6842aa8a564cbc96
clang-p2996/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll
Matt Arsenault b01adc6bed AMDGPU: Strengthen some bfloat tests 
Fix bitcast test, which was splitting apart phis intended to force
bitcasts that survive all the way to selection.

Disable the amdgpu-codegenprepare phi splitting, which defeats the technique
of using a phi to ensure a bitcast reaches all the way to selection. Also
add a variety of bfloat tests. These probably need revisiting to avoid the
cast folding into argument loads. Also round out set of bfloat bitcast and
ABI tests.

Add codegen tests for more bf16 operations The promotion of these works
contrary to the comment.
2023-12-20 19:33:45 +07:00

1668 lines
65 KiB
LLVM
Raw Blame History

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI %s
; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX7 %s
; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX8 %s
define float @lds_atomic_fadd_ret_f32(ptr addrspace(3) %ptr) nounwind {
; VI-LABEL: lds_atomic_fadd_ret_f32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, 4.0
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: ds_add_rtn_f32 v0, v0, v1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: lds_atomic_fadd_ret_f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 4.0
; GFX9-NEXT: ds_add_rtn_f32 v0, v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: lds_atomic_fadd_ret_f32:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v1, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, v1
; GFX7-NEXT: v_add_f32_e32 v1, 4.0, v2
; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB0_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: lds_atomic_fadd_ret_f32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v1, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, v1
; GFX8-NEXT: v_add_f32_e32 v1, 4.0, v2
; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB0_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(3) %ptr, float 4.0 seq_cst
ret float %result
}
define void @lds_atomic_fadd_noret_f32(ptr addrspace(3) %ptr) nounwind {
; VI-LABEL: lds_atomic_fadd_noret_f32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, 4.0
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: ds_add_f32 v0, v1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: lds_atomic_fadd_noret_f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 4.0
; GFX9-NEXT: ds_add_f32 v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: lds_atomic_fadd_noret_f32:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v1, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_add_f32_e32 v2, 4.0, v1
; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, v2
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB1_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: lds_atomic_fadd_noret_f32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v1, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_add_f32_e32 v2, 4.0, v1
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB1_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(3) %ptr, float 4.0 seq_cst
ret void
}
define amdgpu_kernel void @lds_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %ptrf, i32 %idx) {
; VI-LABEL: lds_ds_fadd:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c
; VI-NEXT: s_mov_b64 s[4:5], exec
; VI-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; VI-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_i32 s3, s3, 4
; VI-NEXT: ; implicit-def: $vgpr1
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_and_saveexec_b64 s[6:7], vcc
; VI-NEXT: s_cbranch_execz .LBB2_2
; VI-NEXT: ; %bb.1:
; VI-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; VI-NEXT: s_lshl_b32 s8, s3, 3
; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; VI-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; VI-NEXT: v_mov_b32_e32 v2, s8
; VI-NEXT: ds_add_rtn_f32 v1, v2, v1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: .LBB2_2:
; VI-NEXT: s_or_b64 exec, exec, s[6:7]
; VI-NEXT: s_mov_b64 s[6:7], exec
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0
; VI-NEXT: v_mbcnt_hi_u32_b32 v1, s7, v1
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_cbranch_execz .LBB2_4
; VI-NEXT: ; %bb.3:
; VI-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, s6
; VI-NEXT: s_lshl_b32 s3, s3, 4
; VI-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; VI-NEXT: v_mov_b32_e32 v2, s3
; VI-NEXT: ds_add_f32 v2, v1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: .LBB2_4:
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; VI-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; VI-NEXT: s_mov_b64 s[4:5], exec
; VI-NEXT: v_add_f32_e32 v2, s8, v0
; VI-NEXT: v_bfrev_b32_e32 v1, 1
; VI-NEXT: ; implicit-def: $vgpr0
; VI-NEXT: .LBB2_5: ; %ComputeLoop
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_ff1_i32_b32 s3, s5
; VI-NEXT: s_ff1_i32_b32 s6, s4
; VI-NEXT: s_add_i32 s3, s3, 32
; VI-NEXT: s_min_u32 s3, s6, s3
; VI-NEXT: s_lshl_b64 s[6:7], 1, s3
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: v_readlane_b32 s9, v2, s3
; VI-NEXT: s_mov_b32 m0, s3
; VI-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7]
; VI-NEXT: v_writelane_b32 v0, s8, m0
; VI-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-NEXT: v_add_f32_e32 v1, s9, v1
; VI-NEXT: s_cbranch_scc1 .LBB2_5
; VI-NEXT: ; %bb.6: ; %ComputeEnd
; VI-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; VI-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; VI-NEXT: ; implicit-def: $vgpr2
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_cbranch_execz .LBB2_8
; VI-NEXT: ; %bb.7:
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: ds_add_rtn_f32 v2, v2, v1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: .LBB2_8:
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s2, v2
; VI-NEXT: v_add_f32_e32 v2, s2, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: lds_ds_fadd:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c
; GFX9-NEXT: s_mov_b64 s[4:5], exec
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_add_i32 s3, s3, 4
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX9-NEXT: s_cbranch_execz .LBB2_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9-NEXT: s_lshl_b32 s8, s3, 3
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; GFX9-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s8
; GFX9-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB2_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-NEXT: s_mov_b64 s[6:7], exec
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, s7, v1
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB2_4
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, s6
; GFX9-NEXT: s_lshl_b32 s3, s3, 4
; GFX9-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s3
; GFX9-NEXT: ds_add_f32 v2, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB2_4:
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX9-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX9-NEXT: s_mov_b64 s[4:5], exec
; GFX9-NEXT: v_add_f32_e32 v2, s8, v0
; GFX9-NEXT: v_bfrev_b32_e32 v1, 1
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: .LBB2_5: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b32 s3, s5
; GFX9-NEXT: s_ff1_i32_b32 s6, s4
; GFX9-NEXT: s_add_i32 s3, s3, 32
; GFX9-NEXT: s_min_u32 s3, s6, s3
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s3
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: v_readlane_b32 s9, v2, s3
; GFX9-NEXT: s_mov_b32 m0, s3
; GFX9-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7]
; GFX9-NEXT: v_writelane_b32 v0, s8, m0
; GFX9-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-NEXT: v_add_f32_e32 v1, s9, v1
; GFX9-NEXT: s_cbranch_scc1 .LBB2_5
; GFX9-NEXT: ; %bb.6: ; %ComputeEnd
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr2
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB2_8
; GFX9-NEXT: ; %bb.7:
; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: ds_add_rtn_f32 v2, v2, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB2_8:
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s2, v2
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_add_f32_e32 v0, s2, v0
; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX7-LABEL: lds_ds_fadd:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xb
; GFX7-NEXT: s_mov_b64 s[6:7], exec
; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_i32 s3, s3, 4
; GFX7-NEXT: ; implicit-def: $vgpr1
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7-NEXT: s_cbranch_execz .LBB2_4
; GFX7-NEXT: ; %bb.1:
; GFX7-NEXT: s_lshl_b32 s8, s3, 3
; GFX7-NEXT: v_mov_b32_e32 v2, s8
; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s6
; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v4, v1
; GFX7-NEXT: v_add_f32_e32 v1, v4, v3
; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB2_2
; GFX7-NEXT: ; %bb.3: ; %Flow15
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: .LBB2_4: ; %Flow16
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_mov_b64 s[6:7], exec
; GFX7-NEXT: v_readfirstlane_b32 s8, v1
; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s6, 0
; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s7, v1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7-NEXT: s_cbranch_execz .LBB2_7
; GFX7-NEXT: ; %bb.5:
; GFX7-NEXT: s_lshl_b32 s3, s3, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s3
; GFX7-NEXT: ds_read_b32 v3, v1
; GFX7-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB2_6: ; %atomicrmw.start2
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB2_6
; GFX7-NEXT: .LBB2_7: ; %Flow14
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v2, s2
; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX7-NEXT: v_add_f32_e32 v0, s8, v0
; GFX7-NEXT: s_mov_b64 s[2:3], 0
; GFX7-NEXT: .LBB2_8: ; %atomicrmw.start8
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: v_add_f32_e32 v1, v3, v0
; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX7-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_cbranch_execnz .LBB2_8
; GFX7-NEXT: ; %bb.9: ; %atomicrmw.end7
; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_store_dword v1, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: lds_ds_fadd:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xb
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_i32 s3, s3, 4
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB2_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_lshl_b32 s8, s3, 3
; GFX8-NEXT: v_mov_b32_e32 v2, s8
; GFX8-NEXT: ds_read_b32 v1, v2
; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v3, s6
; GFX8-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v1
; GFX8-NEXT: v_add_f32_e32 v1, v4, v3
; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB2_2
; GFX8-NEXT: ; %bb.3: ; %Flow17
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: .LBB2_4: ; %Flow18
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: v_readfirstlane_b32 s8, v1
; GFX8-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s6, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s7, v1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB2_7
; GFX8-NEXT: ; %bb.5:
; GFX8-NEXT: s_lshl_b32 s3, s3, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: ds_read_b32 v3, v1
; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB2_6: ; %atomicrmw.start2
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_add_f32_e32 v4, v3, v2
; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB2_6
; GFX8-NEXT: .LBB2_7: ; %Flow16
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, s2
; GFX8-NEXT: ds_read_b32 v1, v2
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX8-NEXT: v_add_f32_e32 v0, s8, v0
; GFX8-NEXT: s_mov_b64 s[2:3], 0
; GFX8-NEXT: .LBB2_8: ; %atomicrmw.start8
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_add_f32_e32 v1, v3, v0
; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX8-NEXT: s_cbranch_execnz .LBB2_8
; GFX8-NEXT: ; %bb.9: ; %atomicrmw.end7
; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: buffer_store_dword v1, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
%idx.add = add nuw i32 %idx, 4
%shl0 = shl i32 %idx.add, 3
%shl1 = shl i32 %idx.add, 4
%ptr0 = inttoptr i32 %shl0 to ptr addrspace(3)
%ptr1 = inttoptr i32 %shl1 to ptr addrspace(3)
%a1 = atomicrmw fadd ptr addrspace(3) %ptr0, float 4.2e+1 seq_cst
%a2 = atomicrmw fadd ptr addrspace(3) %ptr1, float 4.2e+1 seq_cst
%a3 = atomicrmw fadd ptr addrspace(3) %ptrf, float %a1 seq_cst
store float %a3, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @lds_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspace(3) %ptrf, i32 %idx) {
; VI-LABEL: lds_ds_fadd_one_as:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c
; VI-NEXT: s_mov_b64 s[4:5], exec
; VI-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; VI-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_i32 s3, s3, 4
; VI-NEXT: ; implicit-def: $vgpr1
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_and_saveexec_b64 s[6:7], vcc
; VI-NEXT: s_cbranch_execz .LBB3_2
; VI-NEXT: ; %bb.1:
; VI-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; VI-NEXT: s_lshl_b32 s8, s3, 3
; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; VI-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; VI-NEXT: v_mov_b32_e32 v2, s8
; VI-NEXT: ds_add_rtn_f32 v1, v2, v1
; VI-NEXT: .LBB3_2:
; VI-NEXT: s_or_b64 exec, exec, s[6:7]
; VI-NEXT: s_mov_b64 s[6:7], exec
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0
; VI-NEXT: v_mbcnt_hi_u32_b32 v1, s7, v1
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_cbranch_execz .LBB3_4
; VI-NEXT: ; %bb.3:
; VI-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; VI-NEXT: v_cvt_f32_ubyte0_e32 v1, s6
; VI-NEXT: s_lshl_b32 s3, s3, 4
; VI-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; VI-NEXT: v_mov_b32_e32 v2, s3
; VI-NEXT: ds_add_f32 v2, v1
; VI-NEXT: .LBB3_4:
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; VI-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; VI-NEXT: s_mov_b64 s[4:5], exec
; VI-NEXT: v_add_f32_e32 v2, s8, v0
; VI-NEXT: v_bfrev_b32_e32 v1, 1
; VI-NEXT: ; implicit-def: $vgpr0
; VI-NEXT: .LBB3_5: ; %ComputeLoop
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_ff1_i32_b32 s3, s5
; VI-NEXT: s_ff1_i32_b32 s6, s4
; VI-NEXT: s_add_i32 s3, s3, 32
; VI-NEXT: s_min_u32 s3, s6, s3
; VI-NEXT: s_lshl_b64 s[6:7], 1, s3
; VI-NEXT: v_readfirstlane_b32 s8, v1
; VI-NEXT: v_readlane_b32 s9, v2, s3
; VI-NEXT: s_mov_b32 m0, s3
; VI-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7]
; VI-NEXT: v_writelane_b32 v0, s8, m0
; VI-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-NEXT: v_add_f32_e32 v1, s9, v1
; VI-NEXT: s_cbranch_scc1 .LBB3_5
; VI-NEXT: ; %bb.6: ; %ComputeEnd
; VI-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; VI-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; VI-NEXT: ; implicit-def: $vgpr2
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; VI-NEXT: s_cbranch_execz .LBB3_8
; VI-NEXT: ; %bb.7:
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: ds_add_rtn_f32 v2, v2, v1
; VI-NEXT: .LBB3_8:
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_readfirstlane_b32 s2, v2
; VI-NEXT: v_add_f32_e32 v2, s2, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: lds_ds_fadd_one_as:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c
; GFX9-NEXT: s_mov_b64 s[4:5], exec
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_add_i32 s3, s3, 4
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX9-NEXT: s_cbranch_execz .LBB3_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9-NEXT: s_lshl_b32 s8, s3, 3
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; GFX9-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s8
; GFX9-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX9-NEXT: .LBB3_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-NEXT: s_mov_b64 s[6:7], exec
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, s7, v1
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB3_4
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, s6
; GFX9-NEXT: s_lshl_b32 s3, s3, 4
; GFX9-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s3
; GFX9-NEXT: ds_add_f32 v2, v1
; GFX9-NEXT: .LBB3_4:
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX9-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX9-NEXT: s_mov_b64 s[4:5], exec
; GFX9-NEXT: v_add_f32_e32 v2, s8, v0
; GFX9-NEXT: v_bfrev_b32_e32 v1, 1
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: .LBB3_5: ; %ComputeLoop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b32 s3, s5
; GFX9-NEXT: s_ff1_i32_b32 s6, s4
; GFX9-NEXT: s_add_i32 s3, s3, 32
; GFX9-NEXT: s_min_u32 s3, s6, s3
; GFX9-NEXT: s_lshl_b64 s[6:7], 1, s3
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
; GFX9-NEXT: v_readlane_b32 s9, v2, s3
; GFX9-NEXT: s_mov_b32 m0, s3
; GFX9-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7]
; GFX9-NEXT: v_writelane_b32 v0, s8, m0
; GFX9-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-NEXT: v_add_f32_e32 v1, s9, v1
; GFX9-NEXT: s_cbranch_scc1 .LBB3_5
; GFX9-NEXT: ; %bb.6: ; %ComputeEnd
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr2
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-NEXT: s_cbranch_execz .LBB3_8
; GFX9-NEXT: ; %bb.7:
; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: ds_add_rtn_f32 v2, v2, v1
; GFX9-NEXT: .LBB3_8:
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s2, v2
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_add_f32_e32 v0, s2, v0
; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX7-LABEL: lds_ds_fadd_one_as:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xb
; GFX7-NEXT: s_mov_b64 s[6:7], exec
; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_i32 s3, s3, 4
; GFX7-NEXT: ; implicit-def: $vgpr1
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7-NEXT: s_cbranch_execz .LBB3_4
; GFX7-NEXT: ; %bb.1:
; GFX7-NEXT: s_lshl_b32 s8, s3, 3
; GFX7-NEXT: v_mov_b32_e32 v2, s8
; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s6
; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v4, v1
; GFX7-NEXT: v_add_f32_e32 v1, v4, v3
; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB3_2
; GFX7-NEXT: ; %bb.3: ; %Flow15
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: .LBB3_4: ; %Flow16
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_mov_b64 s[6:7], exec
; GFX7-NEXT: v_readfirstlane_b32 s8, v1
; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s6, 0
; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s7, v1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7-NEXT: s_cbranch_execz .LBB3_7
; GFX7-NEXT: ; %bb.5:
; GFX7-NEXT: s_lshl_b32 s3, s3, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s3
; GFX7-NEXT: ds_read_b32 v3, v1
; GFX7-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB3_6: ; %atomicrmw.start2
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB3_6
; GFX7-NEXT: .LBB3_7: ; %Flow14
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v2, s2
; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX7-NEXT: v_add_f32_e32 v0, s8, v0
; GFX7-NEXT: s_mov_b64 s[2:3], 0
; GFX7-NEXT: .LBB3_8: ; %atomicrmw.start8
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: v_add_f32_e32 v1, v3, v0
; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX7-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_cbranch_execnz .LBB3_8
; GFX7-NEXT: ; %bb.9: ; %atomicrmw.end7
; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: buffer_store_dword v1, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: lds_ds_fadd_one_as:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xb
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_i32 s3, s3, 4
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB3_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_lshl_b32 s8, s3, 3
; GFX8-NEXT: v_mov_b32_e32 v2, s8
; GFX8-NEXT: ds_read_b32 v1, v2
; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v3, s6
; GFX8-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v1
; GFX8-NEXT: v_add_f32_e32 v1, v4, v3
; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB3_2
; GFX8-NEXT: ; %bb.3: ; %Flow17
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: .LBB3_4: ; %Flow18
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: v_readfirstlane_b32 s8, v1
; GFX8-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s6, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s7, v1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB3_7
; GFX8-NEXT: ; %bb.5:
; GFX8-NEXT: s_lshl_b32 s3, s3, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: ds_read_b32 v3, v1
; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB3_6: ; %atomicrmw.start2
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_add_f32_e32 v4, v3, v2
; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB3_6
; GFX8-NEXT: .LBB3_7: ; %Flow16
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, s2
; GFX8-NEXT: ds_read_b32 v1, v2
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX8-NEXT: v_add_f32_e32 v0, s8, v0
; GFX8-NEXT: s_mov_b64 s[2:3], 0
; GFX8-NEXT: .LBB3_8: ; %atomicrmw.start8
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_add_f32_e32 v1, v3, v0
; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX8-NEXT: s_cbranch_execnz .LBB3_8
; GFX8-NEXT: ; %bb.9: ; %atomicrmw.end7
; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: buffer_store_dword v1, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
%idx.add = add nuw i32 %idx, 4
%shl0 = shl i32 %idx.add, 3
%shl1 = shl i32 %idx.add, 4
%ptr0 = inttoptr i32 %shl0 to ptr addrspace(3)
%ptr1 = inttoptr i32 %shl1 to ptr addrspace(3)
%a1 = atomicrmw fadd ptr addrspace(3) %ptr0, float 4.2e+1 syncscope("one-as") seq_cst
%a2 = atomicrmw fadd ptr addrspace(3) %ptr1, float 4.2e+1 syncscope("one-as") seq_cst
%a3 = atomicrmw fadd ptr addrspace(3) %ptrf, float %a1 syncscope("one-as") seq_cst
store float %a3, ptr addrspace(1) %out
ret void
}
define double @lds_atomic_fadd_ret_f64(ptr addrspace(3) %ptr) nounwind {
; VI-LABEL: lds_atomic_fadd_ret_f64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: v_mov_b32_e32 v2, v0
; VI-NEXT: ds_read_b64 v[0:1], v0
; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: .LBB4_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v1
; VI-NEXT: v_mov_b32_e32 v3, v0
; VI-NEXT: v_add_f64 v[0:1], v[3:4], 4.0
; VI-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
; VI-NEXT: s_cbranch_execnz .LBB4_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: lds_atomic_fadd_ret_f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, v0
; GFX9-NEXT: ds_read_b64 v[0:1], v0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, v1
; GFX9-NEXT: v_mov_b32_e32 v3, v0
; GFX9-NEXT: v_add_f64 v[0:1], v[3:4], 4.0
; GFX9-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB4_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: lds_atomic_fadd_ret_f64:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: v_mov_b32_e32 v2, v0
; GFX7-NEXT: ds_read_b64 v[0:1], v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v4, v1
; GFX7-NEXT: v_mov_b32_e32 v3, v0
; GFX7-NEXT: v_add_f64 v[0:1], v[3:4], 4.0
; GFX7-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB4_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: lds_atomic_fadd_ret_f64:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: ds_read_b64 v[0:1], v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v1
; GFX8-NEXT: v_mov_b32_e32 v3, v0
; GFX8-NEXT: v_add_f64 v[0:1], v[3:4], 4.0
; GFX8-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB4_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst
ret double %result
}
define void @lds_atomic_fadd_noret_f64(ptr addrspace(3) %ptr) nounwind {
; VI-LABEL: lds_atomic_fadd_noret_f64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: ds_read_b64 v[1:2], v0
; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: .LBB5_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_add_f64 v[3:4], v[1:2], 4.0
; VI-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4]
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]
; VI-NEXT: v_mov_b32_e32 v1, v3
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v2, v4
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
; VI-NEXT: s_cbranch_execnz .LBB5_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: lds_atomic_fadd_noret_f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_b64 v[1:2], v0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], 4.0
; GFX9-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]
; GFX9-NEXT: v_mov_b32_e32 v1, v3
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v4
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB5_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: lds_atomic_fadd_noret_f64:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b64 v[1:2], v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_add_f64 v[3:4], v[1:2], 4.0
; GFX7-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4]
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]
; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v2, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB5_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: lds_atomic_fadd_noret_f64:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b64 v[1:2], v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_add_f64 v[3:4], v[1:2], 4.0
; GFX8-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]
; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB5_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst
ret void
}
define float @lds_atomic_fsub_ret_f32(ptr addrspace(3) %ptr, float %val) nounwind {
; VI-LABEL: lds_atomic_fsub_ret_f32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: ds_read_b32 v2, v0
; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: .LBB6_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: v_sub_f32_e32 v2, v3, v1
; VI-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
; VI-NEXT: s_cbranch_execnz .LBB6_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: v_mov_b32_e32 v0, v2
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: lds_atomic_fsub_ret_f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_b32 v2, v0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v2
; GFX9-NEXT: v_sub_f32_e32 v2, v3, v1
; GFX9-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB6_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: lds_atomic_fsub_ret_f32:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v2, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v2
; GFX7-NEXT: v_sub_f32_e32 v2, v3, v1
; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB6_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: lds_atomic_fsub_ret_f32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v2, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: v_sub_f32_e32 v2, v3, v1
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB6_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fsub ptr addrspace(3) %ptr, float %val seq_cst
ret float %result
}
define void @lds_atomic_fsub_noret_f32(ptr addrspace(3) %ptr, float %val) nounwind {
; VI-LABEL: lds_atomic_fsub_noret_f32:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: ds_read_b32 v2, v0
; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: .LBB7_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_sub_f32_e32 v3, v2, v1
; VI-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v2, v3
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
; VI-NEXT: s_cbranch_execnz .LBB7_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: lds_atomic_fsub_noret_f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_b32 v2, v0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_sub_f32_e32 v3, v2, v1
; GFX9-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v3
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB7_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: lds_atomic_fsub_noret_f32:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v2, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_sub_f32_e32 v3, v2, v1
; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v2, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB7_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: lds_atomic_fsub_noret_f32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v2, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_sub_f32_e32 v3, v2, v1
; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB7_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fsub ptr addrspace(3) %ptr, float %val seq_cst
ret void
}
define double @lds_atomic_fsub_ret_f64(ptr addrspace(3) %ptr, double %val) nounwind {
; VI-LABEL: lds_atomic_fsub_ret_f64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: ds_read_b64 v[3:4], v0
; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: .LBB8_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v6, v4
; VI-NEXT: v_mov_b32_e32 v5, v3
; VI-NEXT: v_add_f64 v[3:4], v[5:6], -v[1:2]
; VI-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[5:6], v[3:4]
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
; VI-NEXT: s_cbranch_execnz .LBB8_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: v_mov_b32_e32 v0, v3
; VI-NEXT: v_mov_b32_e32 v1, v4
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: lds_atomic_fsub_ret_f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_b64 v[3:4], v0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v6, v4
; GFX9-NEXT: v_mov_b32_e32 v5, v3
; GFX9-NEXT: v_add_f64 v[3:4], v[5:6], -v[1:2]
; GFX9-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[5:6], v[3:4]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB8_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, v3
; GFX9-NEXT: v_mov_b32_e32 v1, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: lds_atomic_fsub_ret_f64:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b64 v[3:4], v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: v_mov_b32_e32 v5, v3
; GFX7-NEXT: v_add_f64 v[3:4], v[5:6], -v[1:2]
; GFX7-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[5:6], v[3:4]
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB8_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: v_mov_b32_e32 v1, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: lds_atomic_fsub_ret_f64:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b64 v[3:4], v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v4
; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: v_add_f64 v[3:4], v[5:6], -v[1:2]
; GFX8-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[5:6], v[3:4]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6]
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB8_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v0, v3
; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fsub ptr addrspace(3) %ptr, double %val seq_cst
ret double %result
}
define void @lds_atomic_fsub_noret_f64(ptr addrspace(3) %ptr, double %val) nounwind {
; VI-LABEL: lds_atomic_fsub_noret_f64:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: ds_read_b64 v[3:4], v0
; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: .LBB9_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_add_f64 v[5:6], v[3:4], -v[1:2]
; VI-NEXT: ds_cmpst_rtn_b64 v[5:6], v0, v[3:4], v[5:6]
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[3:4]
; VI-NEXT: v_mov_b32_e32 v3, v5
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v4, v6
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
; VI-NEXT: s_cbranch_execnz .LBB9_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: lds_atomic_fsub_noret_f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_b64 v[3:4], v0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_add_f64 v[5:6], v[3:4], -v[1:2]
; GFX9-NEXT: ds_cmpst_rtn_b64 v[5:6], v0, v[3:4], v[5:6]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[3:4]
; GFX9-NEXT: v_mov_b32_e32 v3, v5
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v4, v6
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB9_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: lds_atomic_fsub_noret_f64:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b64 v[3:4], v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_add_f64 v[5:6], v[3:4], -v[1:2]
; GFX7-NEXT: ds_cmpst_rtn_b64 v[5:6], v0, v[3:4], v[5:6]
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[3:4]
; GFX7-NEXT: v_mov_b32_e32 v3, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v4, v6
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB9_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: lds_atomic_fsub_noret_f64:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b64 v[3:4], v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_add_f64 v[5:6], v[3:4], -v[1:2]
; GFX8-NEXT: ds_cmpst_rtn_b64 v[5:6], v0, v[3:4], v[5:6]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[3:4]
; GFX8-NEXT: v_mov_b32_e32 v3, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v4, v6
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fsub ptr addrspace(3) %ptr, double %val seq_cst
ret void
}
define bfloat @lds_atomic_fadd_ret_bf16(ptr addrspace(3) %ptr) nounwind {
; VI-LABEL: lds_atomic_fadd_ret_bf16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_and_b32_e32 v1, -4, v0
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: ds_read_b32 v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; VI-NEXT: s_mov_b32 s4, 0xffff
; VI-NEXT: v_and_b32_e32 v0, 24, v2
; VI-NEXT: v_lshlrev_b32_e64 v2, v2, s4
; VI-NEXT: v_not_b32_e32 v2, v2
; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: .LBB10_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v4, v3
; VI-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f32_e32 v3, 4.0, v3
; VI-NEXT: v_and_b32_e32 v5, v4, v2
; VI-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; VI-NEXT: v_or_b32_e32 v3, v5, v3
; VI-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
; VI-NEXT: s_cbranch_execnz .LBB10_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: v_lshrrev_b32_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: lds_atomic_fadd_ret_bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v1, -4, v0
; GFX9-NEXT: ds_read_b32 v2, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; GFX9-NEXT: s_mov_b32 s4, 0xffff
; GFX9-NEXT: v_and_b32_e32 v0, 24, v3
; GFX9-NEXT: v_lshlrev_b32_e64 v3, v3, s4
; GFX9-NEXT: v_not_b32_e32 v3, v3
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, v2
; GFX9-NEXT: v_lshrrev_b32_sdwa v2, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_f32_e32 v2, 4.0, v2
; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_and_or_b32 v2, v4, v3, v2
; GFX9-NEXT: ds_cmpst_rtn_b32 v2, v1, v4, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB10_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_lshrrev_b32_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: lds_atomic_fadd_ret_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, -4, v0
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v3, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX7-NEXT: v_and_b32_e32 v0, 24, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v2
; GFX7-NEXT: v_not_b32_e32 v2, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v4, v3
; GFX7-NEXT: v_lshrrev_b32_e32 v3, v0, v4
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_add_f32_e32 v3, 4.0, v3
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX7-NEXT: v_and_b32_e32 v5, v4, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v3, v0, v3
; GFX7-NEXT: v_or_b32_e32 v3, v5, v3
; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB10_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v0, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: lds_atomic_fadd_ret_bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v1, -4, v0
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v3, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX8-NEXT: v_and_b32_e32 v0, 24, v2
; GFX8-NEXT: v_lshl_b32_e32 v2, 0xffff, v2
; GFX8-NEXT: v_not_b32_e32 v2, v2
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v3, v0, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_add_f32_e32 v3, 4.0, v3
; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX8-NEXT: v_and_b32_e32 v5, v4, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v3, v0, v3
; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB10_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v0, v0, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(3) %ptr, bfloat 4.0 seq_cst
ret bfloat %result
}
define void @lds_atomic_fadd_noret_bf16(ptr addrspace(3) %ptr) nounwind {
; VI-LABEL: lds_atomic_fadd_noret_bf16:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_and_b32_e32 v1, -4, v0
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: ds_read_b32 v3, v1
; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; VI-NEXT: s_mov_b32 s4, 0xffff
; VI-NEXT: v_and_b32_e32 v0, 24, v2
; VI-NEXT: v_lshlrev_b32_e64 v2, v2, s4
; VI-NEXT: v_not_b32_e32 v2, v2
; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: .LBB11_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; VI-NEXT: v_add_f32_e32 v4, 4.0, v4
; VI-NEXT: v_and_b32_e32 v5, v3, v2
; VI-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; VI-NEXT: v_or_b32_e32 v4, v5, v4
; VI-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v3, v4
; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
; VI-NEXT: s_cbranch_execnz .LBB11_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: lds_atomic_fadd_noret_bf16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v1, -4, v0
; GFX9-NEXT: ds_read_b32 v3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX9-NEXT: s_mov_b32 s4, 0xffff
; GFX9-NEXT: v_and_b32_e32 v0, 24, v2
; GFX9-NEXT: v_lshlrev_b32_e64 v2, v2, s4
; GFX9-NEXT: v_not_b32_e32 v2, v2
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_add_f32_e32 v4, 4.0, v4
; GFX9-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT: v_and_or_b32 v4, v3, v2, v4
; GFX9-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v3, v4
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB11_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: lds_atomic_fadd_noret_bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, -4, v0
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v3, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX7-NEXT: v_and_b32_e32 v0, 24, v2
; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v2
; GFX7-NEXT: v_not_b32_e32 v2, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_lshrrev_b32_e32 v4, v0, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX7-NEXT: v_add_f32_e32 v4, 4.0, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX7-NEXT: v_and_b32_e32 v5, v3, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v4, v0, v4
; GFX7-NEXT: v_or_b32_e32 v4, v5, v4
; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB11_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: lds_atomic_fadd_noret_bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v1, -4, v0
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v3, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX8-NEXT: v_and_b32_e32 v0, 24, v2
; GFX8-NEXT: v_lshl_b32_e32 v2, 0xffff, v2
; GFX8-NEXT: v_not_b32_e32 v2, v2
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v4, v0, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX8-NEXT: v_add_f32_e32 v4, 4.0, v4
; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX8-NEXT: v_and_b32_e32 v5, v3, v2
; GFX8-NEXT: v_lshlrev_b32_e32 v4, v0, v4
; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(3) %ptr, bfloat 4.0 seq_cst
ret void
}
Reference in New Issue View Git Blame Copy Permalink