1d0370872f
If the runtime flat address resolves to a scratch address, 64-bit atomics do not work correctly. Insert a runtime address space check (which is quite likely to be uniform) and select between the non-atomic and real atomic cases. Consider noalias.addrspace metadata and avoid this expansion when possible (we also need to consider it to avoid infinitely expanding after adding the predication code).
16773 lines
698 KiB
LLVM
16773 lines
698 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
; RUN: llc -mtriple=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=GCN1 %s
|
|
; RUN: llc -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN2 %s
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN3 %s
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw xchg
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_xchg_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB0_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB0_4
|
|
; GCN1-NEXT: .LBB0_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB0_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB0_2
|
|
; GCN1-NEXT: .LBB0_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB0_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB0_4
|
|
; GCN2-NEXT: .LBB0_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB0_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB0_2
|
|
; GCN2-NEXT: .LBB0_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB0_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB0_4
|
|
; GCN3-NEXT: .LBB0_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB0_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB0_2
|
|
; GCN3-NEXT: .LBB0_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw xchg ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_xchg_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB1_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB1_4
|
|
; GCN1-NEXT: .LBB1_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB1_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB1_2
|
|
; GCN1-NEXT: .LBB1_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB1_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB1_4
|
|
; GCN2-NEXT: .LBB1_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB1_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB1_2
|
|
; GCN2-NEXT: .LBB1_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB1_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB1_4
|
|
; GCN3-NEXT: .LBB1_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB1_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB1_2
|
|
; GCN3-NEXT: .LBB1_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_xchg_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB2_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB2_4
|
|
; GCN1-NEXT: .LBB2_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB2_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB2_2
|
|
; GCN1-NEXT: .LBB2_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB2_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB2_4
|
|
; GCN2-NEXT: .LBB2_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB2_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB2_2
|
|
; GCN2-NEXT: .LBB2_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB2_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB2_4
|
|
; GCN3-NEXT: .LBB2_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB2_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB2_2
|
|
; GCN3-NEXT: .LBB2_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_nop 0
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw xchg ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_xchg_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB3_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB3_4
|
|
; GCN1-NEXT: .LBB3_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB3_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB3_2
|
|
; GCN1-NEXT: .LBB3_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB3_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB3_4
|
|
; GCN2-NEXT: .LBB3_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB3_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB3_2
|
|
; GCN2-NEXT: .LBB3_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB3_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB3_4
|
|
; GCN3-NEXT: .LBB3_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB3_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB3_2
|
|
; GCN3-NEXT: .LBB3_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_nop 0
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw xchg ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_xchg_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB4_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB4_4
|
|
; GCN1-NEXT: .LBB4_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB4_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB4_2
|
|
; GCN1-NEXT: .LBB4_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB4_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB4_4
|
|
; GCN2-NEXT: .LBB4_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB4_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB4_2
|
|
; GCN2-NEXT: .LBB4_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB4_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB4_4
|
|
; GCN3-NEXT: .LBB4_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB4_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB4_2
|
|
; GCN3-NEXT: .LBB4_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw xchg ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_xchg_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB5_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB5_4
|
|
; GCN1-NEXT: .LBB5_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB5_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB5_2
|
|
; GCN1-NEXT: .LBB5_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB5_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB5_4
|
|
; GCN2-NEXT: .LBB5_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB5_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB5_2
|
|
; GCN2-NEXT: .LBB5_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB5_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB5_4
|
|
; GCN3-NEXT: .LBB5_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB5_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB5_2
|
|
; GCN3-NEXT: .LBB5_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB6_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB6_3
|
|
; GCN1-NEXT: s_branch .LBB6_4
|
|
; GCN1-NEXT: .LBB6_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB6_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB6_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB6_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB6_3
|
|
; GCN2-NEXT: s_branch .LBB6_4
|
|
; GCN2-NEXT: .LBB6_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB6_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB6_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB6_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB6_3
|
|
; GCN3-NEXT: s_branch .LBB6_4
|
|
; GCN3-NEXT: .LBB6_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB6_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s6
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB6_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw xchg ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB7_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB7_3
|
|
; GCN1-NEXT: s_branch .LBB7_4
|
|
; GCN1-NEXT: .LBB7_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB7_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB7_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB7_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB7_3
|
|
; GCN2-NEXT: s_branch .LBB7_4
|
|
; GCN2-NEXT: .LBB7_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB7_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB7_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB7_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB7_3
|
|
; GCN3-NEXT: s_branch .LBB7_4
|
|
; GCN3-NEXT: .LBB7_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB7_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s6
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB7_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw xchg ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB8_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB8_4
|
|
; GCN1-NEXT: .LBB8_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB8_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB8_2
|
|
; GCN1-NEXT: .LBB8_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB8_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB8_4
|
|
; GCN2-NEXT: .LBB8_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB8_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB8_2
|
|
; GCN2-NEXT: .LBB8_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB8_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB8_4
|
|
; GCN3-NEXT: .LBB8_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB8_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB8_2
|
|
; GCN3-NEXT: .LBB8_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB9_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB9_4
|
|
; GCN1-NEXT: .LBB9_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB9_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB9_2
|
|
; GCN1-NEXT: .LBB9_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB9_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB9_4
|
|
; GCN2-NEXT: .LBB9_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB9_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB9_2
|
|
; GCN2-NEXT: .LBB9_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB9_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB9_4
|
|
; GCN3-NEXT: .LBB9_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB9_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB9_2
|
|
; GCN3-NEXT: .LBB9_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_nop 0
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw xchg ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw xchg f64
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_xchg_f64_noret(ptr %ptr, double %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB10_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB10_4
|
|
; GCN1-NEXT: .LBB10_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB10_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB10_2
|
|
; GCN1-NEXT: .LBB10_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB10_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB10_4
|
|
; GCN2-NEXT: .LBB10_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB10_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB10_2
|
|
; GCN2-NEXT: .LBB10_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB10_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB10_4
|
|
; GCN3-NEXT: .LBB10_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB10_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB10_2
|
|
; GCN3-NEXT: .LBB10_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw xchg ptr %ptr, double %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_xchg_f64_noret_offset(ptr %out, double %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB11_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB11_4
|
|
; GCN1-NEXT: .LBB11_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB11_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB11_2
|
|
; GCN1-NEXT: .LBB11_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB11_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB11_4
|
|
; GCN2-NEXT: .LBB11_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB11_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB11_2
|
|
; GCN2-NEXT: .LBB11_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB11_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB11_4
|
|
; GCN3-NEXT: .LBB11_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB11_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB11_2
|
|
; GCN3-NEXT: .LBB11_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr double, ptr %out, i32 4
|
|
%tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define double @flat_atomic_xchg_f64_ret(ptr %ptr, double %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB12_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB12_4
|
|
; GCN1-NEXT: .LBB12_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB12_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB12_2
|
|
; GCN1-NEXT: .LBB12_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB12_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB12_4
|
|
; GCN2-NEXT: .LBB12_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB12_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB12_2
|
|
; GCN2-NEXT: .LBB12_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB12_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB12_4
|
|
; GCN3-NEXT: .LBB12_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB12_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB12_2
|
|
; GCN3-NEXT: .LBB12_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_nop 0
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw xchg ptr %ptr, double %in seq_cst
|
|
ret double %result
|
|
}
|
|
|
|
define double @flat_atomic_xchg_f64_ret_offset(ptr %out, double %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB13_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB13_4
|
|
; GCN1-NEXT: .LBB13_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB13_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB13_2
|
|
; GCN1-NEXT: .LBB13_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB13_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB13_4
|
|
; GCN2-NEXT: .LBB13_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB13_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB13_2
|
|
; GCN2-NEXT: .LBB13_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB13_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB13_4
|
|
; GCN3-NEXT: .LBB13_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB13_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB13_2
|
|
; GCN3-NEXT: .LBB13_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_nop 0
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr double, ptr %out, i32 4
|
|
%result = atomicrmw xchg ptr %gep, double %in seq_cst
|
|
ret double %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_xchg_f64_noret_scalar(ptr inreg %ptr, double inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB14_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB14_4
|
|
; GCN1-NEXT: .LBB14_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB14_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB14_2
|
|
; GCN1-NEXT: .LBB14_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB14_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB14_4
|
|
; GCN2-NEXT: .LBB14_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB14_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB14_2
|
|
; GCN2-NEXT: .LBB14_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB14_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB14_4
|
|
; GCN3-NEXT: .LBB14_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB14_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB14_2
|
|
; GCN3-NEXT: .LBB14_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw xchg ptr %ptr, double %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_xchg_f64_noret_offset_scalar(ptr inreg %out, double inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB15_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB15_4
|
|
; GCN1-NEXT: .LBB15_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB15_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB15_2
|
|
; GCN1-NEXT: .LBB15_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB15_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB15_4
|
|
; GCN2-NEXT: .LBB15_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB15_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB15_2
|
|
; GCN2-NEXT: .LBB15_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB15_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB15_4
|
|
; GCN3-NEXT: .LBB15_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB15_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB15_2
|
|
; GCN3-NEXT: .LBB15_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr double, ptr %out, i32 4
|
|
%tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx double @flat_atomic_xchg_f64_ret_scalar(ptr inreg %ptr, double inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB16_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB16_3
|
|
; GCN1-NEXT: s_branch .LBB16_4
|
|
; GCN1-NEXT: .LBB16_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB16_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB16_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB16_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB16_3
|
|
; GCN2-NEXT: s_branch .LBB16_4
|
|
; GCN2-NEXT: .LBB16_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB16_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB16_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB16_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB16_3
|
|
; GCN3-NEXT: s_branch .LBB16_4
|
|
; GCN3-NEXT: .LBB16_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB16_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s6
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB16_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw xchg ptr %ptr, double %in seq_cst
|
|
ret double %result
|
|
}
|
|
|
|
define amdgpu_gfx double @flat_atomic_xchg_f64_ret_offset_scalar(ptr inreg %out, double inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB17_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB17_3
|
|
; GCN1-NEXT: s_branch .LBB17_4
|
|
; GCN1-NEXT: .LBB17_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB17_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB17_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB17_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB17_3
|
|
; GCN2-NEXT: s_branch .LBB17_4
|
|
; GCN2-NEXT: .LBB17_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB17_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s6
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s7
|
|
; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB17_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB17_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB17_3
|
|
; GCN3-NEXT: s_branch .LBB17_4
|
|
; GCN3-NEXT: .LBB17_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB17_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s6
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB17_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr double, ptr %out, i32 4
|
|
%result = atomicrmw xchg ptr %gep, double %in seq_cst
|
|
ret double %result
|
|
}
|
|
|
|
define void @flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory(ptr %out, double %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB18_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB18_4
|
|
; GCN1-NEXT: .LBB18_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB18_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB18_2
|
|
; GCN1-NEXT: .LBB18_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB18_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB18_4
|
|
; GCN2-NEXT: .LBB18_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB18_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB18_2
|
|
; GCN2-NEXT: .LBB18_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB18_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB18_4
|
|
; GCN3-NEXT: .LBB18_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB18_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB18_2
|
|
; GCN3-NEXT: .LBB18_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr double, ptr %out, i64 4
|
|
%tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define double @flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory(ptr %out, double %in) {
|
|
; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB19_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB19_4
|
|
; GCN1-NEXT: .LBB19_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB19_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB19_2
|
|
; GCN1-NEXT: .LBB19_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB19_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB19_4
|
|
; GCN2-NEXT: .LBB19_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB19_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB19_2
|
|
; GCN2-NEXT: .LBB19_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB19_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB19_4
|
|
; GCN3-NEXT: .LBB19_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB19_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB19_2
|
|
; GCN3-NEXT: .LBB19_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_nop 0
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr double, ptr %out, i64 4
|
|
%result = atomicrmw xchg ptr %gep, double %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret double %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw add
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_add_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB20_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB20_4
|
|
; GCN1-NEXT: .LBB20_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB20_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB20_2
|
|
; GCN1-NEXT: .LBB20_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB20_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB20_4
|
|
; GCN2-NEXT: .LBB20_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB20_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB20_2
|
|
; GCN2-NEXT: .LBB20_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB20_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB20_4
|
|
; GCN3-NEXT: .LBB20_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB20_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB20_2
|
|
; GCN3-NEXT: .LBB20_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw add ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_add_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB21_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB21_4
|
|
; GCN1-NEXT: .LBB21_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB21_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB21_2
|
|
; GCN1-NEXT: .LBB21_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB21_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB21_4
|
|
; GCN2-NEXT: .LBB21_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB21_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB21_2
|
|
; GCN2-NEXT: .LBB21_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB21_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB21_4
|
|
; GCN3-NEXT: .LBB21_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB21_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB21_2
|
|
; GCN3-NEXT: .LBB21_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_add_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB22_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB22_4
|
|
; GCN1-NEXT: .LBB22_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB22_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB22_2
|
|
; GCN1-NEXT: .LBB22_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB22_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB22_4
|
|
; GCN2-NEXT: .LBB22_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB22_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB22_2
|
|
; GCN2-NEXT: .LBB22_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB22_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB22_4
|
|
; GCN3-NEXT: .LBB22_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB22_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB22_2
|
|
; GCN3-NEXT: .LBB22_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw add ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_add_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB23_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB23_4
|
|
; GCN1-NEXT: .LBB23_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB23_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB23_2
|
|
; GCN1-NEXT: .LBB23_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB23_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB23_4
|
|
; GCN2-NEXT: .LBB23_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB23_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB23_2
|
|
; GCN2-NEXT: .LBB23_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB23_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB23_4
|
|
; GCN3-NEXT: .LBB23_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB23_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB23_2
|
|
; GCN3-NEXT: .LBB23_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw add ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_add_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB24_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB24_4
|
|
; GCN1-NEXT: .LBB24_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB24_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB24_2
|
|
; GCN1-NEXT: .LBB24_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, s6, v1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB24_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB24_4
|
|
; GCN2-NEXT: .LBB24_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB24_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB24_2
|
|
; GCN2-NEXT: .LBB24_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, s6, v1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB24_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB24_4
|
|
; GCN3-NEXT: .LBB24_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB24_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB24_2
|
|
; GCN3-NEXT: .LBB24_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, s6, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw add ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_add_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB25_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB25_4
|
|
; GCN1-NEXT: .LBB25_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB25_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB25_2
|
|
; GCN1-NEXT: .LBB25_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, s6, v1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB25_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB25_4
|
|
; GCN2-NEXT: .LBB25_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB25_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB25_2
|
|
; GCN2-NEXT: .LBB25_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, s6, v1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB25_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB25_4
|
|
; GCN3-NEXT: .LBB25_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB25_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB25_2
|
|
; GCN3-NEXT: .LBB25_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, s6, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_add_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB26_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB26_3
|
|
; GCN1-NEXT: s_branch .LBB26_4
|
|
; GCN1-NEXT: .LBB26_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB26_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB26_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB26_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB26_3
|
|
; GCN2-NEXT: s_branch .LBB26_4
|
|
; GCN2-NEXT: .LBB26_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB26_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB26_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB26_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB26_3
|
|
; GCN3-NEXT: s_branch .LBB26_4
|
|
; GCN3-NEXT: .LBB26_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB26_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, s6, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB26_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw add ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_add_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB27_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB27_3
|
|
; GCN1-NEXT: s_branch .LBB27_4
|
|
; GCN1-NEXT: .LBB27_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB27_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB27_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB27_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB27_3
|
|
; GCN2-NEXT: s_branch .LBB27_4
|
|
; GCN2-NEXT: .LBB27_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB27_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB27_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB27_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB27_3
|
|
; GCN3-NEXT: s_branch .LBB27_4
|
|
; GCN3-NEXT: .LBB27_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB27_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, s6, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB27_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw add ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB28_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB28_4
|
|
; GCN1-NEXT: .LBB28_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB28_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB28_2
|
|
; GCN1-NEXT: .LBB28_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB28_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB28_4
|
|
; GCN2-NEXT: .LBB28_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB28_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB28_2
|
|
; GCN2-NEXT: .LBB28_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB28_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB28_4
|
|
; GCN3-NEXT: .LBB28_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB28_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB28_2
|
|
; GCN3-NEXT: .LBB28_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB29_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB29_4
|
|
; GCN1-NEXT: .LBB29_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB29_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB29_2
|
|
; GCN1-NEXT: .LBB29_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB29_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB29_4
|
|
; GCN2-NEXT: .LBB29_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB29_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB29_2
|
|
; GCN2-NEXT: .LBB29_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB29_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB29_4
|
|
; GCN3-NEXT: .LBB29_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB29_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB29_2
|
|
; GCN3-NEXT: .LBB29_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw add ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw sub
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_sub_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB30_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB30_4
|
|
; GCN1-NEXT: .LBB30_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB30_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB30_2
|
|
; GCN1-NEXT: .LBB30_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB30_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB30_4
|
|
; GCN2-NEXT: .LBB30_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB30_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB30_2
|
|
; GCN2-NEXT: .LBB30_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB30_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB30_4
|
|
; GCN3-NEXT: .LBB30_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB30_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB30_2
|
|
; GCN3-NEXT: .LBB30_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_sub_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB31_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB31_4
|
|
; GCN1-NEXT: .LBB31_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB31_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB31_2
|
|
; GCN1-NEXT: .LBB31_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB31_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB31_4
|
|
; GCN2-NEXT: .LBB31_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB31_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB31_2
|
|
; GCN2-NEXT: .LBB31_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB31_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB31_4
|
|
; GCN3-NEXT: .LBB31_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB31_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB31_2
|
|
; GCN3-NEXT: .LBB31_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_sub_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB32_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB32_4
|
|
; GCN1-NEXT: .LBB32_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB32_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB32_2
|
|
; GCN1-NEXT: .LBB32_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB32_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB32_4
|
|
; GCN2-NEXT: .LBB32_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB32_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB32_2
|
|
; GCN2-NEXT: .LBB32_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB32_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB32_4
|
|
; GCN3-NEXT: .LBB32_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB32_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB32_2
|
|
; GCN3-NEXT: .LBB32_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw sub ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_sub_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB33_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB33_4
|
|
; GCN1-NEXT: .LBB33_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB33_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB33_2
|
|
; GCN1-NEXT: .LBB33_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB33_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB33_4
|
|
; GCN2-NEXT: .LBB33_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB33_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB33_2
|
|
; GCN2-NEXT: .LBB33_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB33_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB33_4
|
|
; GCN3-NEXT: .LBB33_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB33_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB33_2
|
|
; GCN3-NEXT: .LBB33_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw sub ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB34_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB34_4
|
|
; GCN1-NEXT: .LBB34_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB34_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB34_2
|
|
; GCN1-NEXT: .LBB34_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_subrev_i32_e32 v1, vcc, s6, v1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB34_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB34_4
|
|
; GCN2-NEXT: .LBB34_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB34_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB34_2
|
|
; GCN2-NEXT: .LBB34_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_subrev_u32_e32 v1, vcc, s6, v1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB34_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB34_4
|
|
; GCN3-NEXT: .LBB34_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB34_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB34_2
|
|
; GCN3-NEXT: .LBB34_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_subrev_co_u32_e32 v1, vcc, s6, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB35_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB35_4
|
|
; GCN1-NEXT: .LBB35_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB35_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB35_2
|
|
; GCN1-NEXT: .LBB35_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_subrev_i32_e32 v1, vcc, s6, v1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB35_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB35_4
|
|
; GCN2-NEXT: .LBB35_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB35_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB35_2
|
|
; GCN2-NEXT: .LBB35_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_subrev_u32_e32 v1, vcc, s6, v1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB35_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB35_4
|
|
; GCN3-NEXT: .LBB35_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB35_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB35_2
|
|
; GCN3-NEXT: .LBB35_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_subrev_co_u32_e32 v1, vcc, s6, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB36_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB36_3
|
|
; GCN1-NEXT: s_branch .LBB36_4
|
|
; GCN1-NEXT: .LBB36_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB36_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB36_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB36_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB36_3
|
|
; GCN2-NEXT: s_branch .LBB36_4
|
|
; GCN2-NEXT: .LBB36_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB36_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB36_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB36_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB36_3
|
|
; GCN3-NEXT: s_branch .LBB36_4
|
|
; GCN3-NEXT: .LBB36_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB36_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB36_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw sub ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB37_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB37_3
|
|
; GCN1-NEXT: s_branch .LBB37_4
|
|
; GCN1-NEXT: .LBB37_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB37_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB37_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB37_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB37_3
|
|
; GCN2-NEXT: s_branch .LBB37_4
|
|
; GCN2-NEXT: .LBB37_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB37_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc
|
|
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB37_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB37_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB37_3
|
|
; GCN3-NEXT: s_branch .LBB37_4
|
|
; GCN3-NEXT: .LBB37_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB37_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v0
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB37_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw sub ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB38_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB38_4
|
|
; GCN1-NEXT: .LBB38_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB38_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB38_2
|
|
; GCN1-NEXT: .LBB38_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB38_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB38_4
|
|
; GCN2-NEXT: .LBB38_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB38_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB38_2
|
|
; GCN2-NEXT: .LBB38_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB38_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB38_4
|
|
; GCN3-NEXT: .LBB38_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB38_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB38_2
|
|
; GCN3-NEXT: .LBB38_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB39_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB39_4
|
|
; GCN1-NEXT: .LBB39_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB39_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB39_2
|
|
; GCN1-NEXT: .LBB39_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB39_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB39_4
|
|
; GCN2-NEXT: .LBB39_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB39_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB39_2
|
|
; GCN2-NEXT: .LBB39_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB39_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB39_4
|
|
; GCN3-NEXT: .LBB39_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB39_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB39_2
|
|
; GCN3-NEXT: .LBB39_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw sub ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw and
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_and_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB40_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB40_4
|
|
; GCN1-NEXT: .LBB40_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB40_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB40_2
|
|
; GCN1-NEXT: .LBB40_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB40_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB40_4
|
|
; GCN2-NEXT: .LBB40_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB40_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB40_2
|
|
; GCN2-NEXT: .LBB40_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB40_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB40_4
|
|
; GCN3-NEXT: .LBB40_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB40_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB40_2
|
|
; GCN3-NEXT: .LBB40_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_and_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB41_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB41_4
|
|
; GCN1-NEXT: .LBB41_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB41_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB41_2
|
|
; GCN1-NEXT: .LBB41_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB41_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB41_4
|
|
; GCN2-NEXT: .LBB41_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB41_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB41_2
|
|
; GCN2-NEXT: .LBB41_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB41_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB41_4
|
|
; GCN3-NEXT: .LBB41_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB41_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB41_2
|
|
; GCN3-NEXT: .LBB41_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_and_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB42_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB42_4
|
|
; GCN1-NEXT: .LBB42_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB42_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB42_2
|
|
; GCN1-NEXT: .LBB42_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB42_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB42_4
|
|
; GCN2-NEXT: .LBB42_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB42_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB42_2
|
|
; GCN2-NEXT: .LBB42_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB42_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB42_4
|
|
; GCN3-NEXT: .LBB42_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB42_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB42_2
|
|
; GCN3-NEXT: .LBB42_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw and ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_and_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB43_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB43_4
|
|
; GCN1-NEXT: .LBB43_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB43_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB43_2
|
|
; GCN1-NEXT: .LBB43_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB43_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB43_4
|
|
; GCN2-NEXT: .LBB43_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB43_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB43_2
|
|
; GCN2-NEXT: .LBB43_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB43_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB43_4
|
|
; GCN3-NEXT: .LBB43_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB43_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB43_2
|
|
; GCN3-NEXT: .LBB43_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw and ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB44_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB44_4
|
|
; GCN1-NEXT: .LBB44_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB44_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB44_2
|
|
; GCN1-NEXT: .LBB44_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB44_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB44_4
|
|
; GCN2-NEXT: .LBB44_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB44_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB44_2
|
|
; GCN2-NEXT: .LBB44_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB44_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB44_4
|
|
; GCN3-NEXT: .LBB44_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB44_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB44_2
|
|
; GCN3-NEXT: .LBB44_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB45_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB45_4
|
|
; GCN1-NEXT: .LBB45_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB45_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB45_2
|
|
; GCN1-NEXT: .LBB45_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB45_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB45_4
|
|
; GCN2-NEXT: .LBB45_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB45_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB45_2
|
|
; GCN2-NEXT: .LBB45_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB45_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB45_4
|
|
; GCN3-NEXT: .LBB45_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB45_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB45_2
|
|
; GCN3-NEXT: .LBB45_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB46_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB46_3
|
|
; GCN1-NEXT: s_branch .LBB46_4
|
|
; GCN1-NEXT: .LBB46_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB46_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB46_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB46_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB46_3
|
|
; GCN2-NEXT: s_branch .LBB46_4
|
|
; GCN2-NEXT: .LBB46_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB46_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB46_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB46_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB46_3
|
|
; GCN3-NEXT: s_branch .LBB46_4
|
|
; GCN3-NEXT: .LBB46_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB46_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB46_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw and ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB47_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB47_3
|
|
; GCN1-NEXT: s_branch .LBB47_4
|
|
; GCN1-NEXT: .LBB47_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB47_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB47_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB47_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB47_3
|
|
; GCN2-NEXT: s_branch .LBB47_4
|
|
; GCN2-NEXT: .LBB47_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB47_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB47_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB47_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB47_3
|
|
; GCN3-NEXT: s_branch .LBB47_4
|
|
; GCN3-NEXT: .LBB47_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB47_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB47_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw and ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB48_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB48_4
|
|
; GCN1-NEXT: .LBB48_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB48_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB48_2
|
|
; GCN1-NEXT: .LBB48_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB48_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB48_4
|
|
; GCN2-NEXT: .LBB48_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB48_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB48_2
|
|
; GCN2-NEXT: .LBB48_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB48_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB48_4
|
|
; GCN3-NEXT: .LBB48_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB48_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB48_2
|
|
; GCN3-NEXT: .LBB48_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB49_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB49_4
|
|
; GCN1-NEXT: .LBB49_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB49_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB49_2
|
|
; GCN1-NEXT: .LBB49_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB49_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB49_4
|
|
; GCN2-NEXT: .LBB49_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB49_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB49_2
|
|
; GCN2-NEXT: .LBB49_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB49_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB49_4
|
|
; GCN3-NEXT: .LBB49_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB49_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB49_2
|
|
; GCN3-NEXT: .LBB49_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw and ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw nand
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_nand_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB50_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB50_6
|
|
; GCN1-NEXT: .LBB50_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB50_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB50_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB50_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB50_2
|
|
; GCN1-NEXT: .LBB50_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB50_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB50_6
|
|
; GCN2-NEXT: .LBB50_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB50_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB50_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB50_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB50_2
|
|
; GCN2-NEXT: .LBB50_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB50_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB50_6
|
|
; GCN3-NEXT: .LBB50_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB50_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB50_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB50_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB50_2
|
|
; GCN3-NEXT: .LBB50_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v1
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw nand ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_nand_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB51_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB51_6
|
|
; GCN1-NEXT: .LBB51_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB51_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB51_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB51_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB51_2
|
|
; GCN1-NEXT: .LBB51_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB51_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB51_6
|
|
; GCN2-NEXT: .LBB51_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB51_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB51_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB51_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB51_2
|
|
; GCN2-NEXT: .LBB51_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB51_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB51_6
|
|
; GCN3-NEXT: .LBB51_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB51_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB51_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB51_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB51_2
|
|
; GCN3-NEXT: .LBB51_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v1
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_nand_i64_noret_offset__noalias_private(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_noret_offset__noalias_private:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB52_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v0, v7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v1, v6, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB52_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_noret_offset__noalias_private:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB52_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v0, v7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v1, v6, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB52_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_noret_offset__noalias_private:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB52_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB52_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_nand_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB53_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB53_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB53_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: .LBB53_4: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB53_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB53_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB53_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v5, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB53_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB53_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: .LBB53_4: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB53_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB53_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB53_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB53_6
|
|
; GCN3-NEXT: .LBB53_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB53_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB53_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB53_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB53_2
|
|
; GCN3-NEXT: .LBB53_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw nand ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_nand_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB54_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB54_6
|
|
; GCN1-NEXT: .LBB54_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB54_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB54_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB54_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB54_2
|
|
; GCN1-NEXT: .LBB54_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB54_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB54_6
|
|
; GCN2-NEXT: .LBB54_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB54_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB54_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB54_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB54_2
|
|
; GCN2-NEXT: .LBB54_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB54_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB54_6
|
|
; GCN3-NEXT: .LBB54_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB54_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB54_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB54_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB54_2
|
|
; GCN3-NEXT: .LBB54_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw nand ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_nand_i64_ret_offset__noalias_private(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_ret_offset__noalias_private:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB55_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB55_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_ret_offset__noalias_private:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB55_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB55_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_ret_offset__noalias_private:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB55_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB55_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB56_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB56_6
|
|
; GCN1-NEXT: .LBB56_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB56_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: .LBB56_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB56_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB56_2
|
|
; GCN1-NEXT: .LBB56_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB56_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB56_6
|
|
; GCN2-NEXT: .LBB56_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB56_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: .LBB56_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB56_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB56_2
|
|
; GCN2-NEXT: .LBB56_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB56_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB56_6
|
|
; GCN3-NEXT: .LBB56_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB56_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: .LBB56_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB56_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB56_2
|
|
; GCN3-NEXT: .LBB56_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v1
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw nand ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB57_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB57_6
|
|
; GCN1-NEXT: .LBB57_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB57_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: .LBB57_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB57_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB57_2
|
|
; GCN1-NEXT: .LBB57_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB57_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB57_6
|
|
; GCN2-NEXT: .LBB57_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB57_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: .LBB57_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB57_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB57_2
|
|
; GCN2-NEXT: .LBB57_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB57_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB57_6
|
|
; GCN3-NEXT: .LBB57_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB57_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: .LBB57_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB57_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB57_2
|
|
; GCN3-NEXT: .LBB57_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v1
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar__noalias_addrspace(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_noret_offset_scalar__noalias_addrspace:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN1-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: .LBB58_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB58_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_noret_offset_scalar__noalias_addrspace:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN2-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: .LBB58_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB58_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_noret_offset_scalar__noalias_addrspace:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: .LBB58_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v0, v6
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB58_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB59_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: .LBB59_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN1-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB59_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_branch .LBB59_6
|
|
; GCN1-NEXT: .LBB59_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB59_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v4
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v5
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB59_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB59_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: .LBB59_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN2-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB59_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_branch .LBB59_6
|
|
; GCN2-NEXT: .LBB59_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB59_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v4
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v5
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB59_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB59_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: .LBB59_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN3-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB59_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_branch .LBB59_6
|
|
; GCN3-NEXT: .LBB59_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB59_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v4
|
|
; GCN3-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB59_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw nand ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB60_4
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN1-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN1-NEXT: .LBB60_2: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN1-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB60_2
|
|
; GCN1-NEXT: ; %bb.3: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN1-NEXT: s_branch .LBB60_6
|
|
; GCN1-NEXT: .LBB60_4:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_cbranch_execz .LBB60_6
|
|
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v4
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v5
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB60_6: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB60_4
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: s_add_u32 s36, s34, 4
|
|
; GCN2-NEXT: s_addc_u32 s37, s35, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN2-NEXT: .LBB60_2: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN2-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB60_2
|
|
; GCN2-NEXT: ; %bb.3: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN2-NEXT: s_branch .LBB60_6
|
|
; GCN2-NEXT: .LBB60_4:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_cbranch_execz .LBB60_6
|
|
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v4
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v5
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB60_6: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB60_4
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], 0
|
|
; GCN3-NEXT: .LBB60_2: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN3-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB60_2
|
|
; GCN3-NEXT: ; %bb.3: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
|
|
; GCN3-NEXT: s_branch .LBB60_6
|
|
; GCN3-NEXT: .LBB60_4:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_cbranch_execz .LBB60_6
|
|
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v4
|
|
; GCN3-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB60_6: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw nand ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar__noalias_private(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_ret_offset_scalar__noalias_private:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN1-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: .LBB61_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN1-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB61_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_ret_offset_scalar__noalias_private:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN2-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: .LBB61_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN2-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB61_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_ret_offset_scalar__noalias_private:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: .LBB61_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v0, s7, v7
|
|
; GCN3-NEXT: v_and_b32_e32 v1, s6, v6
|
|
; GCN3-NEXT: v_not_b32_e32 v5, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v1
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB61_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB62_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB62_6
|
|
; GCN1-NEXT: .LBB62_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB62_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB62_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN1-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB62_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB62_2
|
|
; GCN1-NEXT: .LBB62_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB62_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB62_6
|
|
; GCN2-NEXT: .LBB62_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB62_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB62_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN2-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB62_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB62_2
|
|
; GCN2-NEXT: .LBB62_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB62_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB62_6
|
|
; GCN3-NEXT: .LBB62_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB62_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB62_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v4, v7, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v8, v6, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v5, v4
|
|
; GCN3-NEXT: v_not_b32_e32 v4, v8
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB62_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB62_2
|
|
; GCN3-NEXT: .LBB62_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v1, v1
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB63_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB63_6
|
|
; GCN1-NEXT: .LBB63_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB63_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN1-NEXT: .LBB63_4: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN1-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN1-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB63_4
|
|
; GCN1-NEXT: ; %bb.5: ; %Flow
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: ; implicit-def: $vgpr2
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB63_2
|
|
; GCN1-NEXT: .LBB63_6: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN1-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB63_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB63_6
|
|
; GCN2-NEXT: .LBB63_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB63_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN2-NEXT: .LBB63_4: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN2-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN2-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB63_4
|
|
; GCN2-NEXT: ; %bb.5: ; %Flow
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: ; implicit-def: $vgpr2
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB63_2
|
|
; GCN2-NEXT: .LBB63_6: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN2-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB63_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB63_6
|
|
; GCN3-NEXT: .LBB63_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB63_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[6:7], 0
|
|
; GCN3-NEXT: .LBB63_4: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_and_b32_e32 v0, v9, v3
|
|
; GCN3-NEXT: v_and_b32_e32 v1, v8, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v7, v0
|
|
; GCN3-NEXT: v_not_b32_e32 v6, v1
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB63_4
|
|
; GCN3-NEXT: ; %bb.5: ; %Flow
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: ; implicit-def: $vgpr2
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB63_2
|
|
; GCN3-NEXT: .LBB63_6: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v2, v2
|
|
; GCN3-NEXT: v_not_b32_e32 v3, v3
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw nand ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw or
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_or_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB64_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB64_4
|
|
; GCN1-NEXT: .LBB64_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB64_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB64_2
|
|
; GCN1-NEXT: .LBB64_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB64_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB64_4
|
|
; GCN2-NEXT: .LBB64_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB64_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB64_2
|
|
; GCN2-NEXT: .LBB64_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB64_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB64_4
|
|
; GCN3-NEXT: .LBB64_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB64_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB64_2
|
|
; GCN3-NEXT: .LBB64_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_or_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB65_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB65_4
|
|
; GCN1-NEXT: .LBB65_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB65_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB65_2
|
|
; GCN1-NEXT: .LBB65_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB65_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB65_4
|
|
; GCN2-NEXT: .LBB65_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB65_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB65_2
|
|
; GCN2-NEXT: .LBB65_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB65_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB65_4
|
|
; GCN3-NEXT: .LBB65_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB65_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB65_2
|
|
; GCN3-NEXT: .LBB65_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_or_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB66_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB66_4
|
|
; GCN1-NEXT: .LBB66_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB66_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB66_2
|
|
; GCN1-NEXT: .LBB66_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB66_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB66_4
|
|
; GCN2-NEXT: .LBB66_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB66_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB66_2
|
|
; GCN2-NEXT: .LBB66_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB66_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB66_4
|
|
; GCN3-NEXT: .LBB66_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB66_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB66_2
|
|
; GCN3-NEXT: .LBB66_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw or ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_or_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB67_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB67_4
|
|
; GCN1-NEXT: .LBB67_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB67_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB67_2
|
|
; GCN1-NEXT: .LBB67_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB67_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB67_4
|
|
; GCN2-NEXT: .LBB67_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB67_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB67_2
|
|
; GCN2-NEXT: .LBB67_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB67_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB67_4
|
|
; GCN3-NEXT: .LBB67_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB67_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB67_2
|
|
; GCN3-NEXT: .LBB67_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw or ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB68_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB68_4
|
|
; GCN1-NEXT: .LBB68_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB68_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB68_2
|
|
; GCN1-NEXT: .LBB68_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB68_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB68_4
|
|
; GCN2-NEXT: .LBB68_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB68_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB68_2
|
|
; GCN2-NEXT: .LBB68_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB68_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB68_4
|
|
; GCN3-NEXT: .LBB68_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB68_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB68_2
|
|
; GCN3-NEXT: .LBB68_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB69_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB69_4
|
|
; GCN1-NEXT: .LBB69_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB69_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB69_2
|
|
; GCN1-NEXT: .LBB69_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB69_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB69_4
|
|
; GCN2-NEXT: .LBB69_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB69_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB69_2
|
|
; GCN2-NEXT: .LBB69_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB69_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB69_4
|
|
; GCN3-NEXT: .LBB69_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB69_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB69_2
|
|
; GCN3-NEXT: .LBB69_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB70_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB70_3
|
|
; GCN1-NEXT: s_branch .LBB70_4
|
|
; GCN1-NEXT: .LBB70_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB70_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB70_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB70_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB70_3
|
|
; GCN2-NEXT: s_branch .LBB70_4
|
|
; GCN2-NEXT: .LBB70_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB70_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB70_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB70_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB70_3
|
|
; GCN3-NEXT: s_branch .LBB70_4
|
|
; GCN3-NEXT: .LBB70_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB70_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB70_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw or ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB71_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB71_3
|
|
; GCN1-NEXT: s_branch .LBB71_4
|
|
; GCN1-NEXT: .LBB71_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB71_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB71_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB71_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB71_3
|
|
; GCN2-NEXT: s_branch .LBB71_4
|
|
; GCN2-NEXT: .LBB71_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB71_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB71_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB71_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB71_3
|
|
; GCN3-NEXT: s_branch .LBB71_4
|
|
; GCN3-NEXT: .LBB71_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB71_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB71_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw or ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB72_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB72_4
|
|
; GCN1-NEXT: .LBB72_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB72_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB72_2
|
|
; GCN1-NEXT: .LBB72_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB72_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB72_4
|
|
; GCN2-NEXT: .LBB72_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB72_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB72_2
|
|
; GCN2-NEXT: .LBB72_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB72_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB72_4
|
|
; GCN3-NEXT: .LBB72_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB72_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB72_2
|
|
; GCN3-NEXT: .LBB72_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB73_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB73_4
|
|
; GCN1-NEXT: .LBB73_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB73_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB73_2
|
|
; GCN1-NEXT: .LBB73_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB73_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB73_4
|
|
; GCN2-NEXT: .LBB73_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB73_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB73_2
|
|
; GCN2-NEXT: .LBB73_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB73_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB73_4
|
|
; GCN3-NEXT: .LBB73_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB73_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB73_2
|
|
; GCN3-NEXT: .LBB73_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_or_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_or_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw or ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw xor
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_xor_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB74_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB74_4
|
|
; GCN1-NEXT: .LBB74_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB74_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB74_2
|
|
; GCN1-NEXT: .LBB74_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB74_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB74_4
|
|
; GCN2-NEXT: .LBB74_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB74_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB74_2
|
|
; GCN2-NEXT: .LBB74_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB74_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB74_4
|
|
; GCN3-NEXT: .LBB74_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB74_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB74_2
|
|
; GCN3-NEXT: .LBB74_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_xor_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB75_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB75_4
|
|
; GCN1-NEXT: .LBB75_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB75_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB75_2
|
|
; GCN1-NEXT: .LBB75_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB75_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB75_4
|
|
; GCN2-NEXT: .LBB75_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB75_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB75_2
|
|
; GCN2-NEXT: .LBB75_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB75_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB75_4
|
|
; GCN3-NEXT: .LBB75_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB75_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB75_2
|
|
; GCN3-NEXT: .LBB75_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_xor_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB76_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB76_4
|
|
; GCN1-NEXT: .LBB76_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB76_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB76_2
|
|
; GCN1-NEXT: .LBB76_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB76_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB76_4
|
|
; GCN2-NEXT: .LBB76_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB76_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB76_2
|
|
; GCN2-NEXT: .LBB76_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, v1
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, v0
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB76_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB76_4
|
|
; GCN3-NEXT: .LBB76_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB76_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB76_2
|
|
; GCN3-NEXT: .LBB76_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw xor ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_xor_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB77_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB77_4
|
|
; GCN1-NEXT: .LBB77_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB77_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB77_2
|
|
; GCN1-NEXT: .LBB77_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB77_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB77_4
|
|
; GCN2-NEXT: .LBB77_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB77_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB77_2
|
|
; GCN2-NEXT: .LBB77_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB77_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB77_4
|
|
; GCN3-NEXT: .LBB77_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB77_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB77_2
|
|
; GCN3-NEXT: .LBB77_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw xor ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB78_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB78_4
|
|
; GCN1-NEXT: .LBB78_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB78_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB78_2
|
|
; GCN1-NEXT: .LBB78_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB78_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB78_4
|
|
; GCN2-NEXT: .LBB78_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB78_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB78_2
|
|
; GCN2-NEXT: .LBB78_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB78_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB78_4
|
|
; GCN3-NEXT: .LBB78_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB78_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB78_2
|
|
; GCN3-NEXT: .LBB78_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN1-NEXT: s_cbranch_vccnz .LBB79_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB79_4
|
|
; GCN1-NEXT: .LBB79_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB79_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB79_2
|
|
; GCN1-NEXT: .LBB79_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, s6, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, s7, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN2-NEXT: s_cbranch_vccnz .LBB79_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB79_4
|
|
; GCN2-NEXT: .LBB79_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB79_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB79_2
|
|
; GCN2-NEXT: .LBB79_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s34
|
|
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, s6, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, s7, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], -1
|
|
; GCN3-NEXT: s_cbranch_vccnz .LBB79_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB79_4
|
|
; GCN3-NEXT: .LBB79_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB79_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB79_2
|
|
; GCN3-NEXT: .LBB79_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v1, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, s6, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB80_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB80_3
|
|
; GCN1-NEXT: s_branch .LBB80_4
|
|
; GCN1-NEXT: .LBB80_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB80_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
|
|
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB80_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
|
|
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB80_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB80_3
|
|
; GCN2-NEXT: s_branch .LBB80_4
|
|
; GCN2-NEXT: .LBB80_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB80_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB80_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
|
|
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB80_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB80_3
|
|
; GCN3-NEXT: s_branch .LBB80_4
|
|
; GCN3-NEXT: .LBB80_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB80_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB80_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw xor ptr %ptr, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN1-NEXT: s_cbranch_vccz .LBB81_2
|
|
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_cbranch_execz .LBB81_3
|
|
; GCN1-NEXT: s_branch .LBB81_4
|
|
; GCN1-NEXT: .LBB81_2:
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: .LBB81_3: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
|
|
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
|
|
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v4, s6, v0
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v5, s7, v1
|
|
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN1-NEXT: .LBB81_4: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
|
|
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
|
|
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN2-NEXT: s_cbranch_vccz .LBB81_2
|
|
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_cbranch_execz .LBB81_3
|
|
; GCN2-NEXT: s_branch .LBB81_4
|
|
; GCN2-NEXT: .LBB81_2:
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: .LBB81_3: ; %atomicrmw.private
|
|
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: s_add_i32 s34, s34, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v4, s6, v0
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v5, s7, v1
|
|
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
|
|
; GCN2-NEXT: .LBB81_4: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN3-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
|
|
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
|
|
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
|
|
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
|
|
; GCN3-NEXT: s_cbranch_vccz .LBB81_2
|
|
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s34
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s35
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s7
|
|
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_cbranch_execz .LBB81_3
|
|
; GCN3-NEXT: s_branch .LBB81_4
|
|
; GCN3-NEXT: .LBB81_2:
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: .LBB81_3: ; %atomicrmw.private
|
|
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
|
|
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v3, s7, v1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v4, s6, v0
|
|
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: .LBB81_4: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw xor ptr %gep, i64 %in seq_cst
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB82_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB82_4
|
|
; GCN1-NEXT: .LBB82_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB82_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB82_2
|
|
; GCN1-NEXT: .LBB82_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
|
|
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB82_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB82_4
|
|
; GCN2-NEXT: .LBB82_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB82_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB82_2
|
|
; GCN2-NEXT: .LBB82_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
|
|
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB82_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB82_4
|
|
; GCN3-NEXT: .LBB82_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB82_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB82_2
|
|
; GCN3-NEXT: .LBB82_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB83_3
|
|
; GCN1-NEXT: ; %bb.1: ; %Flow
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB83_4
|
|
; GCN1-NEXT: .LBB83_2: ; %atomicrmw.phi
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN1-NEXT: .LBB83_3: ; %atomicrmw.global
|
|
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN1-NEXT: ; implicit-def: $vgpr3
|
|
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execz .LBB83_2
|
|
; GCN1-NEXT: .LBB83_4: ; %atomicrmw.private
|
|
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
|
|
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN1-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
|
|
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB83_3
|
|
; GCN2-NEXT: ; %bb.1: ; %Flow
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB83_4
|
|
; GCN2-NEXT: .LBB83_2: ; %atomicrmw.phi
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN2-NEXT: .LBB83_3: ; %atomicrmw.global
|
|
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN2-NEXT: ; implicit-def: $vgpr3
|
|
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execz .LBB83_2
|
|
; GCN2-NEXT: .LBB83_4: ; %atomicrmw.private
|
|
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
|
|
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN2-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
|
|
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
|
|
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
|
|
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
|
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB83_3
|
|
; GCN3-NEXT: ; %bb.1: ; %Flow
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB83_4
|
|
; GCN3-NEXT: .LBB83_2: ; %atomicrmw.phi
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
; GCN3-NEXT: .LBB83_3: ; %atomicrmw.global
|
|
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
|
|
; GCN3-NEXT: ; implicit-def: $vgpr3
|
|
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execz .LBB83_2
|
|
; GCN3-NEXT: .LBB83_4: ; %atomicrmw.private
|
|
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
|
|
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: s_waitcnt vmcnt(1)
|
|
; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2
|
|
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
|
|
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0)
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw xor ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw max
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_max_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB84_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB84_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB84_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB84_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB84_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB84_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_max_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB85_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB85_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB85_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB85_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB85_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB85_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_max_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v5, v[5:6]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB86_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB86_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v5, v[5:6]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB86_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB86_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB86_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB86_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_max_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB87_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB87_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB87_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB87_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB87_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB87_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s35
|
|
; GCN1-NEXT: flat_load_dword v2, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v3, v[3:4]
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: .LBB88_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB88_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s35
|
|
; GCN2-NEXT: flat_load_dword v2, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v3, v[3:4]
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: .LBB88_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB88_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: .LBB88_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB88_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN1-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: .LBB89_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB89_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN2-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: .LBB89_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB89_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: .LBB89_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB89_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v0, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v1, v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: .LBB90_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB90_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v0, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v1, v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: .LBB90_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB90_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: .LBB90_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB90_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN1-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: .LBB91_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB91_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN2-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: .LBB91_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB91_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: .LBB91_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB91_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_max_i64_addr64_offset:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0xd
|
|
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x9
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN1-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN1-NEXT: .LBB92_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB92_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_max_i64_addr64_offset:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x34
|
|
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN2-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN2-NEXT: .LBB92_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB92_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_max_i64_addr64_offset:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x34
|
|
; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s4, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: .LBB92_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB92_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%gep = getelementptr i64, ptr %ptr, i64 4
|
|
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_max_i64_ret_addr64_offset:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x9
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN1-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s4
|
|
; GCN1-NEXT: .LBB93_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB93_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s2
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s3
|
|
; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_max_i64_ret_addr64_offset:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN2-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s4
|
|
; GCN2-NEXT: .LBB93_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB93_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s2
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s3
|
|
; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_max_i64_ret_addr64_offset:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[10:11], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s4, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s9
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s8
|
|
; GCN3-NEXT: .LBB93_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[8:9], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB93_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%gep = getelementptr i64, ptr %ptr, i64 4
|
|
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
store i64 %tmp0, ptr %out2
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_max_i64_addr64:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0xd
|
|
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x9
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN1-NEXT: .LBB94_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB94_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_max_i64_addr64:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x34
|
|
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN2-NEXT: .LBB94_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB94_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_max_i64_addr64:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x34
|
|
; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s4, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN3-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: .LBB94_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB94_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_max_i64_ret_addr64:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x9
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s4
|
|
; GCN1-NEXT: .LBB95_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB95_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s2
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s3
|
|
; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_max_i64_ret_addr64:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s4
|
|
; GCN2-NEXT: .LBB95_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB95_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s2
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s3
|
|
; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_max_i64_ret_addr64:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[10:11], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s4, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s9
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s8
|
|
; GCN3-NEXT: .LBB95_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[8:9], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB95_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
store i64 %tmp0, ptr %out2
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB96_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB96_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB96_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB96_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB96_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB96_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB97_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB97_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB97_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB97_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB97_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB97_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw umax
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_umax_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB98_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB98_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB98_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB98_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB98_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB98_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_umax_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB99_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB99_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB99_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB99_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB99_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB99_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_umax_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v5, v[5:6]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB100_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB100_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v5, v[5:6]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB100_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB100_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB100_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB100_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_umax_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB101_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB101_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB101_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB101_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB101_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB101_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s35
|
|
; GCN1-NEXT: flat_load_dword v2, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v3, v[3:4]
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: .LBB102_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB102_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s35
|
|
; GCN2-NEXT: flat_load_dword v2, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v3, v[3:4]
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: .LBB102_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB102_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: .LBB102_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB102_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN1-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: .LBB103_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB103_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN2-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: .LBB103_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB103_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: .LBB103_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB103_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v0, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v1, v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: .LBB104_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB104_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v0, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v1, v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: .LBB104_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB104_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: .LBB104_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB104_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN1-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: .LBB105_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB105_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN2-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: .LBB105_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB105_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: .LBB105_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB105_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_umax_i64_addr64_offset:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0xd
|
|
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x9
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN1-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN1-NEXT: .LBB106_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB106_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_umax_i64_addr64_offset:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x34
|
|
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN2-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN2-NEXT: .LBB106_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB106_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_umax_i64_addr64_offset:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x34
|
|
; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s4, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: .LBB106_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB106_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%gep = getelementptr i64, ptr %ptr, i64 4
|
|
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_umax_i64_ret_addr64_offset:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x9
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN1-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s4
|
|
; GCN1-NEXT: .LBB107_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB107_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s2
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s3
|
|
; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_umax_i64_ret_addr64_offset:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN2-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s4
|
|
; GCN2-NEXT: .LBB107_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB107_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s2
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s3
|
|
; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_umax_i64_ret_addr64_offset:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[10:11], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s4, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s9
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s8
|
|
; GCN3-NEXT: .LBB107_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[8:9], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB107_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%gep = getelementptr i64, ptr %ptr, i64 4
|
|
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
store i64 %tmp0, ptr %out2
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_umax_i64_ret_addr64:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x9
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s4
|
|
; GCN1-NEXT: .LBB108_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB108_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s2
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s3
|
|
; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_umax_i64_ret_addr64:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s4
|
|
; GCN2-NEXT: .LBB108_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB108_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s2
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s3
|
|
; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_umax_i64_ret_addr64:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[10:11], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s4, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s9
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s8
|
|
; GCN3-NEXT: .LBB108_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[8:9], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB108_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
store i64 %tmp0, ptr %out2
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB109_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB109_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB109_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB109_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB109_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB109_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB110_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB110_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB110_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB110_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB110_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB110_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw umin
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_umin_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB111_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB111_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB111_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB111_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB111_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB111_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_umin_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB112_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB112_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB112_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB112_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB112_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB112_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_umin_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v5, v[5:6]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB113_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB113_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v5, v[5:6]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB113_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB113_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB113_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB113_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_umin_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB114_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB114_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB114_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB114_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB114_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB114_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s35
|
|
; GCN1-NEXT: flat_load_dword v2, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v3, v[3:4]
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: .LBB115_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB115_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s35
|
|
; GCN2-NEXT: flat_load_dword v2, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v3, v[3:4]
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: .LBB115_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB115_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: .LBB115_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB115_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN1-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: .LBB116_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB116_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN2-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: .LBB116_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB116_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: .LBB116_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB116_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v0, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v1, v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: .LBB117_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB117_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v0, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v1, v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: .LBB117_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB117_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: .LBB117_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB117_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN1-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: .LBB118_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB118_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN2-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: .LBB118_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB118_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: .LBB118_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB118_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB119_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB119_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB119_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB119_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB119_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB119_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB120_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB120_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB120_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB120_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB120_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB120_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw min
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_min_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB121_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB121_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v6, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v7, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB121_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB121_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB121_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB121_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_min_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB122_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB122_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB122_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB122_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB122_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB122_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_min_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v5, v[5:6]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB123_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB123_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v4, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v5, v[5:6]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB123_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB123_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB123_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB123_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_min_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB124_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB124_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB124_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB124_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB124_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB124_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s35
|
|
; GCN1-NEXT: flat_load_dword v2, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v3, v[3:4]
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN1-NEXT: .LBB125_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB125_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s35
|
|
; GCN2-NEXT: flat_load_dword v2, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v3, v[3:4]
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN2-NEXT: .LBB125_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB125_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: .LBB125_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB125_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN1-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN1-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN1-NEXT: .LBB126_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB126_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN2-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s35
|
|
; GCN2-NEXT: flat_load_dword v3, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v2, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN2-NEXT: .LBB126_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB126_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: .LBB126_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB126_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v0, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v1, v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: .LBB127_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB127_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 4
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v0, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v1, v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: .LBB127_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB127_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: .LBB127_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB127_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN1-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN1-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN1-NEXT: .LBB128_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB128_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: s_add_u32 s36, s4, 36
|
|
; GCN2-NEXT: s_addc_u32 s37, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s36
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s37
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[2:3]
|
|
; GCN2-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN2-NEXT: .LBB128_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB128_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: s_mov_b64 s[34:35], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: .LBB128_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB128_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_min_i64_addr64_offset:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0xd
|
|
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x9
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN1-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN1-NEXT: .LBB129_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB129_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_min_i64_addr64_offset:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x34
|
|
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s0, s4
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, s5
|
|
; GCN2-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN2-NEXT: .LBB129_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB129_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_min_i64_addr64_offset:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x34
|
|
; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s4, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: .LBB129_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB129_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%gep = getelementptr i64, ptr %ptr, i64 4
|
|
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_min_i64_ret_addr64_offset:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x9
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN1-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s4
|
|
; GCN1-NEXT: .LBB130_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB130_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s2
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s3
|
|
; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_min_i64_ret_addr64_offset:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN2-NEXT: s_add_u32 s0, s0, 32
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s4
|
|
; GCN2-NEXT: .LBB130_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB130_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s2
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s3
|
|
; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_min_i64_ret_addr64_offset:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[10:11], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s4, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s9
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s8
|
|
; GCN3-NEXT: .LBB130_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[8:9], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB130_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%gep = getelementptr i64, ptr %ptr, i64 4
|
|
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
store i64 %tmp0, ptr %out2
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: atomic_min_i64:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x9
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN1-NEXT: .LBB131_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB131_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_min_i64:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s1
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, s3
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, s2
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s0
|
|
; GCN2-NEXT: .LBB131_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB131_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_min_i64:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
|
|
; GCN3-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s5
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s5
|
|
; GCN3-NEXT: .LBB131_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, v1
|
|
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, v0
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB131_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%tmp0 = atomicrmw min ptr %out, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
|
|
; GCN1-LABEL: atomic_min_i64_ret_addr64:
|
|
; GCN1: ; %bb.0: ; %entry
|
|
; GCN1-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x9
|
|
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN1-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN1-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN1-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v4, s5
|
|
; GCN1-NEXT: v_mov_b32_e32 v5, s4
|
|
; GCN1-NEXT: .LBB132_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB132_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s2
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s3
|
|
; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_endpgm
|
|
;
|
|
; GCN2-LABEL: atomic_min_i64_ret_addr64:
|
|
; GCN2: ; %bb.0: ; %entry
|
|
; GCN2-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
|
|
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
|
|
; GCN2-NEXT: s_add_u32 s0, s0, s6
|
|
; GCN2-NEXT: s_addc_u32 s1, s1, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v4, s5
|
|
; GCN2-NEXT: v_mov_b32_e32 v5, s4
|
|
; GCN2-NEXT: .LBB132_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB132_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s2
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s3
|
|
; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_endpgm
|
|
;
|
|
; GCN3-LABEL: atomic_min_i64_ret_addr64:
|
|
; GCN3: ; %bb.0: ; %entry
|
|
; GCN3-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
|
|
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
|
|
; GCN3-NEXT: s_lshl_b64 s[0:1], s[10:11], 3
|
|
; GCN3-NEXT: s_add_u32 s0, s4, s0
|
|
; GCN3-NEXT: s_addc_u32 s1, s5, s1
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s0
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s1
|
|
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: s_mov_b64 s[0:1], 0
|
|
; GCN3-NEXT: v_mov_b32_e32 v4, s9
|
|
; GCN3-NEXT: v_mov_b32_e32 v5, s8
|
|
; GCN3-NEXT: .LBB132_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v9, v3
|
|
; GCN3-NEXT: v_mov_b32_e32 v8, v2
|
|
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[8:9], v[8:9]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
|
|
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB132_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_endpgm
|
|
entry:
|
|
%ptr = getelementptr i64, ptr %out, i64 %index
|
|
%tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
store i64 %tmp0, ptr %out2
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB133_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN1-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB133_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v7, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v6, v[8:9]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB133_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
|
|
; GCN2-NEXT: v_mov_b32_e32 v7, v1
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: v_mov_b32_e32 v6, v0
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB133_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB133_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB133_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN1-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN1-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN1-NEXT: .LBB134_1: ; %atomicrmw.start
|
|
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN1-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_cbranch_execnz .LBB134_1
|
|
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_load_dword v1, v[0:1]
|
|
; GCN2-NEXT: flat_load_dword v0, v[4:5]
|
|
; GCN2-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN2-NEXT: .LBB134_1: ; %atomicrmw.start
|
|
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v9, v1
|
|
; GCN2-NEXT: v_mov_b32_e32 v8, v0
|
|
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
|
|
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
|
|
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
|
|
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_cbranch_execnz .LBB134_1
|
|
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
|
|
; GCN3-NEXT: s_mov_b64 s[4:5], 0
|
|
; GCN3-NEXT: .LBB134_1: ; %atomicrmw.start
|
|
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v7, v5
|
|
; GCN3-NEXT: v_mov_b32_e32 v6, v4
|
|
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
|
|
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
|
|
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
|
|
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
|
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: s_cbranch_execnz .LBB134_1
|
|
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
|
|
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, v4
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, v5
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw uinc_wrap
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_uinc_wrap_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_uinc_wrap_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] offset:32
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_uinc_wrap_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_uinc_wrap_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] offset:32
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] offset:32
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
; ---------------------------------------------------------------------
|
|
; atomicrmw udec_wrap
|
|
; ---------------------------------------------------------------------
|
|
|
|
define void @flat_atomic_udec_wrap_i64_noret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define void @flat_atomic_udec_wrap_i64_noret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] offset:32
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_udec_wrap_i64_ret(ptr %ptr, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define i64 @flat_atomic_udec_wrap_i64_ret_offset(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1] offset:32
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret void
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN1-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN1-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN1-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN1-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN1-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: s_add_u32 s34, s4, 32
|
|
; GCN2-NEXT: s_addc_u32 s35, s5, 0
|
|
; GCN2-NEXT: v_mov_b32_e32 v2, s34
|
|
; GCN2-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN2-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN2-NEXT: v_mov_b32_e32 v3, s35
|
|
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: v_mov_b32_e32 v0, s6
|
|
; GCN3-NEXT: v_mov_b32_e32 v1, s7
|
|
; GCN3-NEXT: v_mov_b32_e32 v2, s4
|
|
; GCN3-NEXT: v_mov_b32_e32 v3, s5
|
|
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
|
|
ret i64 %result
|
|
}
|
|
|
|
define void @flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] offset:32
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
|
|
ret void
|
|
}
|
|
|
|
define i64 @flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
|
|
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN1: ; %bb.0:
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
|
|
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN1-NEXT: buffer_wbinvl1_vol
|
|
; GCN1-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN2: ; %bb.0:
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
|
|
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
|
|
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
|
|
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN2-NEXT: buffer_wbinvl1_vol
|
|
; GCN2-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:
|
|
; GCN3: ; %bb.0:
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
|
|
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
|
|
; GCN3-NEXT: buffer_wbinvl1_vol
|
|
; GCN3-NEXT: s_setpc_b64 s[30:31]
|
|
%gep = getelementptr i64, ptr %out, i64 4
|
|
%result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
|
|
ret i64 %result
|
|
}
|
|
|
|
!0 = !{}
|
|
!1 = !{i32 5, i32 6}
|