359a792f9b
Previously when combining two loads this pass would sink the first one down to the second one, putting the combined load where the second one was. It would also sink any intervening instructions which depended on the first load down to just after the combined load. For example, if we started with this sequence of instructions (code flowing from left to right): X A B C D E F Y After combining loads X and Y into XY we might end up with: A B C D E F XY But if B D and F depended on X, we would get: A C E XY B D F Now if the original code had some short disjoint live ranges from A to B, C to D and E to F, in the transformed code these live ranges will be long and overlapping. In this way a single merge of two loads could cause an unbounded increase in register pressure. To fix this, change the way the way that loads are moved in order to merge them so that: - The second load is moved up to the first one. (But when merging stores, we still move the first store down to the second one.) - Intervening instructions are never moved. - Instead, if we find an intervening instruction that would need to be moved, give up on the merge. But this case should now be pretty rare because normal stores have no outputs, and normal loads only have address register inputs, but these will be identical for any pair of loads that we try to merge. As well as fixing the unbounded register pressure increase problem, moving loads up and stores down seems like it should usually be a win for memory latency reasons. Differential Revision: https://reviews.llvm.org/D119006
134 lines
5.7 KiB
LLVM
134 lines
5.7 KiB
LLVM
; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
|
|
|
|
|
|
; There is no dependence between the store and the two loads. So we can combine
|
|
; the loads and schedule it freely.
|
|
|
|
; GCN-LABEL: {{^}}ds_combine_nodep
|
|
|
|
; GCN-DAG: ds_write2_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset0:26 offset1:27
|
|
; GCN-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:7 offset1:8
|
|
; GCN: s_waitcnt lgkmcnt({{[0-9]+}})
|
|
define amdgpu_kernel void @ds_combine_nodep(float addrspace(1)* %out, float addrspace(3)* %inptr) {
|
|
|
|
%base = bitcast float addrspace(3)* %inptr to i8 addrspace(3)*
|
|
%addr0 = getelementptr i8, i8 addrspace(3)* %base, i32 24
|
|
%tmp0 = bitcast i8 addrspace(3)* %addr0 to float addrspace(3)*
|
|
%vaddr0 = bitcast float addrspace(3)* %tmp0 to <3 x float> addrspace(3)*
|
|
%load0 = load <3 x float>, <3 x float> addrspace(3)* %vaddr0, align 4
|
|
%v0 = extractelement <3 x float> %load0, i32 2
|
|
|
|
%tmp1 = insertelement <2 x float> undef, float 1.0, i32 0
|
|
%data = insertelement <2 x float> %tmp1, float 2.0, i32 1
|
|
|
|
%tmp2 = getelementptr float, float addrspace(3)* %inptr, i32 26
|
|
%vaddrs = bitcast float addrspace(3)* %tmp2 to <2 x float> addrspace(3)*
|
|
store <2 x float> %data, <2 x float> addrspace(3)* %vaddrs, align 4
|
|
|
|
%vaddr1 = getelementptr float, float addrspace(3)* %inptr, i32 7
|
|
%v1 = load float, float addrspace(3)* %vaddr1, align 4
|
|
|
|
%sum = fadd float %v0, %v1
|
|
store float %sum, float addrspace(1)* %out, align 4
|
|
ret void
|
|
}
|
|
|
|
|
|
; The store depends on the first load, so we could not move the first load down to combine with
|
|
; the second load directly. However, we can move the store after the combined load.
|
|
|
|
; GCN-LABEL: {{^}}ds_combine_WAR
|
|
|
|
; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:7 offset1:27
|
|
; GCN-NEXT: ds_write2_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset0:26 offset1:27
|
|
define amdgpu_kernel void @ds_combine_WAR(float addrspace(1)* %out, float addrspace(3)* %inptr) {
|
|
|
|
%base = bitcast float addrspace(3)* %inptr to i8 addrspace(3)*
|
|
%addr0 = getelementptr i8, i8 addrspace(3)* %base, i32 100
|
|
%tmp0 = bitcast i8 addrspace(3)* %addr0 to float addrspace(3)*
|
|
%vaddr0 = bitcast float addrspace(3)* %tmp0 to <3 x float> addrspace(3)*
|
|
%load0 = load <3 x float>, <3 x float> addrspace(3)* %vaddr0, align 4
|
|
%v0 = extractelement <3 x float> %load0, i32 2
|
|
|
|
%tmp1 = insertelement <2 x float> undef, float 1.0, i32 0
|
|
%data = insertelement <2 x float> %tmp1, float 2.0, i32 1
|
|
|
|
%tmp2 = getelementptr float, float addrspace(3)* %inptr, i32 26
|
|
%vaddrs = bitcast float addrspace(3)* %tmp2 to <2 x float> addrspace(3)*
|
|
store <2 x float> %data, <2 x float> addrspace(3)* %vaddrs, align 4
|
|
|
|
%vaddr1 = getelementptr float, float addrspace(3)* %inptr, i32 7
|
|
%v1 = load float, float addrspace(3)* %vaddr1, align 4
|
|
|
|
%sum = fadd float %v0, %v1
|
|
store float %sum, float addrspace(1)* %out, align 4
|
|
ret void
|
|
}
|
|
|
|
|
|
; The second load depends on the store. We could combine the two loads, putting
|
|
; the combined load at the original place of the second load, but we prefer to
|
|
; leave the first load near the start of the function to hide its latency.
|
|
|
|
; GCN-LABEL: {{^}}ds_combine_RAW
|
|
|
|
; GCN: ds_write2_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset0:26 offset1:27
|
|
; GCN-NEXT: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32
|
|
; GCN-NEXT: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:104
|
|
define amdgpu_kernel void @ds_combine_RAW(float addrspace(1)* %out, float addrspace(3)* %inptr) {
|
|
|
|
%base = bitcast float addrspace(3)* %inptr to i8 addrspace(3)*
|
|
%addr0 = getelementptr i8, i8 addrspace(3)* %base, i32 24
|
|
%tmp0 = bitcast i8 addrspace(3)* %addr0 to float addrspace(3)*
|
|
%vaddr0 = bitcast float addrspace(3)* %tmp0 to <3 x float> addrspace(3)*
|
|
%load0 = load <3 x float>, <3 x float> addrspace(3)* %vaddr0, align 4
|
|
%v0 = extractelement <3 x float> %load0, i32 2
|
|
|
|
%tmp1 = insertelement <2 x float> undef, float 1.0, i32 0
|
|
%data = insertelement <2 x float> %tmp1, float 2.0, i32 1
|
|
|
|
%tmp2 = getelementptr float, float addrspace(3)* %inptr, i32 26
|
|
%vaddrs = bitcast float addrspace(3)* %tmp2 to <2 x float> addrspace(3)*
|
|
store <2 x float> %data, <2 x float> addrspace(3)* %vaddrs, align 4
|
|
|
|
%vaddr1 = getelementptr float, float addrspace(3)* %inptr, i32 26
|
|
%v1 = load float, float addrspace(3)* %vaddr1, align 4
|
|
|
|
%sum = fadd float %v0, %v1
|
|
store float %sum, float addrspace(1)* %out, align 4
|
|
ret void
|
|
}
|
|
|
|
|
|
; The store depends on the first load, also the second load depends on the store.
|
|
; So we can not combine the two loads.
|
|
|
|
; GCN-LABEL: {{^}}ds_combine_WAR_RAW
|
|
|
|
; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:108
|
|
; GCN-NEXT: ds_write2_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset0:26 offset1:27
|
|
; GCN-NEXT: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:104
|
|
define amdgpu_kernel void @ds_combine_WAR_RAW(float addrspace(1)* %out, float addrspace(3)* %inptr) {
|
|
|
|
%base = bitcast float addrspace(3)* %inptr to i8 addrspace(3)*
|
|
%addr0 = getelementptr i8, i8 addrspace(3)* %base, i32 100
|
|
%tmp0 = bitcast i8 addrspace(3)* %addr0 to float addrspace(3)*
|
|
%vaddr0 = bitcast float addrspace(3)* %tmp0 to <3 x float> addrspace(3)*
|
|
%load0 = load <3 x float>, <3 x float> addrspace(3)* %vaddr0, align 4
|
|
%v0 = extractelement <3 x float> %load0, i32 2
|
|
|
|
%tmp1 = insertelement <2 x float> undef, float 1.0, i32 0
|
|
%data = insertelement <2 x float> %tmp1, float 2.0, i32 1
|
|
|
|
%tmp2 = getelementptr float, float addrspace(3)* %inptr, i32 26
|
|
%vaddrs = bitcast float addrspace(3)* %tmp2 to <2 x float> addrspace(3)*
|
|
store <2 x float> %data, <2 x float> addrspace(3)* %vaddrs, align 4
|
|
|
|
%vaddr1 = getelementptr float, float addrspace(3)* %inptr, i32 26
|
|
%v1 = load float, float addrspace(3)* %vaddr1, align 4
|
|
|
|
%sum = fadd float %v0, %v1
|
|
store float %sum, float addrspace(1)* %out, align 4
|
|
ret void
|
|
}
|