SIInsertWaitcnts inserts waitcnt instructions to resolve data dependencies. The GFX10+ vscnt (VMEM store count) counter is never used in this way. It is only used to resolve memory dependencies, and that is handled by SIMemoryLegalizer. Hence there is no need to conservatively wait for vscnt to be 0 on function entry and before returns. Differential Revision: https://reviews.llvm.org/D153537
683 lines
30 KiB
LLVM
683 lines
30 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
|
|
; RUN: llc -march=amdgcn -mcpu=gfx1100 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-F32FLUSH %s
|
|
; RUN: llc -march=amdgcn -mcpu=gfx1100 -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-F32DENORM %s
|
|
; RUN: llc -march=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX89,GFX9-F32FLUSH %s
|
|
; RUN: llc -march=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX89,GFX9-F32DENORM %s
|
|
; RUN: llc -march=amdgcn -mcpu=gfx803 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX89 %s
|
|
; RUN: llc -march=amdgcn -mcpu=gfx803 -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX89 %s
|
|
|
|
; fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
|
|
define float @fadd_fpext_fmul_f16_to_f32(half %x, half %y, float %z) #0 {
|
|
; GFX11-LABEL: fadd_fpext_fmul_f16_to_f32:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_cvt_f32_f16_e32 v0, v0
|
|
; GFX11-NEXT: v_add_f32_e32 v0, v0, v2
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fadd_fpext_fmul_f16_to_f32:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fadd_fpext_fmul_f16_to_f32:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v0, v0, v1
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v0, v0
|
|
; GFX9-F32DENORM-NEXT: v_add_f32_e32 v0, v0, v2
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul half %x, %y
|
|
%mul.ext = fpext half %mul to float
|
|
%add = fadd float %mul.ext, %z
|
|
ret float %add
|
|
}
|
|
|
|
; f16->f64 is not free.
|
|
define double @fadd_fpext_fmul_f16_to_f64(half %x, half %y, double %z) #0 {
|
|
; GFX11-LABEL: fadd_fpext_fmul_f16_to_f64:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_cvt_f32_f16_e32 v0, v0
|
|
; GFX11-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
|
|
; GFX11-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX89-LABEL: fadd_fpext_fmul_f16_to_f64:
|
|
; GFX89: ; %bb.0: ; %entry
|
|
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX89-NEXT: v_mul_f16_e32 v0, v0, v1
|
|
; GFX89-NEXT: v_cvt_f32_f16_e32 v0, v0
|
|
; GFX89-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
|
|
; GFX89-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
|
|
; GFX89-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul half %x, %y
|
|
%mul.ext = fpext half %mul to double
|
|
%add = fadd double %mul.ext, %z
|
|
ret double %add
|
|
}
|
|
|
|
; f32->f64 is not free.
|
|
define double @fadd_fpext_fmul_f32_to_f64(float %x, float %y, double %z) #0 {
|
|
; GFX11-LABEL: fadd_fpext_fmul_f32_to_f64:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f32_e32 v0, v0, v1
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
|
|
; GFX11-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX89-LABEL: fadd_fpext_fmul_f32_to_f64:
|
|
; GFX89: ; %bb.0: ; %entry
|
|
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX89-NEXT: v_mul_f32_e32 v0, v0, v1
|
|
; GFX89-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
|
|
; GFX89-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
|
|
; GFX89-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul float %x, %y
|
|
%mul.ext = fpext float %mul to double
|
|
%add = fadd double %mul.ext, %z
|
|
ret double %add
|
|
}
|
|
|
|
; fold (fadd x, (fpext (fmul y, z))) -> (fma (fpext y), (fpext z), x)
|
|
define float @fadd_fpext_fmul_f16_to_f32_commute(half %x, half %y, float %z) #0 {
|
|
; GFX11-LABEL: fadd_fpext_fmul_f16_to_f32_commute:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_cvt_f32_f16_e32 v0, v0
|
|
; GFX11-NEXT: v_add_f32_e32 v0, v2, v0
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fadd_fpext_fmul_f16_to_f32_commute:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fadd_fpext_fmul_f16_to_f32_commute:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v0, v0, v1
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v0, v0
|
|
; GFX9-F32DENORM-NEXT: v_add_f32_e32 v0, v2, v0
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul half %x, %y
|
|
%mul.ext = fpext half %mul to float
|
|
%add = fadd float %z, %mul.ext
|
|
ret float %add
|
|
}
|
|
|
|
; fold (fadd (fma x, y, (fpext (fmul u, v))), z)
|
|
; -> (fma x, y, (fma (fpext u), (fpext v), z))
|
|
define float @fadd_muladd_fpext_fmul_f16_to_f32(float %x, float %y, half %u, half %v, float %z) #0 {
|
|
; GFX11-LABEL: fadd_muladd_fpext_fmul_f16_to_f32:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v2, v2, v3
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1]
|
|
; GFX11-NEXT: v_add_f32_e32 v0, v0, v4
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fadd_muladd_fpext_fmul_f16_to_f32:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v2, v2, v3, v4 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1
|
|
; GFX9-F32FLUSH-NEXT: v_mov_b32_e32 v0, v2
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fadd_muladd_fpext_fmul_f16_to_f32:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2
|
|
; GFX9-F32DENORM-NEXT: v_fma_f32 v0, v0, v1, v2
|
|
; GFX9-F32DENORM-NEXT: v_add_f32_e32 v0, v0, v4
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul half %u, %v
|
|
%mul.ext = fpext half %mul to float
|
|
%fma = call float @llvm.fmuladd.f32(float %x, float %y, float %mul.ext)
|
|
%add = fadd float %fma, %z
|
|
ret float %add
|
|
}
|
|
|
|
; fold (fadd x, (fma y, z, (fpext (fmul u, v)))
|
|
; -> (fma y, z, (fma (fpext u), (fpext v), x))
|
|
define float @fadd_muladd_fpext_fmul_f16_to_f32_commute(float %x, float %y, half %u, half %v, float %z) #0 {
|
|
; GFX11-LABEL: fadd_muladd_fpext_fmul_f16_to_f32_commute:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v2, v2, v3
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1]
|
|
; GFX11-NEXT: v_add_f32_e32 v0, v4, v0
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fadd_muladd_fpext_fmul_f16_to_f32_commute:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v2, v2, v3, v4 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1
|
|
; GFX9-F32FLUSH-NEXT: v_mov_b32_e32 v0, v2
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fadd_muladd_fpext_fmul_f16_to_f32_commute:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2
|
|
; GFX9-F32DENORM-NEXT: v_fma_f32 v0, v0, v1, v2
|
|
; GFX9-F32DENORM-NEXT: v_add_f32_e32 v0, v4, v0
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul half %u, %v
|
|
%mul.ext = fpext half %mul to float
|
|
%fma = call float @llvm.fmuladd.f32(float %x, float %y, float %mul.ext)
|
|
%add = fadd float %z, %fma
|
|
ret float %add
|
|
}
|
|
|
|
define float @fadd_fmad_fpext_fmul_f16_to_f32(float %x, float %y, half %u, half %v, float %z) #0 {
|
|
; GFX11-LABEL: fadd_fmad_fpext_fmul_f16_to_f32:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v2, v2, v3
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1]
|
|
; GFX11-NEXT: v_add_f32_e32 v0, v0, v4
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fadd_fmad_fpext_fmul_f16_to_f32:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v2, v2, v3, v4 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1
|
|
; GFX9-F32FLUSH-NEXT: v_mov_b32_e32 v0, v2
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fadd_fmad_fpext_fmul_f16_to_f32:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2
|
|
; GFX9-F32DENORM-NEXT: v_fma_f32 v0, v0, v1, v2
|
|
; GFX9-F32DENORM-NEXT: v_add_f32_e32 v0, v0, v4
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul half %u, %v
|
|
%mul.ext = fpext half %mul to float
|
|
%mul1 = fmul contract float %x, %y
|
|
%fmad = fadd contract float %mul1, %mul.ext
|
|
%add = fadd float %fmad, %z
|
|
ret float %add
|
|
}
|
|
|
|
; fold (fadd (fma x, y, (fpext (fmul u, v))), z)
|
|
; -> (fma x, y, (fma (fpext u), (fpext v), z))
|
|
define float @fadd_fma_fpext_fmul_f16_to_f32(float %x, float %y, half %u, half %v, float %z) #0 {
|
|
; GFX11-LABEL: fadd_fma_fpext_fmul_f16_to_f32:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v2, v2, v3
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1]
|
|
; GFX11-NEXT: v_add_f32_e32 v0, v0, v4
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fadd_fma_fpext_fmul_f16_to_f32:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v2, v2, v3, v4 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1
|
|
; GFX9-F32FLUSH-NEXT: v_mov_b32_e32 v0, v2
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fadd_fma_fpext_fmul_f16_to_f32:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2
|
|
; GFX9-F32DENORM-NEXT: v_fma_f32 v0, v0, v1, v2
|
|
; GFX9-F32DENORM-NEXT: v_add_f32_e32 v0, v0, v4
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul contract half %u, %v
|
|
%mul.ext = fpext half %mul to float
|
|
%fma = call float @llvm.fma.f32(float %x, float %y, float %mul.ext)
|
|
%add = fadd float %fma, %z
|
|
ret float %add
|
|
}
|
|
|
|
define float @fadd_fma_fpext_fmul_f16_to_f32_commute(float %x, float %y, half %u, half %v, float %z) #0 {
|
|
; GFX11-LABEL: fadd_fma_fpext_fmul_f16_to_f32_commute:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v2, v2, v3
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_fma_mix_f32 v0, v0, v1, v2 op_sel_hi:[0,0,1]
|
|
; GFX11-NEXT: v_add_f32_e32 v0, v4, v0
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fadd_fma_fpext_fmul_f16_to_f32_commute:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v2, v2, v3, v4 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1
|
|
; GFX9-F32FLUSH-NEXT: v_mov_b32_e32 v0, v2
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fadd_fma_fpext_fmul_f16_to_f32_commute:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2
|
|
; GFX9-F32DENORM-NEXT: v_fma_f32 v0, v0, v1, v2
|
|
; GFX9-F32DENORM-NEXT: v_add_f32_e32 v0, v4, v0
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul contract half %u, %v
|
|
%mul.ext = fpext half %mul to float
|
|
%fma = call float @llvm.fma.f32(float %x, float %y, float %mul.ext)
|
|
%add = fadd float %z, %fma
|
|
ret float %add
|
|
}
|
|
|
|
; fold (fadd x, (fpext (fma y, z, (fmul u, v)))
|
|
; -> (fma (fpext y), (fpext z), (fma (fpext u), (fpext v), x))
|
|
define float @fadd_fpext_fmuladd_f16_to_f32(float %x, half %y, half %z, half %u, half %v) #0 {
|
|
; GFX11-LABEL: fadd_fpext_fmuladd_f16_to_f32:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v3, v3, v4
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_fmac_f16_e32 v3, v1, v2
|
|
; GFX11-NEXT: v_cvt_f32_f16_e32 v1, v3
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
|
|
; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fadd_fpext_fmuladd_f16_to_f32:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v3, v4, v0 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v1, v2, v0 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fadd_fpext_fmuladd_f16_to_f32:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v3, v3, v4
|
|
; GFX9-F32DENORM-NEXT: v_fma_f16 v1, v1, v2, v3
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v1, v1
|
|
; GFX9-F32DENORM-NEXT: v_add_f32_e32 v0, v0, v1
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul contract half %u, %v
|
|
%fma = call half @llvm.fmuladd.f16(half %y, half %z, half %mul)
|
|
%ext.fma = fpext half %fma to float
|
|
%add = fadd float %x, %ext.fma
|
|
ret float %add
|
|
}
|
|
|
|
define float @fadd_fpext_fma_f16_to_f32(float %x, half %y, half %z, half %u, half %v) #0 {
|
|
; GFX11-LABEL: fadd_fpext_fma_f16_to_f32:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v3, v3, v4
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_fmac_f16_e32 v3, v1, v2
|
|
; GFX11-NEXT: v_cvt_f32_f16_e32 v1, v3
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
|
|
; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fadd_fpext_fma_f16_to_f32:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v3, v4, v0 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v1, v2, v0 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fadd_fpext_fma_f16_to_f32:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v3, v3, v4
|
|
; GFX9-F32DENORM-NEXT: v_fma_f16 v1, v1, v2, v3
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v1, v1
|
|
; GFX9-F32DENORM-NEXT: v_add_f32_e32 v0, v0, v1
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul contract half %u, %v
|
|
%fma = call half @llvm.fma.f16(half %y, half %z, half %mul)
|
|
%ext.fma = fpext half %fma to float
|
|
%add = fadd float %x, %ext.fma
|
|
ret float %add
|
|
}
|
|
|
|
define float @fadd_fpext_fma_f16_to_f32_commute(float %x, half %y, half %z, half %u, half %v) #0 {
|
|
; GFX11-LABEL: fadd_fpext_fma_f16_to_f32_commute:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v3, v3, v4
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_fmac_f16_e32 v3, v1, v2
|
|
; GFX11-NEXT: v_cvt_f32_f16_e32 v1, v3
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
|
|
; GFX11-NEXT: v_add_f32_e32 v0, v1, v0
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fadd_fpext_fma_f16_to_f32_commute:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v3, v4, v0 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v1, v2, v0 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fadd_fpext_fma_f16_to_f32_commute:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v3, v3, v4
|
|
; GFX9-F32DENORM-NEXT: v_fma_f16 v1, v1, v2, v3
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v1, v1
|
|
; GFX9-F32DENORM-NEXT: v_add_f32_e32 v0, v1, v0
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul contract half %u, %v
|
|
%fma = call half @llvm.fma.f16(half %y, half %z, half %mul)
|
|
%ext.fma = fpext half %fma to float
|
|
%add = fadd float %ext.fma, %x
|
|
ret float %add
|
|
}
|
|
|
|
; fold (fsub (fpext (fmul x, y)), z)
|
|
; -> (fma (fpext x), (fpext y), (fneg z))
|
|
define float @fsub_fpext_fmul_f16_to_f32(half %x, half %y, float %z) #0 {
|
|
; GFX11-LABEL: fsub_fpext_fmul_f16_to_f32:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_cvt_f32_f16_e32 v0, v0
|
|
; GFX11-NEXT: v_sub_f32_e32 v0, v0, v2
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fsub_fpext_fmul_f16_to_f32:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, v1, -v2 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fsub_fpext_fmul_f16_to_f32:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v0, v0, v1
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v0, v0
|
|
; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v2
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul half %x, %y
|
|
%mul.ext = fpext half %mul to float
|
|
%add = fsub float %mul.ext, %z
|
|
ret float %add
|
|
}
|
|
|
|
; fold (fsub x, (fpext (fmul y, z)))
|
|
; -> (fma (fneg (fpext y)), (fpext z), x)
|
|
define float @fsub_fpext_fmul_f16_to_f32_commute(float %x, half %y, half %z) #0 {
|
|
; GFX11-F32FLUSH-LABEL: fsub_fpext_fmul_f16_to_f32_commute:
|
|
; GFX11-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX11-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-F32FLUSH-NEXT: v_fma_mix_f32 v0, -v1, v2, v0 op_sel_hi:[1,1,0]
|
|
; GFX11-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX11-F32DENORM-LABEL: fsub_fpext_fmul_f16_to_f32_commute:
|
|
; GFX11-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX11-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-F32DENORM-NEXT: v_mul_f16_e32 v1, v1, v2
|
|
; GFX11-F32DENORM-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-F32DENORM-NEXT: v_cvt_f32_f16_e32 v1, v1
|
|
; GFX11-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v1
|
|
; GFX11-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fsub_fpext_fmul_f16_to_f32_commute:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, -v1, v2, v0 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fsub_fpext_fmul_f16_to_f32_commute:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v1, v1, v2
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v1, v1
|
|
; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v1
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul contract half %y, %z
|
|
%mul.ext = fpext half %mul to float
|
|
%add = fsub contract float %x, %mul.ext
|
|
ret float %add
|
|
}
|
|
|
|
; fold (fsub (fpext (fneg (fmul, x, y))), z)
|
|
; -> (fneg (fma (fpext x), (fpext y), z))
|
|
define float @fsub_fpext_fneg_fmul_f16_to_f32(half %x, half %y, float %z) #0 {
|
|
; GFX11-LABEL: fsub_fpext_fneg_fmul_f16_to_f32:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e64 v0, v0, -v1
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_cvt_f32_f16_e32 v0, v0
|
|
; GFX11-NEXT: v_sub_f32_e32 v0, v0, v2
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fsub_fpext_fneg_fmul_f16_to_f32:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, -v1, -v2 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fsub_fpext_fneg_fmul_f16_to_f32:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e64 v0, v0, -v1
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v0, v0
|
|
; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v2
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul half %x, %y
|
|
%neg.mul = fsub half -0.0, %mul
|
|
%neg.mul.ext = fpext half %neg.mul to float
|
|
%add = fsub float %neg.mul.ext, %z
|
|
ret float %add
|
|
}
|
|
|
|
; fold (fsub (fneg (fpext (fmul, x, y))), z)
|
|
; -> (fneg (fma (fpext x)), (fpext y), z)
|
|
define float @fsub_fneg_fpext_fmul_f16_to_f32(half %x, half %y, float %z) #0 {
|
|
; GFX11-LABEL: fsub_fneg_fpext_fmul_f16_to_f32:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e64 v0, v0, -v1
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_cvt_f32_f16_e32 v0, v0
|
|
; GFX11-NEXT: v_sub_f32_e32 v0, v0, v2
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fsub_fneg_fpext_fmul_f16_to_f32:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, -v1, -v2 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fsub_fneg_fpext_fmul_f16_to_f32:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e64 v0, v0, -v1
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v0, v0
|
|
; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v2
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul half %x, %y
|
|
%mul.ext = fpext half %mul to float
|
|
%neg.mul.ext = fneg float %mul.ext
|
|
%add = fsub float %neg.mul.ext, %z
|
|
ret float %add
|
|
}
|
|
|
|
; fold (fsub (fmad x, y, (fpext (fmul u, v))), z)
|
|
; -> (fmad x, y (fmad (fpext u), (fpext v), (fneg z)))
|
|
define float @fsub_muladd_fpext_mul_f16_to_f32(float %x, float %y, float %z, half %u, half %v) #0 {
|
|
; GFX11-LABEL: fsub_muladd_fpext_mul_f16_to_f32:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v3, v3, v4
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_fma_mix_f32 v0, v0, v1, v3 op_sel_hi:[0,0,1]
|
|
; GFX11-NEXT: v_sub_f32_e32 v0, v0, v2
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fsub_muladd_fpext_mul_f16_to_f32:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v2, v3, v4, -v2 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1
|
|
; GFX9-F32FLUSH-NEXT: v_mov_b32_e32 v0, v2
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fsub_muladd_fpext_mul_f16_to_f32:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v3, v3, v4
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v3, v3
|
|
; GFX9-F32DENORM-NEXT: v_fma_f32 v0, v0, v1, v3
|
|
; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v2
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul reassoc half %u, %v
|
|
%mul.ext = fpext half %mul to float
|
|
%fma = call float @llvm.fmuladd.f32(float %x, float %y, float %mul.ext)
|
|
%add = fsub reassoc float %fma, %z
|
|
ret float %add
|
|
}
|
|
|
|
; fold (fsub (fpext (fmad x, y, (fmul u, v))), z)
|
|
; -> (fmad (fpext x), (fpext y),
|
|
; (fmad (fpext u), (fpext v), (fneg z)))
|
|
define float @fsub_fpext_muladd_mul_f16_to_f32(half %x, half %y, float %z, half %u, half %v) #0 {
|
|
; GFX11-LABEL: fsub_fpext_muladd_mul_f16_to_f32:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v3, v3, v4
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_fmac_f16_e32 v3, v0, v1
|
|
; GFX11-NEXT: v_cvt_f32_f16_e32 v0, v3
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
|
|
; GFX11-NEXT: v_sub_f32_e32 v0, v0, v2
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX89-LABEL: fsub_fpext_muladd_mul_f16_to_f32:
|
|
; GFX89: ; %bb.0: ; %entry
|
|
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX89-NEXT: v_mul_f16_e32 v3, v3, v4
|
|
; GFX89-NEXT: v_fma_f16 v0, v0, v1, v3
|
|
; GFX89-NEXT: v_cvt_f32_f16_e32 v0, v0
|
|
; GFX89-NEXT: v_sub_f32_e32 v0, v0, v2
|
|
; GFX89-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul half %u, %v
|
|
%fma = call half @llvm.fmuladd.f16(half %x, half %y, half %mul)
|
|
%fma.ext = fpext half %fma to float
|
|
%add = fsub float %fma.ext, %z
|
|
ret float %add
|
|
}
|
|
|
|
; fold (fsub x, (fmad y, z, (fpext (fmul u, v))))
|
|
; -> (fmad (fneg y), z, (fmad (fneg (fpext u)), (fpext v), x))
|
|
define float @fsub_muladd_fpext_mul_f16_to_f32_commute(float %x, float %y, float %z, half %u, half %v) #0 {
|
|
; GFX11-LABEL: fsub_muladd_fpext_mul_f16_to_f32_commute:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v3, v3, v4
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_fma_mix_f32 v1, v1, v2, v3 op_sel_hi:[0,0,1]
|
|
; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32FLUSH-LABEL: fsub_muladd_fpext_mul_f16_to_f32_commute:
|
|
; GFX9-F32FLUSH: ; %bb.0: ; %entry
|
|
; GFX9-F32FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, -v3, v4, v0 op_sel_hi:[1,1,0]
|
|
; GFX9-F32FLUSH-NEXT: v_mad_f32 v0, -v1, v2, v0
|
|
; GFX9-F32FLUSH-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX9-F32DENORM-LABEL: fsub_muladd_fpext_mul_f16_to_f32_commute:
|
|
; GFX9-F32DENORM: ; %bb.0: ; %entry
|
|
; GFX9-F32DENORM-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v3, v3, v4
|
|
; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v3, v3
|
|
; GFX9-F32DENORM-NEXT: v_fma_f32 v1, v1, v2, v3
|
|
; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v1
|
|
; GFX9-F32DENORM-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul reassoc half %u, %v
|
|
%mul.ext = fpext half %mul to float
|
|
%fma = call float @llvm.fmuladd.f32(float %y, float %z, float %mul.ext)
|
|
%add = fsub reassoc float %x, %fma
|
|
ret float %add
|
|
}
|
|
|
|
; fold (fsub x, (fpext (fma y, z, (fmul u, v))))
|
|
; -> (fma (fneg (fpext y)), (fpext z),
|
|
; (fma (fneg (fpext u)), (fpext v), x))
|
|
define float @fsub_fpext_muladd_mul_f16_to_f32_commute(float %x, half %y, half %z, half %u, half %v) #0 {
|
|
; GFX11-LABEL: fsub_fpext_muladd_mul_f16_to_f32_commute:
|
|
; GFX11: ; %bb.0: ; %entry
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX11-NEXT: v_mul_f16_e32 v3, v3, v4
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
|
|
; GFX11-NEXT: v_fmac_f16_e32 v3, v1, v2
|
|
; GFX11-NEXT: v_cvt_f32_f16_e32 v1, v3
|
|
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
|
|
; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1
|
|
; GFX11-NEXT: s_setpc_b64 s[30:31]
|
|
;
|
|
; GFX89-LABEL: fsub_fpext_muladd_mul_f16_to_f32_commute:
|
|
; GFX89: ; %bb.0: ; %entry
|
|
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
|
|
; GFX89-NEXT: v_mul_f16_e32 v3, v3, v4
|
|
; GFX89-NEXT: v_fma_f16 v1, v1, v2, v3
|
|
; GFX89-NEXT: v_cvt_f32_f16_e32 v1, v1
|
|
; GFX89-NEXT: v_sub_f32_e32 v0, v0, v1
|
|
; GFX89-NEXT: s_setpc_b64 s[30:31]
|
|
entry:
|
|
%mul = fmul half %u, %v
|
|
%fma = call half @llvm.fmuladd.f16(half %y, half %z, half %mul)
|
|
%fma.ext = fpext half %fma to float
|
|
%add = fsub float %x, %fma.ext
|
|
ret float %add
|
|
}
|
|
|
|
declare float @llvm.fmuladd.f32(float, float, float) #0
|
|
declare float @llvm.fma.f32(float, float, float) #0
|
|
declare half @llvm.fmuladd.f16(half, half, half) #0
|
|
declare half @llvm.fma.f16(half, half, half) #0
|
|
|
|
attributes #0 = { nounwind readnone speculatable }
|