caio/clang-p2996
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[RISCV] Match vcompress during shuffle lowering (#117748)

Browse Source 
This change matches a subset of vcompress patterns during shuffle
lowering. The subset implemented requires a contiguous prefix of
demanded elements followed by undefs. This subset was chosen for two
reasons: 1) which elements to spurious demand is a non-obvious problem,
and 2) my first several attempts at implementing the general case were
buggy. I decided to go with the simple case to start with.

vcompress scales better with LMUL than a general vrgather, and at least
the SpaceMit X60, has higher throughput even at m1. It also has the
advantage of requiring smaller vector constants at one bit per element
as opposed to vrgather which is a minimum of 8 bits per element. The
downside to using vcompress is that we can't fold a vselect into it, as
there is no masked vcompress variant.

For reference, here are the relevant throughputs from camel-cdr's data
table on BP3 (X60):
  vrgather.vv v8,v16,v24    4.0  16.0  64.0  256.0
  vcompress.vm v8,v16,v24   3.0  10.0  36.0  136.
  vmerge.vvm v8,v16,v24,v0  2.0  4.0   8.0   16.0

The largest concern with the extra vmerge is that we locally increase
register pressure. If we do have masking, we also have a passthru,
without the ability to fold that into the vcompress, we need to keep it
alive a bit longer. This can hurt at e.g. m8 where we have very few
architectural registers. As compared with the vrgather.vv sequence, this
is only one additional m1 VREG - since we no longer need the index
vector. It compares slightly worse against vrgatherie16.vv which can use
index vectors smaller than other operands. Note that we could
potentially fold the vmerge if only tail elements are being preserved; I
haven't investigated this.

It is unfortunately hard given our current lowering structure to know if
we're emitting a shuffle where masking will follow. Thankfully, it
doesn't seem to show up much in practice, so I think we can probably
ignore it.

This patch only handles single source compress idioms at the moment.
This is an effort to avoid interacting with other patches on review for
changing how we canonicalize length changing shuffles.
This commit is contained in:
Philip Reames
2024-11-27 13:23:18 -08:00
committed by GitHub
parent 1669ac434c
commit febbf9105f
8 changed files with 864 additions and 779 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
llvm/lib/Target/RISCV/RISCVISelLowering.cpp
View File

@@ -5155,6 +5155,28 @@ static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN,
return convertFromScalableVector(VT, Vec, DAG, Subtarget);
}
// Matches a subset of compress masks with a contiguous prefix of output
// elements. This could be extended to allow gaps by deciding which
// source elements to spuriously demand.
static bool isCompressMask(ArrayRef<int> Mask) {
int Last = -1;
bool SawUndef = false;
for (int i = 0; i < Mask.size(); i++) {
if (Mask[i] == -1) {
SawUndef = true;
continue;
}
if (SawUndef)
return false;
if (i > Mask[i])
return false;
if (Mask[i] <= Last)
return false;
Last = Mask[i];
}
return true;
}
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
const RISCVSubtarget &Subtarget) {
SDValue V1 = Op.getOperand(0);
@@ -5372,6 +5394,25 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
return V;
// Can we generate a vcompress instead of a vrgather? These scale better
// at high LMUL, at the cost of not being able to fold a following select
// into them. The mask constants are also smaller than the index vector
// constants, and thus easier to materialize.
if (isCompressMask(Mask)) {
SmallVector<SDValue> MaskVals(NumElts,
DAG.getConstant(false, DL, XLenVT));
for (auto Idx : Mask) {
if (Idx == -1)
break;
assert(Idx >= 0 && (unsigned)Idx < NumElts);
MaskVals[Idx] = DAG.getConstant(true, DL, XLenVT);
}
MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
SDValue CompressMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
return DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, V1, CompressMask,
DAG.getUNDEF(VT));
}
if (VT.getScalarSizeInBits() == 8 &&
any_of(Mask, [&](const auto &Idx) { return Idx > 255; })) {
// On such a vector we're unable to use i8 as the index type.

14
llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
View File

@@ -40,16 +40,16 @@ define <4 x float> @hang_when_merging_stores_after_legalization(<8 x float> %x,
; CHECK-LABEL: hang_when_merging_stores_after_legalization:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT: vid.v v12
; CHECK-NEXT: vmv.v.i v12, -14
; CHECK-NEXT: vid.v v14
; CHECK-NEXT: li a0, 7
; CHECK-NEXT: vmadd.vx v14, a0, v12
; CHECK-NEXT: li a0, 129
; CHECK-NEXT: vmv.s.x v15, a0
; CHECK-NEXT: vmv.v.i v0, 12
; CHECK-NEXT: vmul.vx v14, v12, a0
; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma
; CHECK-NEXT: vrgatherei16.vv v12, v8, v14
; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma
; CHECK-NEXT: vadd.vi v8, v14, -14
; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, mu
; CHECK-NEXT: vrgatherei16.vv v12, v10, v8, v0.t
; CHECK-NEXT: vcompress.vm v12, v8, v15
; CHECK-NEXT: vrgatherei16.vv v12, v10, v14, v0.t
; CHECK-NEXT: vmv1r.v v8, v12
; CHECK-NEXT: ret
%z = shufflevector <8 x float> %x, <8 x float> %y, <4 x i32> <i32 0, i32 7, i32 8, i32 15>

16
llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll
View File

@@ -138,17 +138,17 @@ define <4 x double> @vrgather_shuffle_xv_v4f64(<4 x double> %x) {
define <4 x double> @vrgather_shuffle_vx_v4f64(<4 x double> %x) {
; CHECK-LABEL: vrgather_shuffle_vx_v4f64:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
; CHECK-NEXT: vid.v v10
; CHECK-NEXT: lui a0, %hi(.LCPI9_0)
; CHECK-NEXT: fld fa5, %lo(.LCPI9_0)(a0)
; CHECK-NEXT: li a0, 3
; CHECK-NEXT: vmul.vx v12, v10, a0
; CHECK-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
; CHECK-NEXT: vmv.v.i v10, 9
; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma
; CHECK-NEXT: vcompress.vm v12, v8, v10
; CHECK-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
; CHECK-NEXT: vmv.v.i v0, 3
; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, mu
; CHECK-NEXT: vfmv.v.f v10, fa5
; CHECK-NEXT: vrgatherei16.vv v10, v8, v12, v0.t
; CHECK-NEXT: vmv.v.v v8, v10
; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma
; CHECK-NEXT: vfmv.v.f v8, fa5
; CHECK-NEXT: vmerge.vvm v8, v8, v12, v0
; CHECK-NEXT: ret
%s = shufflevector <4 x double> %x, <4 x double> <double 2.0, double 2.0, double 2.0, double 2.0>, <4 x i32> <i32 0, i32 3, i32 6, i32 5>
ret <4 x double> %s

63
llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll
View File

@@ -113,14 +113,12 @@ define <4 x i16> @vrgather_shuffle_xv_v4i16(<4 x i16> %x) {
define <4 x i16> @vrgather_shuffle_vx_v4i16(<4 x i16> %x) {
; CHECK-LABEL: vrgather_shuffle_vx_v4i16:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, mu
; CHECK-NEXT: vid.v v9
; CHECK-NEXT: li a0, 3
; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
; CHECK-NEXT: vmv.v.i v9, 9
; CHECK-NEXT: vmv.v.i v0, 3
; CHECK-NEXT: vmul.vx v10, v9, a0
; CHECK-NEXT: vmv.v.i v9, 5
; CHECK-NEXT: vrgather.vv v9, v8, v10, v0.t
; CHECK-NEXT: vmv1r.v v8, v9
; CHECK-NEXT: vcompress.vm v10, v8, v9
; CHECK-NEXT: vmv.v.i v8, 5
; CHECK-NEXT: vmerge.vvm v8, v8, v10, v0
; CHECK-NEXT: ret
%s = shufflevector <4 x i16> %x, <4 x i16> <i16 5, i16 5, i16 5, i16 5>, <4 x i32> <i32 0, i32 3, i32 6, i32 5>
ret <4 x i16> %s
@@ -723,21 +721,22 @@ define <8 x i32> @shuffle_v8i32_2(<8 x i32> %x, <8 x i32> %y) {
define <8 x i8> @shuffle_v64i8_v8i8(<64 x i8> %wide.vec) {
; CHECK-LABEL: shuffle_v64i8_v8i8:
; CHECK: # %bb.0:
; CHECK-NEXT: li a0, 32
; CHECK-NEXT: lui a0, 4112
; CHECK-NEXT: li a1, 240
; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma
; CHECK-NEXT: vmv.s.x v0, a1
; CHECK-NEXT: lui a1, 98561
; CHECK-NEXT: vsetvli zero, a0, e8, m2, ta, ma
; CHECK-NEXT: vid.v v12
; CHECK-NEXT: vsll.vi v14, v12, 3
; CHECK-NEXT: vrgather.vv v12, v8, v14
; CHECK-NEXT: vsetvli zero, a0, e8, m4, ta, ma
; CHECK-NEXT: vslidedown.vx v8, v8, a0
; CHECK-NEXT: addi a1, a1, -2048
; CHECK-NEXT: li a1, 32
; CHECK-NEXT: addi a0, a0, 257
; CHECK-NEXT: vmv.s.x v14, a0
; CHECK-NEXT: lui a0, 98561
; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma
; CHECK-NEXT: vcompress.vm v12, v8, v14
; CHECK-NEXT: vsetvli zero, a1, e8, m4, ta, ma
; CHECK-NEXT: vslidedown.vx v8, v8, a1
; CHECK-NEXT: addi a0, a0, -2048
; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-NEXT: vmv.v.x v10, a1
; CHECK-NEXT: vsetvli zero, a0, e8, m2, ta, mu
; CHECK-NEXT: vmv.v.x v10, a0
; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, mu
; CHECK-NEXT: vrgather.vv v12, v8, v10, v0.t
; CHECK-NEXT: vmv1r.v v8, v12
; CHECK-NEXT: ret
@@ -748,11 +747,10 @@ define <8 x i8> @shuffle_v64i8_v8i8(<64 x i8> %wide.vec) {
define <8 x i8> @shuffle_compress_singlesrc_e8(<8 x i8> %v) {
; CHECK-LABEL: shuffle_compress_singlesrc_e8:
; CHECK: # %bb.0:
; CHECK-NEXT: lui a0, %hi(.LCPI49_0)
; CHECK-NEXT: addi a0, a0, %lo(.LCPI49_0)
; CHECK-NEXT: li a0, 181
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vle8.v v10, (a0)
; CHECK-NEXT: vrgather.vv v9, v8, v10
; CHECK-NEXT: vmv.s.x v10, a0
; CHECK-NEXT: vcompress.vm v9, v8, v10
; CHECK-NEXT: vmv1r.v v8, v9
; CHECK-NEXT: ret
%out = shufflevector <8 x i8> %v, <8 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 5, i32 7, i32 undef, i32 undef, i32 undef>
@@ -762,11 +760,10 @@ define <8 x i8> @shuffle_compress_singlesrc_e8(<8 x i8> %v) {
define <8 x i16> @shuffle_compress_singlesrc_e16(<8 x i16> %v) {
; CHECK-LABEL: shuffle_compress_singlesrc_e16:
; CHECK: # %bb.0:
; CHECK-NEXT: lui a0, %hi(.LCPI50_0)
; CHECK-NEXT: addi a0, a0, %lo(.LCPI50_0)
; CHECK-NEXT: li a0, 181
; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT: vle16.v v10, (a0)
; CHECK-NEXT: vrgather.vv v9, v8, v10
; CHECK-NEXT: vmv.s.x v10, a0
; CHECK-NEXT: vcompress.vm v9, v8, v10
; CHECK-NEXT: vmv.v.v v8, v9
; CHECK-NEXT: ret
%out = shufflevector <8 x i16> %v, <8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 5, i32 7, i32 undef, i32 undef, i32 undef>
@@ -776,11 +773,10 @@ define <8 x i16> @shuffle_compress_singlesrc_e16(<8 x i16> %v) {
define <8 x i32> @shuffle_compress_singlesrc_e32(<8 x i32> %v) {
; CHECK-LABEL: shuffle_compress_singlesrc_e32:
; CHECK: # %bb.0:
; CHECK-NEXT: lui a0, %hi(.LCPI51_0)
; CHECK-NEXT: addi a0, a0, %lo(.LCPI51_0)
; CHECK-NEXT: li a0, 115
; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-NEXT: vle16.v v12, (a0)
; CHECK-NEXT: vrgatherei16.vv v10, v8, v12
; CHECK-NEXT: vmv.s.x v12, a0
; CHECK-NEXT: vcompress.vm v10, v8, v12
; CHECK-NEXT: vmv.v.v v8, v10
; CHECK-NEXT: ret
%out = shufflevector <8 x i32> %v, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 6, i32 undef, i32 undef, i32 undef>
@@ -790,11 +786,10 @@ define <8 x i32> @shuffle_compress_singlesrc_e32(<8 x i32> %v) {
define <8 x i64> @shuffle_compress_singlesrc_e64(<8 x i64> %v) {
; CHECK-LABEL: shuffle_compress_singlesrc_e64:
; CHECK: # %bb.0:
; CHECK-NEXT: lui a0, %hi(.LCPI52_0)
; CHECK-NEXT: addi a0, a0, %lo(.LCPI52_0)
; CHECK-NEXT: li a0, 181
; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, ma
; CHECK-NEXT: vle16.v v16, (a0)
; CHECK-NEXT: vrgatherei16.vv v12, v8, v16
; CHECK-NEXT: vmv.s.x v16, a0
; CHECK-NEXT: vcompress.vm v12, v8, v16
; CHECK-NEXT: vmv.v.v v8, v12
; CHECK-NEXT: ret
%out = shufflevector <8 x i64> %v, <8 x i64> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 5, i32 7, i32 undef, i32 undef, i32 undef>

1306
llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll
View File

File diff suppressed because it is too large Load Diff

92
llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll
View File

@@ -12,13 +12,15 @@ define void @deinterleave3_0_i8(ptr %in, ptr %out) {
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vid.v v9
; CHECK-NEXT: vmv.v.i v9, -8
; CHECK-NEXT: vid.v v10
; CHECK-NEXT: li a0, 3
; CHECK-NEXT: vmul.vx v9, v9, a0
; CHECK-NEXT: vmadd.vx v10, a0, v9
; CHECK-NEXT: li a0, 73
; CHECK-NEXT: vmv.s.x v9, a0
; CHECK-NEXT: li a0, 56
; CHECK-NEXT: vmv.s.x v0, a0
; CHECK-NEXT: vadd.vi v10, v9, -8
; CHECK-NEXT: vrgather.vv v11, v8, v9
; CHECK-NEXT: vcompress.vm v11, v8, v9
; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v8, 8
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
@@ -37,22 +39,20 @@ define void @deinterleave3_8_i8(ptr %in, ptr %out) {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vmv.v.i v9, 1
; CHECK-NEXT: vid.v v10
; CHECK-NEXT: li a0, 3
; CHECK-NEXT: vmadd.vx v10, a0, v9
; CHECK-NEXT: li a0, 146
; CHECK-NEXT: vmv.s.x v9, a0
; CHECK-NEXT: li a0, 24
; CHECK-NEXT: vrgather.vv v9, v8, v10
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vcompress.vm v10, v8, v9
; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v8, 8
; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
; CHECK-NEXT: vsrl.vi v10, v8, 8
; CHECK-NEXT: vsrl.vi v9, v8, 8
; CHECK-NEXT: vsll.vi v8, v8, 8
; CHECK-NEXT: vmv.s.x v0, a0
; CHECK-NEXT: vor.vv v8, v8, v10
; CHECK-NEXT: vor.vv v8, v8, v9
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vmerge.vvm v8, v9, v8, v0
; CHECK-NEXT: vmerge.vvm v8, v10, v8, v0
; CHECK-NEXT: vse8.v v8, (a1)
; CHECK-NEXT: ret
entry:
@@ -100,15 +100,15 @@ define void @deinterleave4_8_i8(ptr %in, ptr %out) {
; CHECK-NEXT: vmv.v.i v9, -9
; CHECK-NEXT: vid.v v10
; CHECK-NEXT: li a0, 5
; CHECK-NEXT: vmadd.vx v10, a0, v9
; CHECK-NEXT: li a0, 34
; CHECK-NEXT: vmv.v.i v0, 12
; CHECK-NEXT: vmacc.vx v9, a0, v10
; CHECK-NEXT: vsll.vi v10, v10, 2
; CHECK-NEXT: vadd.vi v10, v10, 1
; CHECK-NEXT: vrgather.vv v11, v8, v10
; CHECK-NEXT: vmv.s.x v9, a0
; CHECK-NEXT: vcompress.vm v11, v8, v9
; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v8, 8
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
; CHECK-NEXT: vrgather.vv v11, v8, v9, v0.t
; CHECK-NEXT: vrgather.vv v11, v8, v10, v0.t
; CHECK-NEXT: vse8.v v11, (a1)
; CHECK-NEXT: ret
entry:
@@ -124,12 +124,14 @@ define void @deinterleave5_0_i8(ptr %in, ptr %out) {
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vid.v v9
; CHECK-NEXT: vmv.v.i v9, -8
; CHECK-NEXT: vid.v v10
; CHECK-NEXT: li a0, 5
; CHECK-NEXT: vmadd.vx v10, a0, v9
; CHECK-NEXT: li a0, 33
; CHECK-NEXT: vmv.v.i v0, 12
; CHECK-NEXT: vmul.vx v9, v9, a0
; CHECK-NEXT: vadd.vi v10, v9, -8
; CHECK-NEXT: vrgather.vv v11, v8, v9
; CHECK-NEXT: vmv.s.x v9, a0
; CHECK-NEXT: vcompress.vm v11, v8, v9
; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v8, 8
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
@@ -148,18 +150,16 @@ define void @deinterleave5_8_i8(ptr %in, ptr %out) {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vmv.v.i v9, 1
; CHECK-NEXT: vid.v v10
; CHECK-NEXT: li a0, 5
; CHECK-NEXT: vmadd.vx v10, a0, v9
; CHECK-NEXT: li a0, 66
; CHECK-NEXT: vmv.v.i v0, 4
; CHECK-NEXT: vrgather.vv v9, v8, v10
; CHECK-NEXT: vmv.s.x v9, a0
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vcompress.vm v10, v8, v9
; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v8, 8
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
; CHECK-NEXT: vrgather.vi v9, v8, 3, v0.t
; CHECK-NEXT: vse8.v v9, (a1)
; CHECK-NEXT: vrgather.vi v10, v8, 3, v0.t
; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
@@ -173,12 +173,11 @@ define void @deinterleave6_0_i8(ptr %in, ptr %out) {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vid.v v9
; CHECK-NEXT: li a0, 6
; CHECK-NEXT: li a0, 65
; CHECK-NEXT: vmv.v.i v0, 4
; CHECK-NEXT: vmul.vx v9, v9, a0
; CHECK-NEXT: vrgather.vv v10, v8, v9
; CHECK-NEXT: vmv.s.x v9, a0
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vcompress.vm v10, v8, v9
; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v8, 8
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
@@ -197,18 +196,16 @@ define void @deinterleave6_8_i8(ptr %in, ptr %out) {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vmv.v.i v9, 1
; CHECK-NEXT: vid.v v10
; CHECK-NEXT: li a0, 6
; CHECK-NEXT: vmadd.vx v10, a0, v9
; CHECK-NEXT: li a0, 130
; CHECK-NEXT: vmv.v.i v0, 4
; CHECK-NEXT: vrgather.vv v9, v8, v10
; CHECK-NEXT: vmv.s.x v9, a0
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vcompress.vm v10, v8, v9
; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v8, 8
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu
; CHECK-NEXT: vrgather.vi v9, v8, 5, v0.t
; CHECK-NEXT: vse8.v v9, (a1)
; CHECK-NEXT: vrgather.vi v10, v8, 5, v0.t
; CHECK-NEXT: vse8.v v10, (a1)
; CHECK-NEXT: ret
entry:
%0 = load <16 x i8>, ptr %in, align 1
@@ -222,12 +219,11 @@ define void @deinterleave7_0_i8(ptr %in, ptr %out) {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vid.v v9
; CHECK-NEXT: li a0, 7
; CHECK-NEXT: li a0, 129
; CHECK-NEXT: vmv.v.i v0, 4
; CHECK-NEXT: vmul.vx v9, v9, a0
; CHECK-NEXT: vrgather.vv v10, v8, v9
; CHECK-NEXT: vmv.s.x v9, a0
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; CHECK-NEXT: vcompress.vm v10, v8, v9
; CHECK-NEXT: vsetivli zero, 8, e8, m1, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v8, 8
; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, mu

40
llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
View File

@@ -473,17 +473,35 @@ entry:
; Can't match the m8 result type as the source would have to be m16 which
; isn't a legal type.
define void @vnsrl_0_i32_single_src_m8(ptr %in, ptr %out) {
; CHECK-LABEL: vnsrl_0_i32_single_src_m8:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: li a2, 64
; CHECK-NEXT: vsetvli zero, a2, e16, m4, ta, ma
; CHECK-NEXT: vle32.v v8, (a0)
; CHECK-NEXT: vid.v v16
; CHECK-NEXT: vadd.vv v16, v16, v16
; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma
; CHECK-NEXT: vrgatherei16.vv v24, v8, v16
; CHECK-NEXT: vse32.v v24, (a1)
; CHECK-NEXT: ret
; V-LABEL: vnsrl_0_i32_single_src_m8:
; V: # %bb.0: # %entry
; V-NEXT: li a2, 64
; V-NEXT: vsetvli zero, a2, e32, m8, ta, ma
; V-NEXT: vle32.v v8, (a0)
; V-NEXT: lui a0, 341
; V-NEXT: addiw a0, a0, 1365
; V-NEXT: vsetivli zero, 1, e64, m1, ta, ma
; V-NEXT: vmv.s.x v16, a0
; V-NEXT: vsetvli zero, a2, e32, m8, ta, ma
; V-NEXT: vcompress.vm v24, v8, v16
; V-NEXT: vse32.v v24, (a1)
; V-NEXT: ret
;
; ZVE32F-LABEL: vnsrl_0_i32_single_src_m8:
; ZVE32F: # %bb.0: # %entry
; ZVE32F-NEXT: li a2, 64
; ZVE32F-NEXT: vsetvli zero, a2, e32, m8, ta, ma
; ZVE32F-NEXT: vle32.v v8, (a0)
; ZVE32F-NEXT: vsetivli zero, 2, e32, m1, ta, ma
; ZVE32F-NEXT: vmv.v.i v16, 0
; ZVE32F-NEXT: lui a0, 341
; ZVE32F-NEXT: addi a0, a0, 1365
; ZVE32F-NEXT: vsetvli zero, zero, e32, m1, tu, ma
; ZVE32F-NEXT: vmv.s.x v16, a0
; ZVE32F-NEXT: vsetvli zero, a2, e32, m8, ta, ma
; ZVE32F-NEXT: vcompress.vm v24, v8, v16
; ZVE32F-NEXT: vse32.v v24, (a1)
; ZVE32F-NEXT: ret
entry:
%0 = load <64 x i32>, ptr %in, align 4
%shuffle.i5 = shufflevector <64 x i32> %0, <64 x i32> poison, <64 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>

71
llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
View File

@@ -95,26 +95,25 @@ define {<4 x i64>, <4 x i64>} @vector_deinterleave_v4i64_v8i64(<8 x i64> %vec) {
; CHECK-LABEL: vector_deinterleave_v4i64_v8i64:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
; CHECK-NEXT: vid.v v12
; CHECK-NEXT: vmv.v.i v14, 5
; CHECK-NEXT: vid.v v15
; CHECK-NEXT: vsetivli zero, 4, e64, m4, ta, ma
; CHECK-NEXT: vslidedown.vi v16, v8, 4
; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
; CHECK-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
; CHECK-NEXT: vmv.v.i v0, 12
; CHECK-NEXT: vadd.vv v14, v12, v12
; CHECK-NEXT: vmv.v.i v18, 10
; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma
; CHECK-NEXT: vcompress.vm v12, v8, v14
; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma
; CHECK-NEXT: vadd.vv v14, v15, v15
; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma
; CHECK-NEXT: vrgatherei16.vv v12, v8, v14
; CHECK-NEXT: vcompress.vm v10, v8, v18
; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma
; CHECK-NEXT: vadd.vi v10, v14, -4
; CHECK-NEXT: vadd.vi v8, v14, -4
; CHECK-NEXT: vadd.vi v9, v14, -3
; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, mu
; CHECK-NEXT: vrgatherei16.vv v12, v16, v10, v0.t
; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma
; CHECK-NEXT: vadd.vi v15, v14, 1
; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma
; CHECK-NEXT: vrgatherei16.vv v10, v8, v15
; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma
; CHECK-NEXT: vadd.vi v8, v14, -3
; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, mu
; CHECK-NEXT: vrgatherei16.vv v10, v16, v8, v0.t
; CHECK-NEXT: vrgatherei16.vv v12, v16, v8, v0.t
; CHECK-NEXT: vrgatherei16.vv v10, v16, v9, v0.t
; CHECK-NEXT: vmv.v.v v8, v12
; CHECK-NEXT: ret
%retval = call {<4 x i64>, <4 x i64>} @llvm.vector.deinterleave2.v8i64(<8 x i64> %vec)
@@ -124,26 +123,27 @@ define {<4 x i64>, <4 x i64>} @vector_deinterleave_v4i64_v8i64(<8 x i64> %vec) {
define {<8 x i64>, <8 x i64>} @vector_deinterleave_v8i64_v16i64(<16 x i64> %vec) {
; CHECK-LABEL: vector_deinterleave_v8i64_v16i64:
; CHECK: # %bb.0:
; CHECK-NEXT: li a0, 85
; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT: vid.v v16
; CHECK-NEXT: vmv.v.i v0, -16
; CHECK-NEXT: vid.v v16
; CHECK-NEXT: vsetivli zero, 8, e64, m8, ta, ma
; CHECK-NEXT: vslidedown.vi v24, v8, 8
; CHECK-NEXT: vmv.s.x v12, a0
; CHECK-NEXT: li a0, 170
; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT: vadd.vv v20, v16, v16
; CHECK-NEXT: vmv.s.x v21, a0
; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma
; CHECK-NEXT: vrgatherei16.vv v16, v8, v20
; CHECK-NEXT: vcompress.vm v16, v8, v12
; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma
; CHECK-NEXT: vadd.vi v12, v20, -8
; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, mu
; CHECK-NEXT: vrgatherei16.vv v16, v24, v12, v0.t
; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma
; CHECK-NEXT: vadd.vi v21, v20, 1
; CHECK-NEXT: vadd.vi v22, v20, -8
; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma
; CHECK-NEXT: vrgatherei16.vv v12, v8, v21
; CHECK-NEXT: vcompress.vm v12, v8, v21
; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma
; CHECK-NEXT: vadd.vi v8, v20, -7
; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, mu
; CHECK-NEXT: vrgatherei16.vv v16, v24, v22, v0.t
; CHECK-NEXT: vrgatherei16.vv v12, v24, v8, v0.t
; CHECK-NEXT: vmv.v.v v8, v16
; CHECK-NEXT: ret
@@ -241,26 +241,25 @@ define {<4 x double>, <4 x double>} @vector_deinterleave_v4f64_v8f64(<8 x double
; CHECK-LABEL: vector_deinterleave_v4f64_v8f64:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
; CHECK-NEXT: vid.v v12
; CHECK-NEXT: vmv.v.i v14, 5
; CHECK-NEXT: vid.v v15
; CHECK-NEXT: vsetivli zero, 4, e64, m4, ta, ma
; CHECK-NEXT: vslidedown.vi v16, v8, 4
; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
; CHECK-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
; CHECK-NEXT: vmv.v.i v0, 12
; CHECK-NEXT: vadd.vv v14, v12, v12
; CHECK-NEXT: vmv.v.i v18, 10
; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma
; CHECK-NEXT: vcompress.vm v12, v8, v14
; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma
; CHECK-NEXT: vadd.vv v14, v15, v15
; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma
; CHECK-NEXT: vrgatherei16.vv v12, v8, v14
; CHECK-NEXT: vcompress.vm v10, v8, v18
; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma
; CHECK-NEXT: vadd.vi v10, v14, -4
; CHECK-NEXT: vadd.vi v8, v14, -4
; CHECK-NEXT: vadd.vi v9, v14, -3
; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, mu
; CHECK-NEXT: vrgatherei16.vv v12, v16, v10, v0.t
; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma
; CHECK-NEXT: vadd.vi v15, v14, 1
; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma
; CHECK-NEXT: vrgatherei16.vv v10, v8, v15
; CHECK-NEXT: vsetvli zero, zero, e16, mf2, ta, ma
; CHECK-NEXT: vadd.vi v8, v14, -3
; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, mu
; CHECK-NEXT: vrgatherei16.vv v10, v16, v8, v0.t
; CHECK-NEXT: vrgatherei16.vv v12, v16, v8, v0.t
; CHECK-NEXT: vrgatherei16.vv v10, v16, v9, v0.t
; CHECK-NEXT: vmv.v.v v8, v12
; CHECK-NEXT: ret
%retval = call {<4 x double>, <4 x double>} @llvm.vector.deinterleave2.v8f64(<8 x double> %vec)