From ec910751fe5c4a0e0fe4e232da0f8bed307bf4e8 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Sun, 20 Feb 2022 18:50:28 +0000 Subject: [PATCH] [X86] combineX86ShufflesRecursively - attempt to fold ISD::EXTRACT_SUBVECTOR into a shuffle chain Peek through if we're extracting a non-zero'th subvector in an attempt to fold the extract into a lane-crossing shuffle This also exposes a failure to fold extract_subvector(movddup(x),c) -> movddup(extract_subvector(x,c)) --- llvm/lib/Target/X86/X86ISelLowering.cpp | 47 +++++-- llvm/test/CodeGen/X86/avx512-hadd-hsub.ll | 5 +- .../copy-low-subvec-elt-to-high-subvec-elt.ll | 36 ++++-- .../CodeGen/X86/vector-shuffle-256-v16.ll | 32 ++++- .../CodeGen/X86/vector-shuffle-256-v32.ll | 34 ++++- .../test/CodeGen/X86/vector-shuffle-256-v8.ll | 120 +++++++++++++++--- .../CodeGen/X86/vector-shuffle-512-v32.ll | 4 +- 7 files changed, 220 insertions(+), 58 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 3e9b51dbcaa0..c372919f44f7 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -38424,16 +38424,27 @@ static SDValue combineX86ShufflesRecursively( APInt OpUndef, OpZero; APInt OpDemandedElts = APInt::getAllOnes(VT.getVectorNumElements()); bool IsOpVariableMask = isTargetShuffleVariableMask(Op.getOpcode()); - if (!getTargetShuffleInputs(Op, OpDemandedElts, OpInputs, OpMask, OpUndef, - OpZero, DAG, Depth, false)) - return SDValue(); - - // Shuffle inputs must not be larger than the shuffle result. - // TODO: Relax this for single input faux shuffles (trunc/extract_subvector). - if (llvm::any_of(OpInputs, [VT](SDValue OpInput) { - return OpInput.getValueSizeInBits() > VT.getSizeInBits(); - })) + if (getTargetShuffleInputs(Op, OpDemandedElts, OpInputs, OpMask, OpUndef, + OpZero, DAG, Depth, false)) { + // Shuffle inputs must not be larger than the shuffle result. + // TODO: Relax this for single input faux shuffles (e.g. trunc). + if (llvm::any_of(OpInputs, [VT](SDValue OpInput) { + return OpInput.getValueSizeInBits() > VT.getSizeInBits(); + })) + return SDValue(); + } else if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR && + (RootSizeInBits % Op.getOperand(0).getValueSizeInBits()) == 0 && + !isNullConstant(Op.getOperand(1))) { + SDValue SrcVec = Op.getOperand(0); + int ExtractIdx = Op.getConstantOperandVal(1); + unsigned NumElts = VT.getVectorNumElements(); + OpInputs.assign({SrcVec}); + OpMask.assign(NumElts, SM_SentinelUndef); + std::iota(OpMask.begin(), OpMask.end(), ExtractIdx); + OpZero = OpUndef = APInt::getNullValue(NumElts); + } else { return SDValue(); + } // If the shuffle result was smaller than the root, we need to adjust the // mask indices and pad the mask with undefs. @@ -53436,8 +53447,8 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG, // If we're extracting the lowest subvector and we're the only user, // we may be able to perform this with a smaller vector width. unsigned InOpcode = InVec.getOpcode(); - if (IdxVal == 0 && InVec.hasOneUse()) { - if (VT == MVT::v2f64 && InVecVT == MVT::v4f64) { + if (InVec.hasOneUse()) { + if (IdxVal == 0 && VT == MVT::v2f64 && InVecVT == MVT::v4f64) { // v2f64 CVTDQ2PD(v4i32). if (InOpcode == ISD::SINT_TO_FP && InVec.getOperand(0).getValueType() == MVT::v4i32) { @@ -53454,7 +53465,8 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG, return DAG.getNode(X86ISD::VFPEXT, SDLoc(N), VT, InVec.getOperand(0)); } } - if ((InOpcode == ISD::ANY_EXTEND || + if (IdxVal == 0 && + (InOpcode == ISD::ANY_EXTEND || InOpcode == ISD::ANY_EXTEND_VECTOR_INREG || InOpcode == ISD::ZERO_EXTEND || InOpcode == ISD::ZERO_EXTEND_VECTOR_INREG || @@ -53469,7 +53481,7 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG, unsigned ExtOp = getOpcode_EXTEND_VECTOR_INREG(InOpcode); return DAG.getNode(ExtOp, DL, VT, Ext); } - if (InOpcode == ISD::VSELECT && + if (IdxVal == 0 && InOpcode == ISD::VSELECT && InVec.getOperand(0).getValueType().is256BitVector() && InVec.getOperand(1).getValueType().is256BitVector() && InVec.getOperand(2).getValueType().is256BitVector()) { @@ -53479,7 +53491,7 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG, SDValue Ext2 = extractSubVector(InVec.getOperand(2), 0, DAG, DL, 128); return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, Ext2); } - if (InOpcode == ISD::TRUNCATE && Subtarget.hasVLX() && + if (IdxVal == 0 && InOpcode == ISD::TRUNCATE && Subtarget.hasVLX() && (VT.is128BitVector() || VT.is256BitVector())) { SDLoc DL(N); SDValue InVecSrc = InVec.getOperand(0); @@ -53487,6 +53499,13 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG, SDValue Ext = extractSubVector(InVecSrc, 0, DAG, DL, Scale * SizeInBits); return DAG.getNode(InOpcode, DL, VT, Ext); } + if (InOpcode == X86ISD::MOVDDUP && + (VT.is128BitVector() || VT.is256BitVector())) { + SDLoc DL(N); + SDValue Ext0 = + extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits); + return DAG.getNode(InOpcode, DL, VT, Ext0); + } } // Always split vXi64 logical shifts where we're extracting the upper 32-bits diff --git a/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll b/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll index 9e6c8b8becbc..85266a7a682c 100644 --- a/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll +++ b/llvm/test/CodeGen/X86/avx512-hadd-hsub.ll @@ -223,9 +223,8 @@ define double @fsub_noundef_ee (<8 x double> %x225, <8 x double> %x227) { ; SKX-LABEL: fsub_noundef_ee: ; SKX: # %bb.0: ; SKX-NEXT: vextractf32x4 $2, %zmm1, %xmm0 -; SKX-NEXT: vmovddup {{.*#+}} xmm1 = xmm0[0,0] -; SKX-NEXT: vsubpd %xmm0, %xmm1, %xmm0 -; SKX-NEXT: vpermilpd {{.*#+}} xmm0 = xmm0[1,0] +; SKX-NEXT: vpermilpd {{.*#+}} xmm1 = xmm0[1,0] +; SKX-NEXT: vsubsd %xmm1, %xmm0, %xmm0 ; SKX-NEXT: vzeroupper ; SKX-NEXT: retq %x226 = shufflevector <8 x double> %x225, <8 x double> %x227, <8 x i32> diff --git a/llvm/test/CodeGen/X86/copy-low-subvec-elt-to-high-subvec-elt.ll b/llvm/test/CodeGen/X86/copy-low-subvec-elt-to-high-subvec-elt.ll index e62444202024..db7b8558a096 100644 --- a/llvm/test/CodeGen/X86/copy-low-subvec-elt-to-high-subvec-elt.ll +++ b/llvm/test/CodeGen/X86/copy-low-subvec-elt-to-high-subvec-elt.ll @@ -439,12 +439,18 @@ define <8 x float> @vec256_eltty_float_source_subvec_1_target_subvec_mask_2_unar } define <8 x float> @vec256_eltty_float_source_subvec_1_target_subvec_mask_2_binary(<8 x float> %x, <8 x float> %y) nounwind { -; CHECK-LABEL: vec256_eltty_float_source_subvec_1_target_subvec_mask_2_binary: -; CHECK: # %bb.0: -; CHECK-NEXT: vextractf128 $1, %ymm1, %xmm1 -; CHECK-NEXT: vbroadcastss %xmm1, %ymm1 -; CHECK-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7] -; CHECK-NEXT: retq +; CHECK-SLOW-LABEL: vec256_eltty_float_source_subvec_1_target_subvec_mask_2_binary: +; CHECK-SLOW: # %bb.0: +; CHECK-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1 +; CHECK-SLOW-NEXT: vbroadcastss %xmm1, %ymm1 +; CHECK-SLOW-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7] +; CHECK-SLOW-NEXT: retq +; +; CHECK-FAST-LABEL: vec256_eltty_float_source_subvec_1_target_subvec_mask_2_binary: +; CHECK-FAST: # %bb.0: +; CHECK-FAST-NEXT: vpermilps {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4] +; CHECK-FAST-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7] +; CHECK-FAST-NEXT: retq %r = shufflevector <8 x float> %x, <8 x float> %y, <8 x i32> ret <8 x float> %r } @@ -562,12 +568,18 @@ define <8 x i32> @vec256_eltty_i32_source_subvec_1_target_subvec_mask_2_unary(<8 } define <8 x i32> @vec256_eltty_i32_source_subvec_1_target_subvec_mask_2_binary(<8 x i32> %x, <8 x i32> %y) nounwind { -; CHECK-LABEL: vec256_eltty_i32_source_subvec_1_target_subvec_mask_2_binary: -; CHECK: # %bb.0: -; CHECK-NEXT: vextractf128 $1, %ymm1, %xmm1 -; CHECK-NEXT: vbroadcastss %xmm1, %ymm1 -; CHECK-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7] -; CHECK-NEXT: retq +; CHECK-SLOW-LABEL: vec256_eltty_i32_source_subvec_1_target_subvec_mask_2_binary: +; CHECK-SLOW: # %bb.0: +; CHECK-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1 +; CHECK-SLOW-NEXT: vbroadcastss %xmm1, %ymm1 +; CHECK-SLOW-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7] +; CHECK-SLOW-NEXT: retq +; +; CHECK-FAST-LABEL: vec256_eltty_i32_source_subvec_1_target_subvec_mask_2_binary: +; CHECK-FAST: # %bb.0: +; CHECK-FAST-NEXT: vpermilps {{.*#+}} ymm1 = ymm1[0,0,0,0,4,4,4,4] +; CHECK-FAST-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7] +; CHECK-FAST-NEXT: retq %r = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> ret <8 x i32> %r } diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll index 2e7e59ab456e..63bc2331d4f7 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll @@ -4,8 +4,8 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=ALL,AVX1OR2,AVX2OR512VL,AVX2,AVX2-FAST,AVX2-FAST-ALL ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=ALL,AVX1OR2,AVX2OR512VL,AVX2,AVX2-FAST,AVX2-FAST-PERLANE ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=ALL,AVX2OR512VL,AVX512VL,AVX512VL-SLOW -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=ALL,AVX2OR512VL,AVX512VL,AVX512VL-FAST -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=ALL,AVX2OR512VL,AVX512VL,AVX512VL-FAST +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=ALL,AVX2OR512VL,AVX512VL,AVX512VL-FAST,AVX512VL-FAST-CROSSLANE +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512bw,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=ALL,AVX2OR512VL,AVX512VL,AVX512VL-FAST,AVX512VL-FAST-PERLANE ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=ALL,XOP,XOPAVX1 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=ALL,XOP,XOPAVX2 @@ -6826,11 +6826,29 @@ define <16 x i16> @shuffle_v16i16_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8(<16 x i16> %a, ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2OR512VL-LABEL: shuffle_v16i16_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8: -; AVX2OR512VL: # %bb.0: -; AVX2OR512VL-NEXT: vextracti128 $1, %ymm0, %xmm0 -; AVX2OR512VL-NEXT: vpbroadcastw %xmm0, %ymm0 -; AVX2OR512VL-NEXT: retq +; AVX2-LABEL: shuffle_v16i16_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8: +; AVX2: # %bb.0: +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; AVX2-NEXT: vpbroadcastw %xmm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512VL-SLOW-LABEL: shuffle_v16i16_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8: +; AVX512VL-SLOW: # %bb.0: +; AVX512VL-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm0 +; AVX512VL-SLOW-NEXT: vpbroadcastw %xmm0, %ymm0 +; AVX512VL-SLOW-NEXT: retq +; +; AVX512VL-FAST-CROSSLANE-LABEL: shuffle_v16i16_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8: +; AVX512VL-FAST-CROSSLANE: # %bb.0: +; AVX512VL-FAST-CROSSLANE-NEXT: vmovdqa {{.*#+}} ymm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; AVX512VL-FAST-CROSSLANE-NEXT: vpermw %ymm0, %ymm1, %ymm0 +; AVX512VL-FAST-CROSSLANE-NEXT: retq +; +; AVX512VL-FAST-PERLANE-LABEL: shuffle_v16i16_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8: +; AVX512VL-FAST-PERLANE: # %bb.0: +; AVX512VL-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm0 +; AVX512VL-FAST-PERLANE-NEXT: vpbroadcastw %xmm0, %ymm0 +; AVX512VL-FAST-PERLANE-NEXT: retq ; ; XOPAVX1-LABEL: shuffle_v16i16_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8_8: ; XOPAVX1: # %bb.0: diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll index d6731f851d0b..73d94b208b6e 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll @@ -4555,11 +4555,35 @@ define <32 x i8> @shuffle_v32i8_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_ ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 ; AVX1-NEXT: retq ; -; AVX2OR512VL-LABEL: shuffle_v32i8_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16: -; AVX2OR512VL: # %bb.0: -; AVX2OR512VL-NEXT: vextracti128 $1, %ymm0, %xmm0 -; AVX2OR512VL-NEXT: vpbroadcastb %xmm0, %ymm0 -; AVX2OR512VL-NEXT: retq +; AVX2-LABEL: shuffle_v32i8_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16: +; AVX2: # %bb.0: +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; AVX2-NEXT: vpbroadcastb %xmm0, %ymm0 +; AVX2-NEXT: retq +; +; AVX512VLBW-LABEL: shuffle_v32i8_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16: +; AVX512VLBW: # %bb.0: +; AVX512VLBW-NEXT: vextracti128 $1, %ymm0, %xmm0 +; AVX512VLBW-NEXT: vpbroadcastb %xmm0, %ymm0 +; AVX512VLBW-NEXT: retq +; +; AVX512VLVBMI-SLOW-LABEL: shuffle_v32i8_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16: +; AVX512VLVBMI-SLOW: # %bb.0: +; AVX512VLVBMI-SLOW-NEXT: vextracti128 $1, %ymm0, %xmm0 +; AVX512VLVBMI-SLOW-NEXT: vpbroadcastb %xmm0, %ymm0 +; AVX512VLVBMI-SLOW-NEXT: retq +; +; AVX512VLVBMI-FAST-ALL-LABEL: shuffle_v32i8_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16: +; AVX512VLVBMI-FAST-ALL: # %bb.0: +; AVX512VLVBMI-FAST-ALL-NEXT: vmovdqa {{.*#+}} ymm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] +; AVX512VLVBMI-FAST-ALL-NEXT: vpermb %ymm0, %ymm1, %ymm0 +; AVX512VLVBMI-FAST-ALL-NEXT: retq +; +; AVX512VLVBMI-FAST-PERLANE-LABEL: shuffle_v32i8_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16: +; AVX512VLVBMI-FAST-PERLANE: # %bb.0: +; AVX512VLVBMI-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm0 +; AVX512VLVBMI-FAST-PERLANE-NEXT: vpbroadcastb %xmm0, %ymm0 +; AVX512VLVBMI-FAST-PERLANE-NEXT: retq ; ; XOPAVX1-LABEL: shuffle_v32i8_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16_16: ; XOPAVX1: # %bb.0: diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll index 96d25cc66cb0..2bc2f9e91c8d 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll @@ -1466,11 +1466,41 @@ define <8 x float> @shuffle_v8f32_44444444(<8 x float> %a, <8 x float> %b) { ; AVX1-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4] ; AVX1-NEXT: retq ; -; AVX2OR512VL-LABEL: shuffle_v8f32_44444444: -; AVX2OR512VL: # %bb.0: -; AVX2OR512VL-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVX2OR512VL-NEXT: vbroadcastss %xmm0, %ymm0 -; AVX2OR512VL-NEXT: retq +; AVX2-SLOW-LABEL: shuffle_v8f32_44444444: +; AVX2-SLOW: # %bb.0: +; AVX2-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX2-SLOW-NEXT: vbroadcastss %xmm0, %ymm0 +; AVX2-SLOW-NEXT: retq +; +; AVX2-FAST-ALL-LABEL: shuffle_v8f32_44444444: +; AVX2-FAST-ALL: # %bb.0: +; AVX2-FAST-ALL-NEXT: vbroadcastss {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4] +; AVX2-FAST-ALL-NEXT: vpermps %ymm0, %ymm1, %ymm0 +; AVX2-FAST-ALL-NEXT: retq +; +; AVX2-FAST-PERLANE-LABEL: shuffle_v8f32_44444444: +; AVX2-FAST-PERLANE: # %bb.0: +; AVX2-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX2-FAST-PERLANE-NEXT: vbroadcastss %xmm0, %ymm0 +; AVX2-FAST-PERLANE-NEXT: retq +; +; AVX512VL-SLOW-LABEL: shuffle_v8f32_44444444: +; AVX512VL-SLOW: # %bb.0: +; AVX512VL-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX512VL-SLOW-NEXT: vbroadcastss %xmm0, %ymm0 +; AVX512VL-SLOW-NEXT: retq +; +; AVX512VL-FAST-ALL-LABEL: shuffle_v8f32_44444444: +; AVX512VL-FAST-ALL: # %bb.0: +; AVX512VL-FAST-ALL-NEXT: vbroadcastss {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4] +; AVX512VL-FAST-ALL-NEXT: vpermps %ymm0, %ymm1, %ymm0 +; AVX512VL-FAST-ALL-NEXT: retq +; +; AVX512VL-FAST-PERLANE-LABEL: shuffle_v8f32_44444444: +; AVX512VL-FAST-PERLANE: # %bb.0: +; AVX512VL-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX512VL-FAST-PERLANE-NEXT: vbroadcastss %xmm0, %ymm0 +; AVX512VL-FAST-PERLANE-NEXT: retq %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> ret <8 x float> %shuffle } @@ -3085,11 +3115,41 @@ define <8 x i32> @shuffle_v8i32_44444444(<8 x i32> %a, <8 x i32> %b) { ; AVX1-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4] ; AVX1-NEXT: retq ; -; AVX2OR512VL-LABEL: shuffle_v8i32_44444444: -; AVX2OR512VL: # %bb.0: -; AVX2OR512VL-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVX2OR512VL-NEXT: vbroadcastss %xmm0, %ymm0 -; AVX2OR512VL-NEXT: retq +; AVX2-SLOW-LABEL: shuffle_v8i32_44444444: +; AVX2-SLOW: # %bb.0: +; AVX2-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX2-SLOW-NEXT: vbroadcastss %xmm0, %ymm0 +; AVX2-SLOW-NEXT: retq +; +; AVX2-FAST-ALL-LABEL: shuffle_v8i32_44444444: +; AVX2-FAST-ALL: # %bb.0: +; AVX2-FAST-ALL-NEXT: vbroadcastss {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4] +; AVX2-FAST-ALL-NEXT: vpermps %ymm0, %ymm1, %ymm0 +; AVX2-FAST-ALL-NEXT: retq +; +; AVX2-FAST-PERLANE-LABEL: shuffle_v8i32_44444444: +; AVX2-FAST-PERLANE: # %bb.0: +; AVX2-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX2-FAST-PERLANE-NEXT: vbroadcastss %xmm0, %ymm0 +; AVX2-FAST-PERLANE-NEXT: retq +; +; AVX512VL-SLOW-LABEL: shuffle_v8i32_44444444: +; AVX512VL-SLOW: # %bb.0: +; AVX512VL-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX512VL-SLOW-NEXT: vbroadcastss %xmm0, %ymm0 +; AVX512VL-SLOW-NEXT: retq +; +; AVX512VL-FAST-ALL-LABEL: shuffle_v8i32_44444444: +; AVX512VL-FAST-ALL: # %bb.0: +; AVX512VL-FAST-ALL-NEXT: vbroadcastss {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4] +; AVX512VL-FAST-ALL-NEXT: vpermps %ymm0, %ymm1, %ymm0 +; AVX512VL-FAST-ALL-NEXT: retq +; +; AVX512VL-FAST-PERLANE-LABEL: shuffle_v8i32_44444444: +; AVX512VL-FAST-PERLANE: # %bb.0: +; AVX512VL-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX512VL-FAST-PERLANE-NEXT: vbroadcastss %xmm0, %ymm0 +; AVX512VL-FAST-PERLANE-NEXT: retq %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> ret <8 x i32> %shuffle } @@ -3101,11 +3161,41 @@ define <8 x i32> @shuffle_v8i32_44444444_bc(<8 x float> %a, <8 x float> %b) { ; AVX1-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4] ; AVX1-NEXT: retq ; -; AVX2OR512VL-LABEL: shuffle_v8i32_44444444_bc: -; AVX2OR512VL: # %bb.0: -; AVX2OR512VL-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVX2OR512VL-NEXT: vbroadcastss %xmm0, %ymm0 -; AVX2OR512VL-NEXT: retq +; AVX2-SLOW-LABEL: shuffle_v8i32_44444444_bc: +; AVX2-SLOW: # %bb.0: +; AVX2-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX2-SLOW-NEXT: vbroadcastss %xmm0, %ymm0 +; AVX2-SLOW-NEXT: retq +; +; AVX2-FAST-ALL-LABEL: shuffle_v8i32_44444444_bc: +; AVX2-FAST-ALL: # %bb.0: +; AVX2-FAST-ALL-NEXT: vbroadcastss {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4] +; AVX2-FAST-ALL-NEXT: vpermps %ymm0, %ymm1, %ymm0 +; AVX2-FAST-ALL-NEXT: retq +; +; AVX2-FAST-PERLANE-LABEL: shuffle_v8i32_44444444_bc: +; AVX2-FAST-PERLANE: # %bb.0: +; AVX2-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX2-FAST-PERLANE-NEXT: vbroadcastss %xmm0, %ymm0 +; AVX2-FAST-PERLANE-NEXT: retq +; +; AVX512VL-SLOW-LABEL: shuffle_v8i32_44444444_bc: +; AVX512VL-SLOW: # %bb.0: +; AVX512VL-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX512VL-SLOW-NEXT: vbroadcastss %xmm0, %ymm0 +; AVX512VL-SLOW-NEXT: retq +; +; AVX512VL-FAST-ALL-LABEL: shuffle_v8i32_44444444_bc: +; AVX512VL-FAST-ALL: # %bb.0: +; AVX512VL-FAST-ALL-NEXT: vbroadcastss {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4] +; AVX512VL-FAST-ALL-NEXT: vpermps %ymm0, %ymm1, %ymm0 +; AVX512VL-FAST-ALL-NEXT: retq +; +; AVX512VL-FAST-PERLANE-LABEL: shuffle_v8i32_44444444_bc: +; AVX512VL-FAST-PERLANE: # %bb.0: +; AVX512VL-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX512VL-FAST-PERLANE-NEXT: vbroadcastss %xmm0, %ymm0 +; AVX512VL-FAST-PERLANE-NEXT: retq %tmp0 = bitcast <8 x float> %a to <8 x i32> %tmp1 = bitcast <8 x float> %b to <8 x i32> %shuffle = shufflevector <8 x i32> %tmp0, <8 x i32> %tmp1, <8 x i32> diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll index 2b76d668f5fe..08b56a32f334 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll @@ -29,8 +29,8 @@ define <32 x i16> @shuffle_v32i16_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_0 ; ; SKX-LABEL: shuffle_v32i16_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08_08: ; SKX: ## %bb.0: -; SKX-NEXT: vextracti128 $1, %ymm0, %xmm0 -; SKX-NEXT: vpbroadcastw %xmm0, %zmm0 +; SKX-NEXT: vmovdqa64 {{.*#+}} zmm1 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] +; SKX-NEXT: vpermw %zmm0, %zmm1, %zmm0 ; SKX-NEXT: retq %c = shufflevector <32 x i16> %a, <32 x i16> undef, <32 x i32> ret <32 x i16> %c