diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 5a92d6bab31a..0fcaac060569 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1390,14 +1390,32 @@ InstructionCost RISCVTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, InstructionCost Cost = 0; if (Opcode == Instruction::Store && OpInfo.isConstant()) Cost += getStoreImmCost(Src, OpInfo, CostKind); - InstructionCost BaseCost = - BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, - CostKind, OpInfo, I); + + std::pair LT = getTypeLegalizationCost(Src); + + InstructionCost BaseCost = [&]() { + InstructionCost Cost = LT.first; + if (CostKind != TTI::TCK_RecipThroughput) + return Cost; + + // Our actual lowering for the case where a wider legal type is available + // uses the a VL predicated load on the wider type. This is reflected in + // the result of getTypeLegalizationCost, but BasicTTI assumes the + // widened cases are scalarized. + const DataLayout &DL = this->getDataLayout(); + if (Src->isVectorTy() && LT.second.isVector() && + TypeSize::isKnownLT(DL.getTypeStoreSizeInBits(Src), + LT.second.getSizeInBits())) + return Cost; + + return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, + CostKind, OpInfo, I); + }(); + // Assume memory ops cost scale with the number of vector registers // possible accessed by the instruction. Note that BasicTTI already // handles the LT.first term for us. - if (std::pair LT = getTypeLegalizationCost(Src); - LT.second.isVector() && CostKind != TTI::TCK_CodeSize) + if (LT.second.isVector() && CostKind != TTI::TCK_CodeSize) BaseCost *= TLI->getLMULCost(LT.second); return Cost + BaseCost; diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-load-store.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-load-store.ll index c9d7bd971e0f..2448056076d8 100644 --- a/llvm/test/Analysis/CostModel/RISCV/rvv-load-store.ll +++ b/llvm/test/Analysis/CostModel/RISCV/rvv-load-store.ll @@ -592,16 +592,16 @@ define void @load_oddsize_vectors(ptr %p) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load <32 x i1>, ptr %p, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <1 x i32>, ptr %p, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <2 x i32>, ptr %p, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %16 = load <3 x i32>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = load <3 x i32>, ptr %p, align 16 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %17 = load <4 x i32>, ptr %p, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %18 = load <5 x i32>, ptr %p, align 32 -; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %19 = load <6 x i32>, ptr %p, align 32 -; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %20 = load <7 x i32>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %18 = load <5 x i32>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %19 = load <6 x i32>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %20 = load <7 x i32>, ptr %p, align 32 ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %21 = load <8 x i32>, ptr %p, align 32 -; CHECK-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %22 = load <9 x i32>, ptr %p, align 64 -; CHECK-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %23 = load <15 x i32>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %22 = load <9 x i32>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %23 = load <15 x i32>, ptr %p, align 64 ; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %24 = load <16 x i32>, ptr %p, align 64 -; CHECK-NEXT: Cost Model: Found an estimated cost of 256 for instruction: %25 = load <31 x i32>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %25 = load <31 x i32>, ptr %p, align 128 ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %26 = load <32 x i32>, ptr %p, align 128 ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; @@ -682,15 +682,15 @@ define void @store_oddsize_vectors(ptr %p) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <32 x i1> undef, ptr %p, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <1 x i32> undef, ptr %p, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> undef, ptr %p, align 8 -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <3 x i32> undef, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> undef, ptr %p, align 16 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> undef, ptr %p, align 16 -; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <5 x i32> undef, ptr %p, align 32 -; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: store <6 x i32> undef, ptr %p, align 32 -; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <7 x i32> undef, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <5 x i32> undef, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <6 x i32> undef, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <7 x i32> undef, ptr %p, align 32 ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <8 x i32> undef, ptr %p, align 32 -; CHECK-NEXT: Cost Model: Found an estimated cost of 64 for instruction: store <15 x i32> undef, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <15 x i32> undef, ptr %p, align 64 ; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <16 x i32> undef, ptr %p, align 64 -; CHECK-NEXT: Cost Model: Found an estimated cost of 256 for instruction: store <31 x i32> undef, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <31 x i32> undef, ptr %p, align 128 ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <32 x i32> undef, ptr %p, align 128 ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll b/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll index b699b246a328..bb716d78ca41 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll @@ -7,24 +7,22 @@ define void @small_trip_count_min_vlen_128(ptr nocapture %a) nounwind vscale_ran ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP1:%.*]] = mul i32 [[TMP0]], 2 -; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[TMP1]], 1 -; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 4, [[TMP2]] -; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]] +; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], 1 +; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 4, [[TMP1]] +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP0]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] -; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP4:%.*]] = mul i32 [[TMP3]], 2 +; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32() ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i32(i32 [[TMP5]], i32 4) -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i32.p0(ptr [[TMP7]], i32 4, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP8:%.*]] = add nsw [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: call void @llvm.masked.store.nxv2i32.p0( [[TMP8]], ptr [[TMP7]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP4]] +; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv1i1.i32(i32 [[TMP3]], i32 4) +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv1i32.p0(ptr [[TMP5]], i32 4, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP6:%.*]] = add nsw [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: call void @llvm.masked.store.nxv1i32.p0( [[TMP6]], ptr [[TMP5]], i32 4, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP2]] ; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]