diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h index 99f72792ce40..1c60eae7f2f8 100644 --- a/llvm/include/llvm/IR/InstrTypes.h +++ b/llvm/include/llvm/IR/InstrTypes.h @@ -912,6 +912,11 @@ public: /// Determine if this is an equals/not equals predicate. bool isEquality() const { return isEquality(getPredicate()); } + /// Determine if one operand of this compare can always be replaced by the + /// other operand, ignoring provenance considerations. If \p Invert, check for + /// equivalence with the inverse predicate. + bool isEquivalence(bool Invert = false) const; + /// Return true if the predicate is relational (not EQ or NE). static bool isRelational(Predicate P) { return !isEquality(P); } diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index c3349c9772c7..0d6df7279063 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -792,6 +792,16 @@ inline cstfp_pred_ty m_NonZeroFP() { return cstfp_pred_ty(); } +struct is_non_zero_not_denormal_fp { + bool isValue(const APFloat &C) { return !C.isDenormal() && C.isNonZero(); } +}; + +/// Match a floating-point non-zero that is not a denormal. +/// For vectors, this includes constants with undefined elements. +inline cstfp_pred_ty m_NonZeroNotDenormalFP() { + return cstfp_pred_ty(); +} + /////////////////////////////////////////////////////////////////////////////// template struct bind_ty { diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index 009e0c03957c..05e340ffa20a 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -32,6 +32,7 @@ #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" +#include "llvm/IR/PatternMatch.h" #include "llvm/IR/ProfDataUtils.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" @@ -3471,6 +3472,36 @@ bool CmpInst::isEquality(Predicate P) { llvm_unreachable("Unsupported predicate kind"); } +// Returns true if either operand of CmpInst is a provably non-zero +// floating-point constant. +static bool hasNonZeroFPOperands(const CmpInst *Cmp) { + auto *LHS = dyn_cast(Cmp->getOperand(0)); + auto *RHS = dyn_cast(Cmp->getOperand(1)); + if (auto *Const = LHS ? LHS : RHS) { + using namespace llvm::PatternMatch; + return match(Const, m_NonZeroNotDenormalFP()); + } + return false; +} + +// Floating-point equality is not an equivalence when comparing +0.0 with +// -0.0, when comparing NaN with another value, or when flushing +// denormals-to-zero. +bool CmpInst::isEquivalence(bool Invert) const { + switch (Invert ? getInversePredicate() : getPredicate()) { + case CmpInst::Predicate::ICMP_EQ: + return true; + case CmpInst::Predicate::FCMP_UEQ: + if (!hasNoNaNs()) + return false; + [[fallthrough]]; + case CmpInst::Predicate::FCMP_OEQ: + return hasNonZeroFPOperands(this); + default: + return false; + } +} + CmpInst::Predicate CmpInst::getInversePredicate(Predicate pred) { switch (pred) { default: llvm_unreachable("Unknown cmp predicate!"); diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp index ad9b1217089d..8bc2c710ec17 100644 --- a/llvm/lib/Transforms/Scalar/GVN.cpp +++ b/llvm/lib/Transforms/Scalar/GVN.cpp @@ -1989,59 +1989,6 @@ bool GVNPass::processNonLocalLoad(LoadInst *Load) { return Changed; } -static bool impliesEquivalanceIfTrue(CmpInst* Cmp) { - if (Cmp->getPredicate() == CmpInst::Predicate::ICMP_EQ) - return true; - - // Floating point comparisons can be equal, but not equivalent. Cases: - // NaNs for unordered operators - // +0.0 vs 0.0 for all operators - if (Cmp->getPredicate() == CmpInst::Predicate::FCMP_OEQ || - (Cmp->getPredicate() == CmpInst::Predicate::FCMP_UEQ && - Cmp->getFastMathFlags().noNaNs())) { - Value *LHS = Cmp->getOperand(0); - Value *RHS = Cmp->getOperand(1); - // If we can prove either side non-zero, then equality must imply - // equivalence. - // FIXME: We should do this optimization if 'no signed zeros' is - // applicable via an instruction-level fast-math-flag or some other - // indicator that relaxed FP semantics are being used. - if (isa(LHS) && !cast(LHS)->isZero()) - return true; - if (isa(RHS) && !cast(RHS)->isZero()) - return true; - // TODO: Handle vector floating point constants - } - return false; -} - -static bool impliesEquivalanceIfFalse(CmpInst* Cmp) { - if (Cmp->getPredicate() == CmpInst::Predicate::ICMP_NE) - return true; - - // Floating point comparisons can be equal, but not equivelent. Cases: - // NaNs for unordered operators - // +0.0 vs 0.0 for all operators - if ((Cmp->getPredicate() == CmpInst::Predicate::FCMP_ONE && - Cmp->getFastMathFlags().noNaNs()) || - Cmp->getPredicate() == CmpInst::Predicate::FCMP_UNE) { - Value *LHS = Cmp->getOperand(0); - Value *RHS = Cmp->getOperand(1); - // If we can prove either side non-zero, then equality must imply - // equivalence. - // FIXME: We should do this optimization if 'no signed zeros' is - // applicable via an instruction-level fast-math-flag or some other - // indicator that relaxed FP semantics are being used. - if (isa(LHS) && !cast(LHS)->isZero()) - return true; - if (isa(RHS) && !cast(RHS)->isZero()) - return true; - // TODO: Handle vector floating point constants - } - return false; -} - - static bool hasUsersIn(Value *V, BasicBlock *BB) { return llvm::any_of(V->users(), [BB](User *U) { auto *I = dyn_cast(U); @@ -2143,7 +2090,7 @@ bool GVNPass::processAssumeIntrinsic(AssumeInst *IntrinsicI) { // call void @llvm.assume(i1 %cmp) // ret float %load ; will change it to ret float %0 if (auto *CmpI = dyn_cast(V)) { - if (impliesEquivalanceIfTrue(CmpI)) { + if (CmpI->isEquivalence()) { Value *CmpLHS = CmpI->getOperand(0); Value *CmpRHS = CmpI->getOperand(1); // Heuristically pick the better replacement -- the choice of heuristic @@ -2577,8 +2524,7 @@ bool GVNPass::propagateEquality(Value *LHS, Value *RHS, // If "A == B" is known true, or "A != B" is known false, then replace // A with B everywhere in the scope. For floating point operations, we // have to be careful since equality does not always imply equivalance. - if ((isKnownTrue && impliesEquivalanceIfTrue(Cmp)) || - (isKnownFalse && impliesEquivalanceIfFalse(Cmp))) + if (Cmp->isEquivalence(isKnownFalse)) Worklist.push_back(std::make_pair(Op0, Op1)); // If "A >= B" is known true, replace "A < B" with false everywhere. diff --git a/llvm/test/Transforms/GVN/edge.ll b/llvm/test/Transforms/GVN/edge.ll index 9703195d3b64..83c4c336f647 100644 --- a/llvm/test/Transforms/GVN/edge.ll +++ b/llvm/test/Transforms/GVN/edge.ll @@ -224,6 +224,34 @@ return: ret double %retval } +; Denormals may be flushed to zero in some cases by the backend. +; Hence, treat denormals as 0. +define float @fcmp_oeq_denormal(float %x, float %y) { +; CHECK-LABEL: define float @fcmp_oeq_denormal( +; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[CMP:%.*]] = fcmp oeq float [[Y]], 0x3800000000000000 +; CHECK-NEXT: br i1 [[CMP]], label %[[IF:.*]], label %[[RETURN:.*]] +; CHECK: [[IF]]: +; CHECK-NEXT: [[DIV:%.*]] = fdiv float [[X]], [[Y]] +; CHECK-NEXT: br label %[[RETURN]] +; CHECK: [[RETURN]]: +; CHECK-NEXT: [[RETVAL:%.*]] = phi float [ [[DIV]], %[[IF]] ], [ [[X]], %[[ENTRY]] ] +; CHECK-NEXT: ret float [[RETVAL]] +; +entry: + %cmp = fcmp oeq float %y, 0x3800000000000000 + br i1 %cmp, label %if, label %return + +if: + %div = fdiv float %x, %y + br label %return + +return: + %retval = phi float [ %div, %if ], [ %x, %entry ] + ret float %retval +} + define double @fcmp_une_zero(double %x, double %y) { ; CHECK-LABEL: define double @fcmp_une_zero( ; CHECK-SAME: double [[X:%.*]], double [[Y:%.*]]) { @@ -251,7 +279,7 @@ return: } ; We also cannot propagate a value if it's not a constant. -; This is because the value could be 0.0 or -0.0. +; This is because the value could be 0.0, -0.0, or a denormal. define double @fcmp_oeq_maybe_zero(double %x, double %y, double %z1, double %z2) { ; CHECK-LABEL: define double @fcmp_oeq_maybe_zero(