Restore "[LLVM] move verification of convergence control to a class template""

The refactored template can now be used with MachineVerifier.

Resubmitted after fixing build errors:

- Shared libraries build failed with undefined references due to "extern
  template" declarations.
- Modules build failed due to a cycle dependence between llvm/ADT and llvm/IR.
  The Generic*Impl.h files should be in llvm/IR to prevent this.

Differential Revision: https://reviews.llvm.org/D156522

This restores commit 93a3706711.
Originally reverted in 466bd99811.

llvm/include/llvm/ADT/GenericConvergenceVerifier.h

@@ -0,0 +1,75 @@
+//===- GenericConvergenceVerifier.h ---------------------------*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// A verifier for the static rules of convergence control tokens that works
+/// with both LLVM IR and MIR.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_GENERICCONVERGENCEVERIFIER_H
+#define LLVM_ADT_GENERICCONVERGENCEVERIFIER_H
+
+#include "llvm/ADT/GenericCycleInfo.h"
+
+namespace llvm {
+
+template <typename ContextT> class GenericConvergenceVerifier {
+public:
+  using BlockT = typename ContextT::BlockT;
+  using FunctionT = typename ContextT::FunctionT;
+  using ValueRefT = typename ContextT::ValueRefT;
+  using InstructionT = typename ContextT::InstructionT;
+  using DominatorTreeT = typename ContextT::DominatorTreeT;
+  using CycleInfoT = GenericCycleInfo<ContextT>;
+  using CycleT = typename CycleInfoT::CycleT;
+
+  void initialize(raw_ostream *OS,
+                  function_ref<void(const Twine &Message)> FailureCB,
+                  const FunctionT &F) {
+    clear();
+    this->OS = OS;
+    this->FailureCB = FailureCB;
+    Context = ContextT(&F);
+  }
+
+  void clear();
+  void visit(const InstructionT &I);
+  void verify(const DominatorTreeT &DT);
+
+  bool sawTokens() const { return ConvergenceKind == ControlledConvergence; }
+
+private:
+  raw_ostream *OS;
+  std::function<void(const Twine &Message)> FailureCB;
+  DominatorTreeT *DT;
+  CycleInfoT CI;
+  ContextT Context;
+
+  /// Whether the current function has convergencectrl operand bundles.
+  enum {
+    ControlledConvergence,
+    UncontrolledConvergence,
+    NoConvergence
+  } ConvergenceKind = NoConvergence;
+
+  // Cache token uses found so far. Note that we track the unique definitions
+  // and not the token values.
+  DenseMap<const InstructionT *, const InstructionT *> Tokens;
+
+  const InstructionT *findAndCheckConvergenceTokenUsed(const InstructionT &I);
+  bool isControlledConvergent(const InstructionT &I);
+  bool isConvergent(const InstructionT &I) const;
+
+  void reportFailure(const Twine &Message, ArrayRef<Printable> Values);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_ADT_GENERICCONVERGENCEVERIFIER_H

llvm/include/llvm/ADT/GenericSSAContext.h

@@ -24,6 +24,10 @@ namespace llvm {
 template <typename, bool> class DominatorTreeBase;
 template <typename> class SmallVectorImpl;
 
+namespace Intrinsic {
+typedef unsigned ID;
+}
+
 // Specializations of this template should provide the types used by the
 // template GenericSSAContext below.
 template <typename _FunctionT> struct GenericSSATraits;
@@ -78,6 +82,8 @@ public:
 
   const FunctionT *getFunction() const { return F; }
 
+  static Intrinsic::ID getIntrinsicID(const InstructionT &I);
+
   static void appendBlockDefs(SmallVectorImpl<ValueRefT> &defs, BlockT &block);
   static void appendBlockDefs(SmallVectorImpl<ConstValueRefT> &defs,
                               const BlockT &block);
@@ -91,6 +97,7 @@ public:
   const BlockT *getDefBlock(ConstValueRefT value) const;
 
   Printable print(const BlockT *block) const;
+  Printable printAsOperand(const BlockT *BB) const;
   Printable print(const InstructionT *inst) const;
   Printable print(ConstValueRefT value) const;
 };

llvm/include/llvm/IR/ConvergenceVerifier.h

@@ -0,0 +1,27 @@
+//===- ConvergenceVerifier.h - Verify convergenctrl -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file declares the LLVM IR specialization of the
+/// GenericConvergenceVerifier template.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_IR_CONVERGENCEVERIFIER_H
+#define LLVM_IR_CONVERGENCEVERIFIER_H
+
+#include "llvm/ADT/GenericConvergenceVerifier.h"
+#include "llvm/IR/SSAContext.h"
+
+namespace llvm {
+
+using ConvergenceVerifier = GenericConvergenceVerifier<SSAContext>;
+
+} // namespace llvm
+
+#endif // LLVM_IR_CONVERGENCEVERIFIER_H

llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h

@@ -0,0 +1,202 @@
+//===- GenericConvergenceVerifierImpl.h -----------------------*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// A verifier for the static rules of convergence control tokens that works
+/// with both LLVM IR and MIR.
+///
+/// This template implementation resides in a separate file so that it does not
+/// get injected into every .cpp file that includes the generic header.
+///
+/// DO NOT INCLUDE THIS FILE WHEN MERELY USING CYCLEINFO.
+///
+/// This file should only be included by files that implement a
+/// specialization of the relevant templates. Currently these are:
+/// - llvm/lib/IR/Verifier.cpp
+/// - llvm/lib/CodeGen/MachineVerifier.cpp
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_IR_GENERICCONVERGENCEVERIFIERIMPL_H
+#define LLVM_IR_GENERICCONVERGENCEVERIFIERIMPL_H
+
+#include "llvm/ADT/GenericConvergenceVerifier.h"
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/IR/Intrinsics.h"
+
+using namespace llvm;
+
+#define Check(C, ...)                                                          \
+  do {                                                                         \
+    if (!(C)) {                                                                \
+      reportFailure(__VA_ARGS__);                                              \
+      return;                                                                  \
+    }                                                                          \
+  } while (false)
+
+#define CheckOrNull(C, ...)                                                    \
+  do {                                                                         \
+    if (!(C)) {                                                                \
+      reportFailure(__VA_ARGS__);                                              \
+      return {};                                                               \
+    }                                                                          \
+  } while (false)
+
+static bool isConvergenceControlIntrinsic(unsigned IntrinsicID) {
+  switch (IntrinsicID) {
+  default:
+    return false;
+  case Intrinsic::experimental_convergence_anchor:
+  case Intrinsic::experimental_convergence_entry:
+  case Intrinsic::experimental_convergence_loop:
+    return true;
+  }
+}
+
+namespace llvm {
+template <class ContextT> void GenericConvergenceVerifier<ContextT>::clear() {
+  Tokens.clear();
+  CI.clear();
+  ConvergenceKind = NoConvergence;
+}
+
+template <class ContextT>
+void GenericConvergenceVerifier<ContextT>::visit(const InstructionT &I) {
+  if (isControlledConvergent(I)) {
+    Check(isConvergent(I),
+          "Expected convergent attribute on a controlled convergent call.",
+          {Context.print(&I)});
+    Check(ConvergenceKind != UncontrolledConvergence,
+          "Cannot mix controlled and uncontrolled convergence in the same "
+          "function.",
+          {Context.print(&I)});
+    ConvergenceKind = ControlledConvergence;
+  } else if (isConvergent(I)) {
+    Check(ConvergenceKind != ControlledConvergence,
+          "Cannot mix controlled and uncontrolled convergence in the same "
+          "function.",
+          {Context.print(&I)});
+    ConvergenceKind = UncontrolledConvergence;
+  }
+}
+
+template <class ContextT>
+void GenericConvergenceVerifier<ContextT>::reportFailure(
+    const Twine &Message, ArrayRef<Printable> DumpedValues) {
+  FailureCB(Message);
+  if (OS) {
+    for (auto V : DumpedValues)
+      *OS << V << '\n';
+  }
+}
+
+template <class ContextT>
+void GenericConvergenceVerifier<ContextT>::verify(const DominatorTreeT &DT) {
+  assert(Context.getFunction());
+  const auto &F = *Context.getFunction();
+
+  DenseMap<const BlockT *, SmallVector<const InstructionT *, 8>> LiveTokenMap;
+  DenseMap<const CycleT *, const InstructionT *> CycleHearts;
+
+  // Just like the DominatorTree, compute the CycleInfo locally so that we
+  // can run the verifier outside of a pass manager and we don't rely on
+  // potentially out-dated analysis results.
+  CI.compute(const_cast<FunctionT &>(F));
+
+  auto checkToken = [&](const InstructionT *Token, const InstructionT *User,
+                        SmallVectorImpl<const InstructionT *> &LiveTokens) {
+    Check(llvm::is_contained(LiveTokens, Token),
+          "Convergence region is not well-nested.",
+          {Context.print(Token), Context.print(User)});
+    while (LiveTokens.back() != Token)
+      LiveTokens.pop_back();
+
+    // Check static rules about cycles.
+    auto *BB = User->getParent();
+    auto *BBCycle = CI.getCycle(BB);
+    if (!BBCycle)
+      return;
+
+    auto *DefBB = Token->getParent();
+    if (DefBB == BB || BBCycle->contains(DefBB)) {
+      // degenerate occurrence of a loop intrinsic
+      return;
+    }
+
+    Check(ContextT::getIntrinsicID(*User) ==
+              Intrinsic::experimental_convergence_loop,
+          "Convergence token used by an instruction other than "
+          "llvm.experimental.convergence.loop in a cycle that does "
+          "not contain the token's definition.",
+          {Context.print(User), CI.print(BBCycle)});
+
+    while (true) {
+      auto *Parent = BBCycle->getParentCycle();
+      if (!Parent || Parent->contains(DefBB))
+        break;
+      BBCycle = Parent;
+    };
+
+    Check(BBCycle->isReducible() && BB == BBCycle->getHeader(),
+          "Cycle heart must dominate all blocks in the cycle.",
+          {Context.print(User), Context.printAsOperand(BB), CI.print(BBCycle)});
+    Check(!CycleHearts.count(BBCycle),
+          "Two static convergence token uses in a cycle that does "
+          "not contain either token's definition.",
+          {Context.print(User), Context.print(CycleHearts[BBCycle]),
+           CI.print(BBCycle)});
+    CycleHearts[BBCycle] = User;
+  };
+
+  ReversePostOrderTraversal<const FunctionT *> RPOT(&F);
+  SmallVector<const InstructionT *, 8> LiveTokens;
+  for (auto *BB : RPOT) {
+    LiveTokens.clear();
+    auto LTIt = LiveTokenMap.find(BB);
+    if (LTIt != LiveTokenMap.end()) {
+      LiveTokens = std::move(LTIt->second);
+      LiveTokenMap.erase(LTIt);
+    }
+
+    for (auto &I : *BB) {
+      if (auto *Token = Tokens.lookup(&I))
+        checkToken(Token, &I, LiveTokens);
+      if (isConvergenceControlIntrinsic(ContextT::getIntrinsicID(I)))
+        LiveTokens.push_back(&I);
+    }
+
+    // Propagate token liveness
+    for (auto *Succ : successors(BB)) {
+      auto *SuccNode = DT.getNode(Succ);
+      auto LTIt = LiveTokenMap.find(Succ);
+      if (LTIt == LiveTokenMap.end()) {
+        // We're the first predecessor: all tokens which dominate the
+        // successor are live for now.
+        LTIt = LiveTokenMap.try_emplace(Succ).first;
+        for (auto LiveToken : LiveTokens) {
+          if (!DT.dominates(DT.getNode(LiveToken->getParent()), SuccNode))
+            break;
+          LTIt->second.push_back(LiveToken);
+        }
+      } else {
+        // Compute the intersection of live tokens.
+        auto It = llvm::partition(
+            LTIt->second, [&LiveTokens](const InstructionT *Token) {
+              return llvm::is_contained(LiveTokens, Token);
+            });
+        LTIt->second.erase(It, LTIt->second.end());
+      }
+    }
+  }
+}
+
+} // end namespace llvm
+
+#endif // LLVM_IR_GENERICCONVERGENCEVERIFIERIMPL_H

llvm/lib/CodeGen/MachineSSAContext.cpp

@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/MachineSSAContext.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstr.h"
@@ -58,6 +59,13 @@ bool MachineSSAContext::isConstantOrUndefValuePhi(const MachineInstr &Phi) {
   return Phi.isConstantValuePHI();
 }
 
+template <>
+Intrinsic::ID MachineSSAContext::getIntrinsicID(const MachineInstr &MI) {
+  if (auto *GI = dyn_cast<GIntrinsic>(&MI))
+    return GI->getIntrinsicID();
+  return Intrinsic::not_intrinsic;
+}
+
 template <>
 Printable MachineSSAContext::print(const MachineBasicBlock *Block) const {
   if (!Block)
@@ -83,3 +91,8 @@ template <> Printable MachineSSAContext::print(Register Value) const {
     }
   });
 }
+
+template <>
+Printable MachineSSAContext::printAsOperand(const MachineBasicBlock *BB) const {
+  return Printable([BB](raw_ostream &Out) { BB->printAsOperand(Out); });
+}

llvm/lib/IR/CMakeLists.txt

@@ -10,6 +10,7 @@ add_llvm_component_library(LLVMCore
   ConstantFold.cpp
   ConstantRange.cpp
   Constants.cpp
+  ConvergenceVerifier.cpp
   Core.cpp
   CycleInfo.cpp
   DIBuilder.cpp

llvm/lib/IR/ConvergenceVerifier.cpp

@@ -0,0 +1,77 @@
+//===- ConvergenceVerifier.cpp - Verify convergence control -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/ConvergenceVerifier.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/GenericConvergenceVerifierImpl.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/SSAContext.h"
+
+using namespace llvm;
+
+template <>
+const Instruction *
+GenericConvergenceVerifier<SSAContext>::findAndCheckConvergenceTokenUsed(
+    const Instruction &I) {
+  auto *CB = dyn_cast<CallBase>(&I);
+  if (!CB)
+    return nullptr;
+
+  unsigned Count =
+      CB->countOperandBundlesOfType(LLVMContext::OB_convergencectrl);
+  CheckOrNull(Count <= 1,
+              "The 'convergencectrl' bundle can occur at most once on a call",
+              {Context.print(CB)});
+  if (!Count)
+    return nullptr;
+
+  auto Bundle = CB->getOperandBundle(LLVMContext::OB_convergencectrl);
+  CheckOrNull(Bundle->Inputs.size() == 1 &&
+                  Bundle->Inputs[0]->getType()->isTokenTy(),
+              "The 'convergencectrl' bundle requires exactly one token use.",
+              {Context.print(CB)});
+  auto *Token = Bundle->Inputs[0].get();
+  auto *Def = dyn_cast<Instruction>(Token);
+
+  CheckOrNull(
+      Def && isConvergenceControlIntrinsic(SSAContext::getIntrinsicID(*Def)),
+      "Convergence control tokens can only be produced by calls to the "
+      "convergence control intrinsics.",
+      {Context.print(Token), Context.print(&I)});
+
+  if (Def)
+    Tokens[&I] = Def;
+
+  return Def;
+}
+
+template <>
+bool GenericConvergenceVerifier<SSAContext>::isConvergent(
+    const InstructionT &I) const {
+  if (auto *CB = dyn_cast<CallBase>(&I)) {
+    return CB->isConvergent();
+  }
+  return false;
+}
+
+template <>
+bool GenericConvergenceVerifier<SSAContext>::isControlledConvergent(
+    const InstructionT &I) {
+  // First find a token and place it in the map.
+  if (findAndCheckConvergenceTokenUsed(I))
+    return true;
+
+  // The entry and anchor intrinsics do not use a token, so we do a broad check
+  // here. The loop intrinsic will be checked separately for a missing token.
+  if (isConvergenceControlIntrinsic(SSAContext::getIntrinsicID(I)))
+    return true;
+
+  return false;
+}
+
+template class llvm::GenericConvergenceVerifier<SSAContext>;

llvm/lib/IR/SSAContext.cpp

@@ -16,10 +16,10 @@
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Function.h"
-#include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
-#include "llvm/Support/raw_ostream.h"
+#include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/ModuleSlotTracker.h"
+#include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 
@@ -69,6 +69,12 @@ bool SSAContext::isConstantOrUndefValuePhi(const Instruction &Instr) {
   return false;
 }
 
+template <> Intrinsic::ID SSAContext::getIntrinsicID(const Instruction &I) {
+  if (auto *CB = dyn_cast<CallBase>(&I))
+    return CB->getIntrinsicID();
+  return Intrinsic::not_intrinsic;
+}
+
 template <> Printable SSAContext::print(const Value *V) const {
   return Printable([V](raw_ostream &Out) { V->print(Out); });
 }
@@ -89,3 +95,7 @@ template <> Printable SSAContext::print(const BasicBlock *BB) const {
     Out << MST.getLocalSlot(BB);
   });
 }
+
+template <> Printable SSAContext::printAsOperand(const BasicBlock *BB) const {
+  return Printable([BB](raw_ostream &Out) { BB->printAsOperand(Out); });
+}

llvm/lib/IR/Verifier.cpp

@@ -73,7 +73,7 @@
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/Constants.h"
-#include "llvm/IR/CycleInfo.h"
+#include "llvm/IR/ConvergenceVerifier.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/DebugInfoMetadata.h"
@@ -329,13 +329,6 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport {
   /// The current source language.
   dwarf::SourceLanguage CurrentSourceLang = dwarf::DW_LANG_lo_user;
 
-  /// Whether the current function has convergencectrl operand bundles.
-  enum {
-    ControlledConvergence,
-    UncontrolledConvergence,
-    NoConvergence
-  } ConvergenceKind = NoConvergence;
-
   /// Whether source was present on the first DIFile encountered in each CU.
   DenseMap<const DICompileUnit *, bool> HasSourceDebugInfo;
 
@@ -370,6 +363,7 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport {
   SmallVector<const DILocalVariable *, 16> DebugFnArgs;
 
   TBAAVerifier TBAAVerifyHelper;
+  ConvergenceVerifier CV;
 
   SmallVector<IntrinsicInst *, 4> NoAliasScopeDecls;
 
@@ -411,12 +405,19 @@ public:
       return false;
     }
 
+    auto FailureCB = [this](const Twine &Message) {
+      this->CheckFailed(Message);
+    };
+    CV.initialize(OS, FailureCB, F);
+
     Broken = false;
     // FIXME: We strip const here because the inst visitor strips const.
     visit(const_cast<Function &>(F));
     verifySiblingFuncletUnwinds();
-    if (ConvergenceKind == ControlledConvergence)
-      verifyConvergenceControl(const_cast<Function &>(F));
+
+    if (CV.sawTokens())
+      CV.verify(DT);
+
     InstsInThisBlock.clear();
     DebugFnArgs.clear();
     LandingPadResultTy = nullptr;
@@ -424,7 +425,6 @@ public:
     SiblingFuncletInfo.clear();
     verifyNoAliasScopeDecl();
     NoAliasScopeDecls.clear();
-    ConvergenceKind = NoConvergence;
 
     return !Broken;
   }
@@ -600,7 +600,6 @@ private:
   void verifyStatepoint(const CallBase &Call);
   void verifyFrameRecoverIndices();
   void verifySiblingFuncletUnwinds();
-  void verifyConvergenceControl(Function &F);
 
   void verifyFragmentExpression(const DbgVariableIntrinsic &I);
   template <typename ValueOrMetadata>
@@ -2535,138 +2534,6 @@ void Verifier::verifySiblingFuncletUnwinds() {
   }
 }
 
-static bool isConvergenceControlIntrinsic(const CallBase &Call) {
-  switch (Call.getIntrinsicID()) {
-  case Intrinsic::experimental_convergence_anchor:
-  case Intrinsic::experimental_convergence_entry:
-  case Intrinsic::experimental_convergence_loop:
-    return true;
-  default:
-    return false;
-  }
-}
-
-static bool isControlledConvergent(const CallBase &Call) {
-  if (Call.countOperandBundlesOfType(LLVMContext::OB_convergencectrl))
-    return true;
-  return isConvergenceControlIntrinsic(Call);
-}
-
-void Verifier::verifyConvergenceControl(Function &F) {
-  DenseMap<BasicBlock *, SmallVector<CallBase *, 8>> LiveTokenMap;
-  DenseMap<const Cycle *, const CallBase *> CycleHearts;
-
-  // Just like the DominatorTree, compute the CycleInfo locally so that we
-  // can run the verifier outside of a pass manager and we don't rely on
-  // potentially out-dated analysis results.
-  CycleInfo CI;
-  CI.compute(F);
-
-  auto checkBundle = [&](OperandBundleUse &Bundle, CallBase *CB,
-                         SmallVectorImpl<CallBase *> &LiveTokens) {
-    Check(Bundle.Inputs.size() == 1 && Bundle.Inputs[0]->getType()->isTokenTy(),
-          "The 'convergencectrl' bundle requires exactly one token use.", CB);
-
-    Value *Token = Bundle.Inputs[0].get();
-    auto *Def = dyn_cast<CallBase>(Token);
-    Check(Def && isConvergenceControlIntrinsic(*Def),
-          "Convergence control tokens can only be produced by calls to the "
-          "convergence control intrinsics.",
-          Token, CB);
-
-    Check(llvm::is_contained(LiveTokens, Token),
-          "Convergence region is not well-nested.", Token, CB);
-
-    while (LiveTokens.back() != Token)
-      LiveTokens.pop_back();
-
-    // Check static rules about cycles.
-    auto *BB = CB->getParent();
-    auto *BBCycle = CI.getCycle(BB);
-    if (!BBCycle)
-      return;
-
-    BasicBlock *DefBB = Def->getParent();
-    if (DefBB == BB || BBCycle->contains(DefBB)) {
-      // degenerate occurrence of a loop intrinsic
-      return;
-    }
-
-    auto *II = dyn_cast<IntrinsicInst>(CB);
-    Check(II &&
-              II->getIntrinsicID() == Intrinsic::experimental_convergence_loop,
-          "Convergence token used by an instruction other than "
-          "llvm.experimental.convergence.loop in a cycle that does "
-          "not contain the token's definition.",
-          CB, CI.print(BBCycle));
-
-    while (true) {
-      auto *Parent = BBCycle->getParentCycle();
-      if (!Parent || Parent->contains(DefBB))
-        break;
-      BBCycle = Parent;
-    };
-
-    Check(BBCycle->isReducible() && BB == BBCycle->getHeader(),
-          "Cycle heart must dominate all blocks in the cycle.", CB, BB,
-          CI.print(BBCycle));
-    Check(!CycleHearts.count(BBCycle),
-          "Two static convergence token uses in a cycle that does "
-          "not contain either token's definition.",
-          CB, CycleHearts[BBCycle], CI.print(BBCycle));
-    CycleHearts[BBCycle] = CB;
-  };
-
-  ReversePostOrderTraversal<Function *> RPOT(&F);
-  SmallVector<CallBase *, 8> LiveTokens;
-  for (BasicBlock *BB : RPOT) {
-    LiveTokens.clear();
-    auto LTIt = LiveTokenMap.find(BB);
-    if (LTIt != LiveTokenMap.end()) {
-      LiveTokens = std::move(LTIt->second);
-      LiveTokenMap.erase(LTIt);
-    }
-
-    for (Instruction &I : *BB) {
-      CallBase *CB = dyn_cast<CallBase>(&I);
-      if (!CB)
-        continue;
-
-      Check(CB->countOperandBundlesOfType(LLVMContext::OB_convergencectrl) <= 1,
-            "The 'convergencetrl' bundle can occur at most once on a call", CB);
-
-      auto Bundle = CB->getOperandBundle(LLVMContext::OB_convergencectrl);
-      if (Bundle)
-        checkBundle(*Bundle, CB, LiveTokens);
-
-      if (CB->getType()->isTokenTy())
-        LiveTokens.push_back(CB);
-    }
-
-    // Propagate token liveness
-    for (BasicBlock *Succ : successors(BB)) {
-      DomTreeNode *SuccNode = DT.getNode(Succ);
-      LTIt = LiveTokenMap.find(Succ);
-      if (LTIt == LiveTokenMap.end()) {
-        // We're the first predecessor: all tokens which dominate the
-        // successor are live for now.
-        LTIt = LiveTokenMap.try_emplace(Succ).first;
-        for (CallBase *LiveToken : LiveTokens) {
-          if (!DT.dominates(DT.getNode(LiveToken->getParent()), SuccNode))
-            break;
-          LTIt->second.push_back(LiveToken);
-        }
-      } else {
-        // Compute the intersection of live tokens.
-        auto It = llvm::partition(LTIt->second, [&LiveTokens](CallBase *Token) {
-          return llvm::is_contained(LiveTokens, Token);
-        });
-        LTIt->second.erase(It, LTIt->second.end());
-      }
-    }
-  }
-}
-
 // visitFunction - Verify that a function is ok.
 //
 void Verifier::visitFunction(const Function &F) {
@@ -3688,22 +3555,7 @@ void Verifier::visitCallBase(CallBase &Call) {
   if (Call.isInlineAsm())
     verifyInlineAsmCall(Call);
 
-  if (isControlledConvergent(Call)) {
-    Check(Call.isConvergent(),
-          "Expected convergent attribute on a controlled convergent call.",
-          Call);
-    Check(ConvergenceKind != UncontrolledConvergence,
-          "Cannot mix controlled and uncontrolled convergence in the same "
-          "function.",
-          Call);
-    ConvergenceKind = ControlledConvergence;
-  } else if (Call.isConvergent()) {
-    Check(ConvergenceKind != ControlledConvergence,
-          "Cannot mix controlled and uncontrolled convergence in the same "
-          "function.",
-          Call);
-    ConvergenceKind = UncontrolledConvergence;
-  }
+  CV.visit(Call);
 
   visitInstruction(Call);
 }

llvm/test/Verifier/convergencectrl-invalid.ll

@@ -26,7 +26,7 @@ define void @multiple_tokens() {
   ret void
 }
 
-; CHECK: The 'convergencetrl' bundle can occur at most once on a call
+; CHECK: The 'convergencectrl' bundle can occur at most once on a call
 ; CHECK-NEXT:  call void @g()
 define void @multiple_bundles() {
   %t07_tok1 = call token @llvm.experimental.convergence.anchor()