[NewPM][CodeGen] Add MachineFunctionAnalysis (#88610)

In new pass system, `MachineFunction` could be an analysis result again, machine module pass can now fetch them from analysis manager. `MachineModuleInfo` no longer owns them. Remove `FreeMachineFunctionPass`, replaced by `InvalidateAnalysisPass<MachineFunctionAnalysis>`. Now `FreeMachineFunction` is replaced by `InvalidateAnalysisPass<MachineFunctionAnalysis>`, the workaround in `MachineFunctionPassManager` is no longer needed, there is no difference between `unittests/MIR/PassBuilderCallbacksTest.cpp` and `unittests/IR/PassBuilderCallbacksTest.cpp`.
2024-04-30 09:54:48 +08:00
parent b1867e18c3
commit 6ea0c0a283
26 changed files with 301 additions and 734 deletions
--- a/llvm/include/llvm/CodeGen/FreeMachineFunction.h
+++ b/llvm/include/llvm/CodeGen/FreeMachineFunction.h
@@ -1,24 +0,0 @@
-//===- llvm/CodeGen/FreeMachineFunction.h -----------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CODEGEN_FREEMACHINEFUNCTION_H
-#define LLVM_CODEGEN_FREEMACHINEFUNCTION_H
-
-#include "llvm/CodeGen/MachinePassManager.h"
-
-namespace llvm {
-
-class FreeMachineFunctionPass : public PassInfoMixin<FreeMachineFunctionPass> {
-public:
-  PreservedAnalyses run(MachineFunction &MF,
-                        MachineFunctionAnalysisManager &MFAM);
-};
-
-} // namespace llvm
-
-#endif // LLVM_CODEGEN_FREEMACHINEFUNCTION_H
--- a/llvm/include/llvm/CodeGen/MIRParser/MIRParser.h
+++ b/llvm/include/llvm/CodeGen/MIRParser/MIRParser.h
@@ -34,6 +34,9 @@ class MachineModuleInfo;
 class SMDiagnostic;
 class StringRef;

+template <typename IRUnitT, typename... ExtraArgTs> class AnalysisManager;
+using ModuleAnalysisManager = AnalysisManager<Module>;
+
 typedef llvm::function_ref<std::optional<std::string>(StringRef, StringRef)>
    DataLayoutCallbackTy;

@@ -60,6 +63,15 @@ public:
  ///
  /// \returns true if an error occurred.
  bool parseMachineFunctions(Module &M, MachineModuleInfo &MMI);
+
+  /// Parses MachineFunctions in the MIR file and add them as the result
+  /// of MachineFunctionAnalysis in ModulePassManager \p MAM.
+  /// User should register at least MachineFunctionAnalysis,
+  /// MachineModuleAnalysis, FunctionAnalysisManagerModuleProxy and
+  /// PassInstrumentationAnalysis in \p MAM before parsing MIR.
+  ///
+  /// \returns true if an error occurred.
+  bool parseMachineFunctions(Module &M, ModuleAnalysisManager &MAM);
 };

 /// This function is the main interface to the MIR serialization format parser.
--- a/llvm/include/llvm/CodeGen/MachineFunctionAnalysis.h
+++ b/llvm/include/llvm/CodeGen/MachineFunctionAnalysis.h
@@ -0,0 +1,50 @@
+//===- llvm/CodeGen/MachineFunctionAnalysis.h -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the MachineFunctionAnalysis class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_MACHINEFUNCTIONANALYSIS
+#define LLVM_CODEGEN_MACHINEFUNCTIONANALYSIS
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class MachineFunction;
+class LLVMTargetMachine;
+
+/// This analysis create MachineFunction for given Function.
+/// To release the MachineFunction, users should invalidate it explicitly.
+class MachineFunctionAnalysis
+    : public AnalysisInfoMixin<MachineFunctionAnalysis> {
+  friend AnalysisInfoMixin<MachineFunctionAnalysis>;
+
+  static AnalysisKey Key;
+
+  const LLVMTargetMachine *TM;
+
+public:
+  class Result {
+    std::unique_ptr<MachineFunction> MF;
+
+  public:
+    Result(std::unique_ptr<MachineFunction> MF) : MF(std::move(MF)) {}
+    MachineFunction &getMF() { return *MF; };
+    bool invalidate(Function &, const PreservedAnalyses &PA,
+                    FunctionAnalysisManager::Invalidator &);
+  };
+
+  MachineFunctionAnalysis(const LLVMTargetMachine *TM) : TM(TM){};
+  Result run(Function &F, FunctionAnalysisManager &FAM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_MachineFunctionAnalysis
--- a/llvm/include/llvm/CodeGen/MachineModuleInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineModuleInfo.h
@@ -147,10 +147,14 @@ public:

  /// Returns the MachineFunction constructed for the IR function \p F.
  /// Creates a new MachineFunction if none exists yet.
+  /// NOTE: New pass manager clients shall not use this method to get
+  /// the `MachineFunction`, use `MachineFunctionAnalysis` instead.
  MachineFunction &getOrCreateMachineFunction(Function &F);

  /// \brief Returns the MachineFunction associated to IR function \p F if there
  /// is one, otherwise nullptr.
+  /// NOTE: New pass manager clients shall not use this method to get
+  /// the `MachineFunction`, use `MachineFunctionAnalysis` instead.
  MachineFunction *getMachineFunction(const Function &F) const;

  /// Delete the MachineFunction \p MF and reset the link in the IR Function to
--- a/llvm/include/llvm/CodeGen/MachinePassManager.h
+++ b/llvm/include/llvm/CodeGen/MachinePassManager.h
@@ -108,6 +108,15 @@ bool MachineFunctionAnalysisManagerModuleProxy::Result::invalidate(
    ModuleAnalysisManager::Invalidator &Inv);
 extern template class InnerAnalysisManagerProxy<MachineFunctionAnalysisManager,
                                                Module>;
+using MachineFunctionAnalysisManagerFunctionProxy =
+    InnerAnalysisManagerProxy<MachineFunctionAnalysisManager, Function>;
+
+template <>
+bool MachineFunctionAnalysisManagerFunctionProxy::Result::invalidate(
+    Function &F, const PreservedAnalyses &PA,
+    FunctionAnalysisManager::Invalidator &Inv);
+extern template class InnerAnalysisManagerProxy<MachineFunctionAnalysisManager,
+                                                Function>;

 extern template class OuterAnalysisManagerProxy<ModuleAnalysisManager,
                                                MachineFunction>;
@@ -129,16 +138,6 @@ public:
      Arg.FAM = nullptr;
    }

-    ~Result() {
-      // FAM is cleared in a moved from state where there is nothing to do.
-      if (!FAM)
-        return;
-
-      // Clear out the analysis manager if we're being destroyed -- it means we
-      // didn't even see an invalidate call when we got invalidated.
-      FAM->clear();
-    }
-
    Result &operator=(Result &&RHS) {
      FAM = RHS.FAM;
      // We have to null out the analysis manager in the moved-from state
@@ -187,18 +186,18 @@ private:
  FunctionAnalysisManager *FAM;
 };

-class ModuleToMachineFunctionPassAdaptor
-    : public PassInfoMixin<ModuleToMachineFunctionPassAdaptor> {
+class FunctionToMachineFunctionPassAdaptor
+    : public PassInfoMixin<FunctionToMachineFunctionPassAdaptor> {
 public:
  using PassConceptT =
      detail::PassConcept<MachineFunction, MachineFunctionAnalysisManager>;

-  explicit ModuleToMachineFunctionPassAdaptor(
+  explicit FunctionToMachineFunctionPassAdaptor(
      std::unique_ptr<PassConceptT> Pass)
      : Pass(std::move(Pass)) {}

-  /// Runs the function pass across every function in the module.
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  /// Runs the function pass across every function in the function.
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
  void printPipeline(raw_ostream &OS,
                     function_ref<StringRef(StringRef)> MapClassName2PassName);

@@ -209,14 +208,14 @@ private:
 };

 template <typename MachineFunctionPassT>
-ModuleToMachineFunctionPassAdaptor
-createModuleToMachineFunctionPassAdaptor(MachineFunctionPassT &&Pass) {
+FunctionToMachineFunctionPassAdaptor
+createFunctionToMachineFunctionPassAdaptor(MachineFunctionPassT &&Pass) {
  using PassModelT = detail::PassModel<MachineFunction, MachineFunctionPassT,
                                       MachineFunctionAnalysisManager>;
  // Do not use make_unique, it causes too many template instantiations,
  // causing terrible compile times.
-  return ModuleToMachineFunctionPassAdaptor(
-      std::unique_ptr<ModuleToMachineFunctionPassAdaptor::PassConceptT>(
+  return FunctionToMachineFunctionPassAdaptor(
+      std::unique_ptr<FunctionToMachineFunctionPassAdaptor::PassConceptT>(
          new PassModelT(std::forward<MachineFunctionPassT>(Pass))));
 }

@@ -244,6 +243,10 @@ extern template class PassManager<MachineFunction>;
 /// Convenience typedef for a pass manager over functions.
 using MachineFunctionPassManager = PassManager<MachineFunction>;

+/// Returns the minimum set of Analyses that all machine function passes must
+/// preserve.
+PreservedAnalyses getMachineFunctionPassPreservedAnalyses();
+
 } // end namespace llvm

 #endif // LLVM_CODEGEN_MACHINEPASSMANAGER_H
--- a/llvm/include/llvm/IR/LLVMContext.h
+++ b/llvm/include/llvm/IR/LLVMContext.h
@@ -155,6 +155,10 @@ public:
  void enableDebugTypeODRUniquing();
  void disableDebugTypeODRUniquing();

+  /// generateMachineFunctionNum - Get a unique number for MachineFunction
+  /// that associated with the given Function.
+  unsigned generateMachineFunctionNum(Function &);
+
  /// Defines the type of a yield callback.
  /// \see LLVMContext::setYieldCallback.
  using YieldCallbackTy = void (*)(LLVMContext *Context, void *OpaqueHandle);
@@ -332,7 +336,7 @@ private:
  void addModule(Module*);

  /// removeModule - Unregister a module from this context.
-  void removeModule(Module*);
+  void removeModule(Module *);
 };

 // Create wrappers for C Binding types (see CBindingWrapping.h).
--- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
@@ -29,7 +29,6 @@
 #include "llvm/CodeGen/DwarfEHPrepare.h"
 #include "llvm/CodeGen/ExpandMemCmp.h"
 #include "llvm/CodeGen/ExpandReductions.h"
-#include "llvm/CodeGen/FreeMachineFunction.h"
 #include "llvm/CodeGen/GCMetadata.h"
 #include "llvm/CodeGen/GlobalMerge.h"
 #include "llvm/CodeGen/IndirectBrExpand.h"
@@ -38,6 +37,8 @@
 #include "llvm/CodeGen/JMCInstrumenter.h"
 #include "llvm/CodeGen/LowerEmuTLS.h"
 #include "llvm/CodeGen/MIRPrinter.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
 #include "llvm/CodeGen/ReplaceWithVeclib.h"
@@ -199,8 +200,13 @@ protected:
    AddMachinePass(ModulePassManager &MPM, const DerivedT &PB)
        : MPM(MPM), PB(PB) {}
    ~AddMachinePass() {
-      if (!MFPM.isEmpty())
-        MPM.addPass(createModuleToMachineFunctionPassAdaptor(std::move(MFPM)));
+      if (!MFPM.isEmpty()) {
+        FunctionPassManager FPM;
+        FPM.addPass(
+            createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
+        FPM.addPass(InvalidateAnalysisPass<MachineFunctionAnalysis>());
+        MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+      }
    }

    template <typename PassT>
@@ -219,8 +225,8 @@ protected:
      } else {
        // Add Module Pass
        if (!MFPM.isEmpty()) {
-          MPM.addPass(
-              createModuleToMachineFunctionPassAdaptor(std::move(MFPM)));
+          MPM.addPass(createModuleToFunctionPassAdaptor(
+              createFunctionToMachineFunctionPassAdaptor(std::move(MFPM))));
          MFPM = MachineFunctionPassManager();
        }

@@ -512,6 +518,7 @@ Error CodeGenPassBuilder<Derived, TargetMachineT>::buildPipeline(

  {
    AddIRPass addIRPass(MPM, derived());
+    addIRPass(RequireAnalysisPass<MachineModuleAnalysis, Module>());
    addIRPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
    addIRPass(RequireAnalysisPass<CollectorMetadataAnalysis, Module>());
    addISelPasses(addIRPass);
@@ -538,7 +545,6 @@ Error CodeGenPassBuilder<Derived, TargetMachineT>::buildPipeline(
  if (PrintMIR)
    addPass(PrintMIRPass(Out), /*Force=*/true);

-  addPass(FreeMachineFunctionPass());
  return verifyStartStop(*StartStopInfo);
 }

--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -124,7 +124,6 @@ MACHINE_FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PI
 #define MACHINE_FUNCTION_PASS(NAME, CREATE_PASS)
 #endif
 MACHINE_FUNCTION_PASS("dead-mi-elimination", DeadMachineInstructionElimPass())
-// MACHINE_FUNCTION_PASS("free-machine-function", FreeMachineFunctionPass())
 MACHINE_FUNCTION_PASS("no-op-machine-function", NoOpMachineFunctionPass())
 MACHINE_FUNCTION_PASS("print", PrintMIRPass())
 MACHINE_FUNCTION_PASS("require-all-machine-function-properties",
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -65,8 +65,8 @@ add_llvm_component_library(LLVMCodeGen
  FEntryInserter.cpp
  FinalizeISel.cpp
  FixupStatepointCallerSaved.cpp
-  FreeMachineFunction.cpp
  FuncletLayout.cpp
+  MachineFunctionAnalysis.cpp
  GCMetadata.cpp
  GCMetadataPrinter.cpp
  GCRootLowering.cpp
--- a/llvm/lib/CodeGen/DeadMachineInstructionElim.cpp
+++ b/llvm/lib/CodeGen/DeadMachineInstructionElim.cpp
@@ -68,7 +68,7 @@ DeadMachineInstructionElimPass::run(MachineFunction &MF,
                                    MachineFunctionAnalysisManager &) {
  if (!DeadMachineInstructionElimImpl().runImpl(MF))
    return PreservedAnalyses::all();
-  PreservedAnalyses PA;
+  PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
  PA.preserveSet<CFGAnalyses>();
  return PA;
 }
--- a/llvm/lib/CodeGen/FreeMachineFunction.cpp
+++ b/llvm/lib/CodeGen/FreeMachineFunction.cpp
@@ -1,22 +0,0 @@
-//===- FreeMachineFunction.cpp --------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/CodeGen/FreeMachineFunction.h"
-#include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineModuleInfo.h"
-
-using namespace llvm;
-
-PreservedAnalyses
-FreeMachineFunctionPass::run(MachineFunction &MF,
-                             MachineFunctionAnalysisManager &MFAM) {
-  auto &MMI = MF.getMMI();
-  MFAM.invalidate(MF, PreservedAnalyses::none());
-  MMI.deleteMachineFunctionFor(MF.getFunction()); // MF is dangling now.
-  return PreservedAnalyses::none();
-}
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -21,6 +21,7 @@
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
@@ -97,13 +98,15 @@ public:
  /// Create an empty function with the given name.
  Function *createDummyFunction(StringRef Name, Module &M);

-  bool parseMachineFunctions(Module &M, MachineModuleInfo &MMI);
+  bool parseMachineFunctions(Module &M, MachineModuleInfo &MMI,
+                             ModuleAnalysisManager *FAM = nullptr);

  /// Parse the machine function in the current YAML document.
  ///
  ///
  /// Return true if an error occurred.
-  bool parseMachineFunction(Module &M, MachineModuleInfo &MMI);
+  bool parseMachineFunction(Module &M, MachineModuleInfo &MMI,
+                            ModuleAnalysisManager *FAM);

  /// Initialize the machine function to the state that's described in the MIR
  /// file.
@@ -275,13 +278,14 @@ MIRParserImpl::parseIRModule(DataLayoutCallbackTy DataLayoutCallback) {
  return M;
 }

-bool MIRParserImpl::parseMachineFunctions(Module &M, MachineModuleInfo &MMI) {
+bool MIRParserImpl::parseMachineFunctions(Module &M, MachineModuleInfo &MMI,
+                                          ModuleAnalysisManager *MAM) {
  if (NoMIRDocuments)
    return false;

  // Parse the machine functions.
  do {
-    if (parseMachineFunction(M, MMI))
+    if (parseMachineFunction(M, MMI, MAM))
      return true;
    In.nextDocument();
  } while (In.setCurrentDocument());
@@ -303,7 +307,8 @@ Function *MIRParserImpl::createDummyFunction(StringRef Name, Module &M) {
  return F;
 }

-bool MIRParserImpl::parseMachineFunction(Module &M, MachineModuleInfo &MMI) {
+bool MIRParserImpl::parseMachineFunction(Module &M, MachineModuleInfo &MMI,
+                                         ModuleAnalysisManager *MAM) {
  // Parse the yaml.
  yaml::MachineFunction YamlMF;
  yaml::EmptyContext Ctx;
@@ -327,14 +332,28 @@ bool MIRParserImpl::parseMachineFunction(Module &M, MachineModuleInfo &MMI) {
                   "' isn't defined in the provided LLVM IR");
    }
  }
-  if (MMI.getMachineFunction(*F) != nullptr)
-    return error(Twine("redefinition of machine function '") + FunctionName +
-                 "'");

-  // Create the MachineFunction.
-  MachineFunction &MF = MMI.getOrCreateMachineFunction(*F);
-  if (initializeMachineFunction(YamlMF, MF))
-    return true;
+  if (!MAM) {
+    if (MMI.getMachineFunction(*F) != nullptr)
+      return error(Twine("redefinition of machine function '") + FunctionName +
+                   "'");
+
+    // Create the MachineFunction.
+    MachineFunction &MF = MMI.getOrCreateMachineFunction(*F);
+    if (initializeMachineFunction(YamlMF, MF))
+      return true;
+  } else {
+    auto &FAM =
+        MAM->getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+    if (FAM.getCachedResult<MachineFunctionAnalysis>(*F))
+      return error(Twine("redefinition of machine function '") + FunctionName +
+                   "'");
+
+    // Create the MachineFunction.
+    MachineFunction &MF = FAM.getResult<MachineFunctionAnalysis>(*F).getMF();
+    if (initializeMachineFunction(YamlMF, MF))
+      return true;
+  }

  return false;
 }
@@ -1101,6 +1120,11 @@ bool MIRParser::parseMachineFunctions(Module &M, MachineModuleInfo &MMI) {
  return Impl->parseMachineFunctions(M, MMI);
 }

+bool MIRParser::parseMachineFunctions(Module &M, ModuleAnalysisManager &MAM) {
+  auto &MMI = MAM.getResult<MachineModuleAnalysis>(M).getMMI();
+  return Impl->parseMachineFunctions(M, MMI, &MAM);
+}
+
 std::unique_ptr<MIRParser> llvm::createMIRParserFromFile(
    StringRef Filename, SMDiagnostic &Error, LLVMContext &Context,
    std::function<void(Function &)> ProcessIRFunction) {
--- a/llvm/lib/CodeGen/MachineFunctionAnalysis.cpp
+++ b/llvm/lib/CodeGen/MachineFunctionAnalysis.cpp
@@ -0,0 +1,46 @@
+//===- MachineFunctionAnalysis.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the definitions of the MachineFunctionAnalysis
+// members.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/Target/TargetMachine.h"
+
+using namespace llvm;
+
+AnalysisKey MachineFunctionAnalysis::Key;
+
+bool MachineFunctionAnalysis::Result::invalidate(
+    Function &, const PreservedAnalyses &PA,
+    FunctionAnalysisManager::Invalidator &) {
+  // Unless it is invalidated explicitly, it should remain preserved.
+  auto PAC = PA.getChecker<MachineFunctionAnalysis>();
+  return !PAC.preservedWhenStateless();
+}
+
+MachineFunctionAnalysis::Result
+MachineFunctionAnalysis::run(Function &F, FunctionAnalysisManager &FAM) {
+  auto &Context = F.getContext();
+  const TargetSubtargetInfo &STI = *TM->getSubtargetImpl(F);
+  auto &MMI = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(F)
+                  .getCachedResult<MachineModuleAnalysis>(*F.getParent())
+                  ->getMMI();
+  auto MF = std::make_unique<MachineFunction>(
+      F, *TM, STI, Context.generateMachineFunctionNum(F), MMI);
+  MF->initTargetMachineFunctionInfo(STI);
+
+  // MRI callback for target specific initializations.
+  TM->registerMachineRegisterInfoCallback(*MF);
+
+  return Result(std::move(MF));
+}
--- a/llvm/lib/CodeGen/MachinePassManager.cpp
+++ b/llvm/lib/CodeGen/MachinePassManager.cpp
@@ -12,21 +12,24 @@

 #include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/IR/PassManagerImpl.h"

 using namespace llvm;

-namespace llvm {
-
 AnalysisKey FunctionAnalysisManagerMachineFunctionProxy::Key;

+namespace llvm {
 template class AnalysisManager<MachineFunction>;
 template class PassManager<MachineFunction>;
 template class InnerAnalysisManagerProxy<MachineFunctionAnalysisManager,
                                         Module>;
+template class InnerAnalysisManagerProxy<MachineFunctionAnalysisManager,
+                                         Function>;
 template class OuterAnalysisManagerProxy<ModuleAnalysisManager,
                                         MachineFunction>;
+} // namespace llvm

 bool FunctionAnalysisManagerMachineFunctionProxy::Result::invalidate(
    MachineFunction &IR, const PreservedAnalyses &PA,
@@ -69,38 +72,65 @@ bool MachineFunctionAnalysisManagerModuleProxy::Result::invalidate(
  return false;
 }

-PreservedAnalyses
-ModuleToMachineFunctionPassAdaptor::run(Module &M, ModuleAnalysisManager &AM) {
-  auto &MMI = AM.getResult<MachineModuleAnalysis>(M).getMMI();
-  MachineFunctionAnalysisManager &MFAM =
-      AM.getResult<MachineFunctionAnalysisManagerModuleProxy>(M).getManager();
-  PassInstrumentation PI = AM.getResult<PassInstrumentationAnalysis>(M);
-  PreservedAnalyses PA = PreservedAnalyses::all();
-  for (Function &F : M) {
-    // Do not codegen any 'available_externally' functions at all, they have
-    // definitions outside the translation unit.
-    if (F.isDeclaration() || F.hasAvailableExternallyLinkage())
-      continue;
+template <>
+bool MachineFunctionAnalysisManagerFunctionProxy::Result::invalidate(
+    Function &F, const PreservedAnalyses &PA,
+    FunctionAnalysisManager::Invalidator &Inv) {
+  // If literally everything is preserved, we're done.
+  if (PA.areAllPreserved())
+    return false; // This is still a valid proxy.

-    MachineFunction &MF = MMI.getOrCreateMachineFunction(F);
-
-    if (!PI.runBeforePass<MachineFunction>(*Pass, MF))
-      continue;
-    PreservedAnalyses PassPA = Pass->run(MF, MFAM);
-    if (MMI.getMachineFunction(F)) {
-      MFAM.invalidate(MF, PassPA);
-      PI.runAfterPass(*Pass, MF, PassPA);
-    } else {
-      MFAM.clear(MF, F.getName());
-      PI.runAfterPassInvalidated<MachineFunction>(*Pass, PassPA);
-    }
-    PA.intersect(std::move(PassPA));
+  // If this proxy isn't marked as preserved, then even if the result remains
+  // valid, the key itself may no longer be valid, so we clear everything.
+  //
+  // Note that in order to preserve this proxy, a module pass must ensure that
+  // the MFAM has been completely updated to handle the deletion of functions.
+  // Specifically, any MFAM-cached results for those functions need to have been
+  // forcibly cleared. When preserved, this proxy will only invalidate results
+  // cached on functions *still in the module* at the end of the module pass.
+  auto PAC = PA.getChecker<MachineFunctionAnalysisManagerFunctionProxy>();
+  if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<Function>>()) {
+    InnerAM->clear();
+    return true;
  }

+  // FIXME: be more precise, see
+  // FunctionAnalysisManagerModuleProxy::Result::invalidate.
+  if (!PA.allAnalysesInSetPreserved<AllAnalysesOn<MachineFunction>>()) {
+    InnerAM->clear();
+    return true;
+  }
+
+  // Return false to indicate that this result is still a valid proxy.
+  return false;
+}
+
+PreservedAnalyses
+FunctionToMachineFunctionPassAdaptor::run(Function &F,
+                                          FunctionAnalysisManager &FAM) {
+  MachineFunctionAnalysisManager &MFAM =
+      FAM.getResult<MachineFunctionAnalysisManagerFunctionProxy>(F)
+          .getManager();
+  PassInstrumentation PI = FAM.getResult<PassInstrumentationAnalysis>(F);
+  PreservedAnalyses PA = PreservedAnalyses::all();
+  // Do not codegen any 'available_externally' functions at all, they have
+  // definitions outside the translation unit.
+  if (F.isDeclaration() || F.hasAvailableExternallyLinkage())
+    return PreservedAnalyses::all();
+
+  MachineFunction &MF = FAM.getResult<MachineFunctionAnalysis>(F).getMF();
+
+  if (!PI.runBeforePass<MachineFunction>(*Pass, MF))
+    return PreservedAnalyses::all();
+  PreservedAnalyses PassPA = Pass->run(MF, MFAM);
+  MFAM.invalidate(MF, PassPA);
+  PI.runAfterPass(*Pass, MF, PassPA);
+  PA.intersect(std::move(PassPA));
+
  return PA;
 }

-void ModuleToMachineFunctionPassAdaptor::printPipeline(
+void FunctionToMachineFunctionPassAdaptor::printPipeline(
    raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
  OS << "machine-function(";
  Pass->printPipeline(OS, MapClassName2PassName);
@@ -112,27 +142,24 @@ PreservedAnalyses
 PassManager<MachineFunction>::run(MachineFunction &MF,
                                  AnalysisManager<MachineFunction> &MFAM) {
  PassInstrumentation PI = MFAM.getResult<PassInstrumentationAnalysis>(MF);
-  Function &F = MF.getFunction();
-  MachineModuleInfo &MMI =
-      MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(MF)
-          .getCachedResult<MachineModuleAnalysis>(*F.getParent())
-          ->getMMI();
  PreservedAnalyses PA = PreservedAnalyses::all();
  for (auto &Pass : Passes) {
    if (!PI.runBeforePass<MachineFunction>(*Pass, MF))
      continue;

    PreservedAnalyses PassPA = Pass->run(MF, MFAM);
-    if (MMI.getMachineFunction(F)) {
-      MFAM.invalidate(MF, PassPA);
-      PI.runAfterPass(*Pass, MF, PassPA);
-    } else {
-      MFAM.clear(MF, F.getName());
-      PI.runAfterPassInvalidated<MachineFunction>(*Pass, PassPA);
-    }
+    MFAM.invalidate(MF, PassPA);
+    PI.runAfterPass(*Pass, MF, PassPA);
    PA.intersect(std::move(PassPA));
  }
  return PA;
 }

-} // namespace llvm
+PreservedAnalyses llvm::getMachineFunctionPassPreservedAnalyses() {
+  PreservedAnalyses PA;
+  // Machine function passes are not allowed to modify the LLVM
+  // representation, therefore we should preserve all IR analyses.
+  PA.template preserveSet<AllAnalysesOn<Module>>();
+  PA.template preserveSet<AllAnalysesOn<Function>>();
+  return PA;
+}
--- a/llvm/lib/IR/LLVMContext.cpp
+++ b/llvm/lib/IR/LLVMContext.cpp
@@ -118,6 +118,13 @@ void LLVMContext::addModule(Module *M) {

 void LLVMContext::removeModule(Module *M) {
  pImpl->OwnedModules.erase(M);
+  pImpl->MachineFunctionNums.erase(M);
+}
+
+unsigned LLVMContext::generateMachineFunctionNum(Function &F) {
+  Module *M = F.getParent();
+  assert(pImpl->OwnedModules.contains(M) && "Unexpected module!");
+  return pImpl->MachineFunctionNums[M]++;
 }

 //===----------------------------------------------------------------------===//
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -1450,6 +1450,10 @@ public:
  /// will be automatically deleted if this context is deleted.
  SmallPtrSet<Module *, 4> OwnedModules;

+  /// MachineFunctionNums - Keep the next available unique number available for
+  /// a MachineFunction in given module. Module must in OwnedModules.
+  DenseMap<Module *, unsigned> MachineFunctionNums;
+
  /// The main remark streamer used by all the other streamers (e.g. IR, MIR,
  /// frontends, etc.). This should only be used by the specific streamers, and
  /// never directly.
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -82,7 +82,6 @@
 #include "llvm/CodeGen/ExpandLargeDivRem.h"
 #include "llvm/CodeGen/ExpandLargeFpConvert.h"
 #include "llvm/CodeGen/ExpandMemCmp.h"
-#include "llvm/CodeGen/FreeMachineFunction.h"
 #include "llvm/CodeGen/GCMetadata.h"
 #include "llvm/CodeGen/GlobalMerge.h"
 #include "llvm/CodeGen/HardwareLoops.h"
@@ -92,6 +91,7 @@
 #include "llvm/CodeGen/JMCInstrumenter.h"
 #include "llvm/CodeGen/LowerEmuTLS.h"
 #include "llvm/CodeGen/MIRPrinter.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
 #include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/SafeStack.h"
 #include "llvm/CodeGen/SelectOptimize.h"
@@ -1230,7 +1230,7 @@ static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
  StringRef NameNoBracket = Name.take_until([](char C) { return C == '<'; });
  if (NameNoBracket == "function")
    return true;
-  if (Name == "loop" || Name == "loop-mssa")
+  if (Name == "loop" || Name == "loop-mssa" || Name == "machine-function")
    return true;

  // Explicitly handle custom-parsed pass names.
@@ -1408,13 +1408,6 @@ Error PassBuilder::parseModulePass(ModulePassManager &MPM,
      MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
      return Error::success();
    }
-    if (Name == "machine-function") {
-      MachineFunctionPassManager MFPM;
-      if (auto Err = parseMachinePassPipeline(MFPM, InnerPipeline))
-        return Err;
-      MPM.addPass(createModuleToMachineFunctionPassAdaptor(std::move(MFPM)));
-      return Error::success();
-    }
    if (auto Params = parseFunctionPipelineName(Name)) {
      if (Params->second)
        return make_error<StringError>(
@@ -1732,6 +1725,13 @@ Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
      FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
      return Error::success();
    }
+    if (Name == "machine-function") {
+      MachineFunctionPassManager MFPM;
+      if (auto Err = parseMachinePassPipeline(MFPM, InnerPipeline))
+        return Err;
+      FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
+      return Error::success();
+    }

    for (auto &C : FunctionPipelineParsingCallbacks)
      if (C(Name, FPM, InnerPipeline))
@@ -1975,6 +1975,8 @@ void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
  if (MFAM) {
    MAM.registerPass(
        [&] { return MachineFunctionAnalysisManagerModuleProxy(*MFAM); });
+    FAM.registerPass(
+        [&] { return MachineFunctionAnalysisManagerFunctionProxy(*MFAM); });
    MFAM->registerPass(
        [&] { return ModuleAnalysisManagerMachineFunctionProxy(MAM); });
    MFAM->registerPass(
@@ -2023,7 +2025,7 @@ Error PassBuilder::parsePassPipeline(ModulePassManager &MPM,
                                 std::move(*Pipeline)}}}};
    } else if (isMachineFunctionPassName(
                   FirstName, MachineFunctionPipelineParsingCallbacks)) {
-      Pipeline = {{"machine-function", std::move(*Pipeline)}};
+      Pipeline = {{"function", {{"machine-function", std::move(*Pipeline)}}}};
    } else {
      for (auto &C : TopLevelPipelineParsingCallbacks)
        if (C(MPM, *Pipeline))
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -254,6 +254,9 @@ FUNCTION_ANALYSIS("demanded-bits", DemandedBitsAnalysis())
 FUNCTION_ANALYSIS("domfrontier", DominanceFrontierAnalysis())
 FUNCTION_ANALYSIS("domtree", DominatorTreeAnalysis())
 FUNCTION_ANALYSIS("func-properties", FunctionPropertiesAnalysis())
+FUNCTION_ANALYSIS(
+    "machine-function-info",
+    MachineFunctionAnalysis(static_cast<const LLVMTargetMachine *>(TM)))
 FUNCTION_ANALYSIS("gc-function", GCFunctionAnalysis())
 FUNCTION_ANALYSIS("inliner-size-estimator", InlineSizeEstimatorAnalysis())
 FUNCTION_ANALYSIS("lazy-value-info", LazyValueAnalysis())
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -297,14 +297,6 @@ void unwrapAndPrint(raw_ostream &OS, Any IR) {
    auto *M = unwrapModule(IR);
    assert(M && "should have unwrapped module");
    printIR(OS, M);
-
-    if (const auto *MF = unwrapIR<MachineFunction>(IR)) {
-      auto &MMI = MF->getMMI();
-      for (const auto &F : *M) {
-        if (auto *MF = MMI.getMachineFunction(F))
-          MF->print(OS);
-      }
-    }
    return;
  }

--- a/llvm/test/tools/llc/new-pm/pipeline.ll
+++ b/llvm/test/tools/llc/new-pm/pipeline.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=x86_64-pc-linux-gnu -enable-new-pm -print-pipeline-passes -filetype=null %s | FileCheck %s

 ; CHECK: require<profile-summary>,require<collector-metadata>
-; CHECK: MachineSanitizerBinaryMetadata,FreeMachineFunctionPass
+; CHECK: MachineSanitizerBinaryMetadata

--- a/llvm/test/tools/llc/new-pm/pipeline.mir
+++ b/llvm/test/tools/llc/new-pm/pipeline.mir
@@ -1,6 +1,6 @@
 # RUN: llc -mtriple=x86_64-pc-linux-gnu -x mir -passes=no-op-machine-function --print-pipeline-passes -filetype=null < %s | FileCheck %s --match-full-lines

-# CHECK: machine-function(no-op-machine-function),PrintMIRPreparePass,machine-function(print,FreeMachineFunctionPass)
+# CHECK: function(machine-function(no-op-machine-function)),PrintMIRPreparePass,function(machine-function(print))

 ---
 name: f
--- a/llvm/test/tools/llc/new-pm/start-stop.ll
+++ b/llvm/test/tools/llc/new-pm/start-stop.ll
@@ -2,4 +2,4 @@
 ; RUN: llc -mtriple=x86_64-pc-linux-gnu -enable-new-pm -print-pipeline-passes -start-before=mergeicmps -stop-after=gc-lowering -o /dev/null %s | FileCheck --match-full-lines %s --check-prefix=OBJ

 ; NULL: function(mergeicmps,expand-memcmp,gc-lowering)
-; OBJ: function(mergeicmps,expand-memcmp,gc-lowering),PrintMIRPreparePass,machine-function(print)
+; OBJ: function(mergeicmps,expand-memcmp,gc-lowering),PrintMIRPreparePass,function(machine-function(print),invalidate<machine-function-info>)
--- a/llvm/tools/llc/NewPMDriver.cpp
+++ b/llvm/tools/llc/NewPMDriver.cpp
@@ -16,7 +16,6 @@
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/CodeGen/CommandFlags.h"
-#include "llvm/CodeGen/FreeMachineFunction.h"
 #include "llvm/CodeGen/MIRParser/MIRParser.h"
 #include "llvm/CodeGen/MIRPrinter.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
@@ -120,11 +119,11 @@ int llvm::compileModuleWithNewPM(
  SI.registerCallbacks(PIC);
  registerCodeGenCallback(PIC, LLVMTM);

+  MachineFunctionAnalysisManager MFAM;
  LoopAnalysisManager LAM;
  FunctionAnalysisManager FAM;
  CGSCCAnalysisManager CGAM;
  ModuleAnalysisManager MAM;
-  MachineFunctionAnalysisManager MFAM;
  PassBuilder PB(Target.get(), PipelineTuningOptions(), std::nullopt, &PIC);
  PB.registerModuleAnalyses(MAM);
  PB.registerCGSCCAnalyses(CGAM);
@@ -137,6 +136,7 @@ int llvm::compileModuleWithNewPM(
  MAM.registerPass([&] { return MachineModuleAnalysis(MMI); });

  ModulePassManager MPM;
+  FunctionPassManager FPM;

  if (!PassPipeline.empty()) {
    // Construct a custom pass pipeline that starts after instruction
@@ -152,10 +152,10 @@ int llvm::compileModuleWithNewPM(
    MPM.addPass(PrintMIRPreparePass(*OS));
    MachineFunctionPassManager MFPM;
    MFPM.addPass(PrintMIRPass(*OS));
-    MFPM.addPass(FreeMachineFunctionPass());
-    MPM.addPass(createModuleToMachineFunctionPassAdaptor(std::move(MFPM)));
+    FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
+    MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));

-    if (MIR->parseMachineFunctions(*M, MMI))
+    if (MIR->parseMachineFunctions(*M, MAM))
      return 1;
  } else {
    ExitOnErr(LLVMTM.buildCodeGenPipeline(
--- a/llvm/unittests/CodeGen/PassManagerTest.cpp
+++ b/llvm/unittests/CodeGen/PassManagerTest.cpp
@@ -184,8 +184,8 @@ TEST_F(PassManagerTest, Basic) {

  MachineModuleInfo MMI(LLVMTM);

-  LoopAnalysisManager LAM;
  MachineFunctionAnalysisManager MFAM;
+  LoopAnalysisManager LAM;
  FunctionAnalysisManager FAM;
  CGSCCAnalysisManager CGAM;
  ModuleAnalysisManager MAM;
@@ -205,13 +205,17 @@ TEST_F(PassManagerTest, Basic) {
  std::vector<int> Counts;

  ModulePassManager MPM;
+  FunctionPassManager FPM;
  MachineFunctionPassManager MFPM;
  MPM.addPass(TestMachineModulePass(Count, Counts));
-  MPM.addPass(createModuleToMachineFunctionPassAdaptor(
+  FPM.addPass(createFunctionToMachineFunctionPassAdaptor(
      TestMachineFunctionPass(Count, Counts)));
+  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
  MPM.addPass(TestMachineModulePass(Count, Counts));
  MFPM.addPass(TestMachineFunctionPass(Count, Counts));
-  MPM.addPass(createModuleToMachineFunctionPassAdaptor(std::move(MFPM)));
+  FPM = FunctionPassManager();
+  FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
+  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));

  testing::internal::CaptureStderr();
  MPM.run(*M, MAM);
@@ -248,8 +252,10 @@ TEST_F(PassManagerTest, DiagnosticHandler) {
  ModulePassManager MPM;
  FunctionPassManager FPM;
  MachineFunctionPassManager MFPM;
+  MPM.addPass(RequireAnalysisPass<MachineModuleAnalysis, Module>());
  MFPM.addPass(ReportWarningPass());
-  MPM.addPass(createModuleToMachineFunctionPassAdaptor(std::move(MFPM)));
+  FPM.addPass(createFunctionToMachineFunctionPassAdaptor(std::move(MFPM)));
+  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
  testing::internal::CaptureStderr();
  MPM.run(*M, MAM);
  std::string Output = testing::internal::GetCapturedStderr();
--- a/llvm/unittests/MIR/CMakeLists.txt
+++ b/llvm/unittests/MIR/CMakeLists.txt
@@ -15,7 +15,6 @@ set(LLVM_LINK_COMPONENTS

 add_llvm_unittest(MIRTests
  MachineMetadata.cpp
-  PassBuilderCallbacksTest.cpp
  )

 target_link_libraries(MIRTests PRIVATE LLVMTestingSupport)
--- a/llvm/unittests/MIR/PassBuilderCallbacksTest.cpp
+++ b/llvm/unittests/MIR/PassBuilderCallbacksTest.cpp
@@ -1,575 +0,0 @@
-//===- unittests/MIR/PassBuilderCallbacksTest.cpp - PB Callback Tests -----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Analysis/CGSCCPassManager.h"
-#include "llvm/Analysis/LoopAnalysisManager.h"
-#include "llvm/CodeGen/FreeMachineFunction.h"
-#include "llvm/MC/TargetRegistry.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Testing/Support/Error.h"
-#include <functional>
-#include <gmock/gmock.h>
-#include <gtest/gtest.h>
-#include <llvm/ADT/Any.h>
-#include <llvm/AsmParser/Parser.h>
-#include <llvm/CodeGen/MIRParser/MIRParser.h>
-#include <llvm/CodeGen/MachineFunction.h>
-#include <llvm/CodeGen/MachineModuleInfo.h>
-#include <llvm/CodeGen/MachinePassManager.h>
-#include <llvm/IR/LLVMContext.h>
-#include <llvm/IR/PassInstrumentation.h>
-#include <llvm/IR/PassManager.h>
-#include <llvm/Passes/PassBuilder.h>
-#include <llvm/Support/Regex.h>
-#include <llvm/Support/SourceMgr.h>
-#include <llvm/Support/TargetSelect.h>
-
-using namespace llvm;
-
-namespace {
-using testing::_;
-using testing::AnyNumber;
-using testing::DoAll;
-using testing::Not;
-using testing::Return;
-using testing::WithArgs;
-
-StringRef MIRString = R"MIR(
--- |
-  define void @test() {
-    ret void
-  }
-...
---
-name:            test
-body:             |
-  bb.0 (%ir-block.0):
-    RET64
-...
-)MIR";
-
-/// Helper for HasName matcher that returns getName both for IRUnit and
-/// for IRUnit pointer wrapper into llvm::Any (wrapped by PassInstrumentation).
-template <typename IRUnitT> std::string getName(const IRUnitT &IR) {
-  return std::string(IR.getName());
-}
-
-template <> std::string getName(const StringRef &name) {
-  return std::string(name);
-}
-
-template <> std::string getName(const Any &WrappedIR) {
-  if (const auto *const *M = llvm::any_cast<const Module *>(&WrappedIR))
-    return (*M)->getName().str();
-  if (const auto *const *F = llvm::any_cast<const Function *>(&WrappedIR))
-    return (*F)->getName().str();
-  if (const auto *const *MF =
-          llvm::any_cast<const MachineFunction *>(&WrappedIR))
-    return (*MF)->getName().str();
-  return "<UNKNOWN>";
-}
-/// Define a custom matcher for objects which support a 'getName' method.
-///
-/// LLVM often has IR objects or analysis objects which expose a name
-/// and in tests it is convenient to match these by name for readability.
-/// Usually, this name is either a StringRef or a plain std::string. This
-/// matcher supports any type exposing a getName() method of this form whose
-/// return value is compatible with an std::ostream. For StringRef, this uses
-/// the shift operator defined above.
-///
-/// It should be used as:
-///
-///   HasName("my_function")
-///
-/// No namespace or other qualification is required.
-MATCHER_P(HasName, Name, "") {
-  *result_listener << "has name '" << getName(arg) << "'";
-  return Name == getName(arg);
-}
-
-MATCHER_P(HasNameRegex, Name, "") {
-  *result_listener << "has name '" << getName(arg) << "'";
-  llvm::Regex r(Name);
-  return r.match(getName(arg));
-}
-
-struct MockPassInstrumentationCallbacks {
-  MockPassInstrumentationCallbacks() {
-    ON_CALL(*this, runBeforePass(_, _)).WillByDefault(Return(true));
-  }
-  MOCK_METHOD2(runBeforePass, bool(StringRef PassID, llvm::Any));
-  MOCK_METHOD2(runBeforeSkippedPass, void(StringRef PassID, llvm::Any));
-  MOCK_METHOD2(runBeforeNonSkippedPass, void(StringRef PassID, llvm::Any));
-  MOCK_METHOD3(runAfterPass,
-               void(StringRef PassID, llvm::Any, const PreservedAnalyses &PA));
-  MOCK_METHOD2(runAfterPassInvalidated,
-               void(StringRef PassID, const PreservedAnalyses &PA));
-  MOCK_METHOD2(runBeforeAnalysis, void(StringRef PassID, llvm::Any));
-  MOCK_METHOD2(runAfterAnalysis, void(StringRef PassID, llvm::Any));
-
-  void registerPassInstrumentation(PassInstrumentationCallbacks &Callbacks) {
-    Callbacks.registerShouldRunOptionalPassCallback(
-        [this](StringRef P, llvm::Any IR) {
-          return this->runBeforePass(P, IR);
-        });
-    Callbacks.registerBeforeSkippedPassCallback(
-        [this](StringRef P, llvm::Any IR) {
-          this->runBeforeSkippedPass(P, IR);
-        });
-    Callbacks.registerBeforeNonSkippedPassCallback(
-        [this](StringRef P, llvm::Any IR) {
-          this->runBeforeNonSkippedPass(P, IR);
-        });
-    Callbacks.registerAfterPassCallback(
-        [this](StringRef P, llvm::Any IR, const PreservedAnalyses &PA) {
-          this->runAfterPass(P, IR, PA);
-        });
-    Callbacks.registerAfterPassInvalidatedCallback(
-        [this](StringRef P, const PreservedAnalyses &PA) {
-          this->runAfterPassInvalidated(P, PA);
-        });
-    Callbacks.registerBeforeAnalysisCallback([this](StringRef P, llvm::Any IR) {
-      return this->runBeforeAnalysis(P, IR);
-    });
-    Callbacks.registerAfterAnalysisCallback(
-        [this](StringRef P, llvm::Any IR) { this->runAfterAnalysis(P, IR); });
-  }
-
-  void ignoreNonMockPassInstrumentation(StringRef IRName) {
-    // Generic EXPECT_CALLs are needed to match instrumentation on unimportant
-    // parts of a pipeline that we do not care about (e.g. various passes added
-    // by default by PassBuilder - Verifier pass etc).
-    // Make sure to avoid ignoring Mock passes/analysis, we definitely want
-    // to check these explicitly.
-    EXPECT_CALL(*this,
-                runBeforePass(Not(HasNameRegex("Mock")), HasName(IRName)))
-        .Times(AnyNumber())
-        .WillRepeatedly(Return(false));
-    EXPECT_CALL(
-        *this, runBeforeSkippedPass(Not(HasNameRegex("Mock")), HasName(IRName)))
-        .Times(AnyNumber());
-    EXPECT_CALL(*this, runBeforeNonSkippedPass(Not(HasNameRegex("Mock")),
-                                               HasName(IRName)))
-        .Times(AnyNumber());
-    EXPECT_CALL(*this,
-                runAfterPass(Not(HasNameRegex("Mock")), HasName(IRName), _))
-        .Times(AnyNumber());
-    EXPECT_CALL(*this,
-                runBeforeAnalysis(Not(HasNameRegex("Mock")), HasName(IRName)))
-        .Times(AnyNumber());
-    EXPECT_CALL(*this,
-                runAfterAnalysis(Not(HasNameRegex("Mock")), HasName(IRName)))
-        .Times(AnyNumber());
-  }
-};
-
-template <typename DerivedT> class MockAnalysisHandleBase {
-public:
-  class Analysis : public AnalysisInfoMixin<Analysis> {
-    friend AnalysisInfoMixin<Analysis>;
-    friend MockAnalysisHandleBase;
-    static AnalysisKey Key;
-
-    DerivedT *Handle;
-
-    Analysis(DerivedT &Handle) : Handle(&Handle) {
-      static_assert(std::is_base_of<MockAnalysisHandleBase, DerivedT>::value,
-                    "Must pass the derived type to this template!");
-    }
-
-  public:
-    class Result {
-      friend MockAnalysisHandleBase;
-
-      DerivedT *Handle;
-
-      Result(DerivedT &Handle) : Handle(&Handle) {}
-
-    public:
-      // Forward invalidation events to the mock handle.
-      bool invalidate(MachineFunction &IR, const PreservedAnalyses &PA,
-                      MachineFunctionAnalysisManager::Invalidator &Inv) {
-        return Handle->invalidate(IR, PA, Inv);
-      }
-    };
-
-    Result run(MachineFunction &IR, MachineFunctionAnalysisManager &AM) {
-      return Handle->run(IR, AM);
-    }
-  };
-
-  Analysis getAnalysis() { return Analysis(static_cast<DerivedT &>(*this)); }
-  typename Analysis::Result getResult() {
-    return typename Analysis::Result(static_cast<DerivedT &>(*this));
-  }
-  static StringRef getName() { return llvm::getTypeName<DerivedT>(); }
-
-protected:
-  // FIXME: MSVC seems unable to handle a lambda argument to Invoke from within
-  // the template, so we use a boring static function.
-  static bool
-  invalidateCallback(MachineFunction &IR, const PreservedAnalyses &PA,
-                     MachineFunctionAnalysisManager::Invalidator &Inv) {
-    auto PAC = PA.template getChecker<Analysis>();
-    return !PAC.preserved() &&
-           !PAC.template preservedSet<AllAnalysesOn<MachineFunction>>();
-  }
-
-  /// Derived classes should call this in their constructor to set up default
-  /// mock actions. (We can't do this in our constructor because this has to
-  /// run after the DerivedT is constructed.)
-  void setDefaults() {
-    ON_CALL(static_cast<DerivedT &>(*this), run(_, _))
-        .WillByDefault(Return(this->getResult()));
-    ON_CALL(static_cast<DerivedT &>(*this), invalidate(_, _, _))
-        .WillByDefault(&invalidateCallback);
-  }
-};
-
-template <typename DerivedT> class MockPassHandleBase {
-public:
-  class Pass : public PassInfoMixin<Pass> {
-    friend MockPassHandleBase;
-
-    DerivedT *Handle;
-
-    Pass(DerivedT &Handle) : Handle(&Handle) {
-      static_assert(std::is_base_of<MockPassHandleBase, DerivedT>::value,
-                    "Must pass the derived type to this template!");
-    }
-
-  public:
-    PreservedAnalyses run(MachineFunction &IR,
-                          MachineFunctionAnalysisManager &AM) {
-      return Handle->run(IR, AM);
-    }
-  };
-
-  static StringRef getName() { return llvm::getTypeName<DerivedT>(); }
-
-  Pass getPass() { return Pass(static_cast<DerivedT &>(*this)); }
-
-protected:
-  /// Derived classes should call this in their constructor to set up default
-  /// mock actions. (We can't do this in our constructor because this has to
-  /// run after the DerivedT is constructed.)
-  void setDefaults() {
-    ON_CALL(static_cast<DerivedT &>(*this), run(_, _))
-        .WillByDefault(Return(PreservedAnalyses::all()));
-  }
-};
-
-struct MockAnalysisHandle : public MockAnalysisHandleBase<MockAnalysisHandle> {
-  MOCK_METHOD2(run, Analysis::Result(MachineFunction &,
-                                     MachineFunctionAnalysisManager &));
-
-  MOCK_METHOD3(invalidate, bool(MachineFunction &, const PreservedAnalyses &,
-                                MachineFunctionAnalysisManager::Invalidator &));
-
-  MockAnalysisHandle() { setDefaults(); }
-};
-
-template <typename DerivedT>
-AnalysisKey MockAnalysisHandleBase<DerivedT>::Analysis::Key;
-
-class MockPassHandle : public MockPassHandleBase<MockPassHandle> {
-public:
-  MOCK_METHOD2(run, PreservedAnalyses(MachineFunction &,
-                                      MachineFunctionAnalysisManager &));
-
-  MockPassHandle() { setDefaults(); }
-};
-
-class MachineFunctionCallbacksTest : public testing::Test {
-protected:
-  static void SetUpTestCase() {
-    InitializeAllTargetInfos();
-    InitializeAllTargets();
-    InitializeAllTargetMCs();
-  }
-
-  LLVMContext Context;
-
-  std::unique_ptr<LLVMTargetMachine> TM;
-  std::unique_ptr<MachineModuleInfo> MMI;
-
-  std::unique_ptr<Module> M;
-
-  PassInstrumentationCallbacks PIC;
-  std::unique_ptr<PassBuilder> PB;
-  ModulePassManager MPM;
-  MachineFunctionAnalysisManager MFAM;
-  LoopAnalysisManager LAM;
-  FunctionAnalysisManager FAM;
-  CGSCCAnalysisManager CGAM;
-  ModuleAnalysisManager MAM;
-
-  MockPassInstrumentationCallbacks CallbacksHandle;
-  MockPassHandle PassHandle;
-  MockAnalysisHandle AnalysisHandle;
-
-  static std::unique_ptr<Module> parseMIR(StringRef MIRCode,
-                                          LLVMContext &Context,
-                                          TargetMachine &TM,
-                                          MachineModuleInfo &MMI) {
-    SMDiagnostic Diagnostic;
-    std::unique_ptr<MemoryBuffer> MBuffer = MemoryBuffer::getMemBuffer(MIRCode);
-    std::unique_ptr<MIRParser> MIR =
-        createMIRParser(std::move(MBuffer), Context);
-    assert(MIR);
-
-    std::unique_ptr<Module> Mod = MIR->parseIRModule();
-    assert(Mod);
-
-    // Module identifier is used in tests below.
-    Mod->setModuleIdentifier("module");
-    Mod->setDataLayout(TM.createDataLayout());
-
-    [[maybe_unused]] bool Ret = MIR->parseMachineFunctions(*Mod, MMI);
-    assert(!Ret);
-
-    return Mod;
-  }
-
-  static PreservedAnalyses
-  getAnalysisResult(MachineFunction &U, MachineFunctionAnalysisManager &MFAM) {
-    MFAM.getResult<MockAnalysisHandle::Analysis>(U);
-    return PreservedAnalyses::all();
-  }
-
-  void SetUp() override {
-    std::string Error;
-    auto TripleName = "x86_64-pc-linux-gnu";
-    auto *T = TargetRegistry::lookupTarget(TripleName, Error);
-    if (!T)
-      GTEST_SKIP();
-    TM = std::unique_ptr<LLVMTargetMachine>(
-        static_cast<LLVMTargetMachine *>(T->createTargetMachine(
-            TripleName, "", "", TargetOptions(), std::nullopt)));
-    if (!TM)
-      GTEST_SKIP();
-
-    MMI = std::make_unique<MachineModuleInfo>(TM.get());
-    M = parseMIR(MIRString, Context, *TM, *MMI);
-    PB = std::make_unique<PassBuilder>(TM.get(), PipelineTuningOptions(),
-                                       std::nullopt, &PIC);
-
-    /// Register a callback for analysis registration.
-    ///
-    /// The callback is a function taking a reference to an AnalyisManager
-    /// object. When called, the callee gets to register its own analyses with
-    /// this PassBuilder instance.
-    PB->registerAnalysisRegistrationCallback(
-        [this](MachineFunctionAnalysisManager &AM) {
-          // Register our mock analysis
-          AM.registerPass([this] { return AnalysisHandle.getAnalysis(); });
-        });
-
-    /// Register a callback for pipeline parsing.
-    ///
-    /// During parsing of a textual pipeline, the PassBuilder will call these
-    /// callbacks for each encountered pass name that it does not know. This
-    /// includes both simple pass names as well as names of sub-pipelines. In
-    /// the latter case, the InnerPipeline is not empty.
-    PB->registerPipelineParsingCallback(
-        [this](StringRef Name, MachineFunctionPassManager &PM,
-               ArrayRef<PassBuilder::PipelineElement> InnerPipeline) {
-          if (parseAnalysisUtilityPasses<MockAnalysisHandle::Analysis>(
-                  "test-analysis", Name, PM))
-            return true;
-
-          /// Parse the name of our pass mock handle
-          if (Name == "test-transform") {
-            PM.addPass(PassHandle.getPass());
-            return true;
-          }
-          return false;
-        });
-
-    /// Register builtin analyses and cross-register the analysis proxies
-    PB->registerModuleAnalyses(MAM);
-    PB->registerCGSCCAnalyses(CGAM);
-    PB->registerFunctionAnalyses(FAM);
-    PB->registerLoopAnalyses(LAM);
-    PB->registerMachineFunctionAnalyses(MFAM);
-    PB->crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
-    MAM.registerPass([&] { return MachineModuleAnalysis(*MMI); });
-  }
-};
-
-TEST_F(MachineFunctionCallbacksTest, Passes) {
-  EXPECT_CALL(AnalysisHandle, run(HasName("test"), _));
-  EXPECT_CALL(PassHandle, run(HasName("test"), _)).WillOnce(&getAnalysisResult);
-
-  StringRef PipelineText = "test-transform";
-  ASSERT_THAT_ERROR(PB->parsePassPipeline(MPM, PipelineText), Succeeded())
-      << "Pipeline was: " << PipelineText;
-  MPM.run(*M, MAM);
-}
-
-TEST_F(MachineFunctionCallbacksTest, InstrumentedPasses) {
-  CallbacksHandle.registerPassInstrumentation(PIC);
-  // Non-mock instrumentation not specifically mentioned below can be ignored.
-  CallbacksHandle.ignoreNonMockPassInstrumentation("test");
-  CallbacksHandle.ignoreNonMockPassInstrumentation("module");
-
-  // PassInstrumentation calls should happen in-sequence, in the same order
-  // as passes/analyses are scheduled.
-  ::testing::Sequence PISequence;
-  EXPECT_CALL(CallbacksHandle,
-              runBeforePass(HasNameRegex("MockPassHandle"), HasName("test")))
-      .InSequence(PISequence)
-      .WillOnce(Return(true));
-  EXPECT_CALL(
-      CallbacksHandle,
-      runBeforeNonSkippedPass(HasNameRegex("MockPassHandle"), HasName("test")))
-      .InSequence(PISequence);
-  EXPECT_CALL(
-      CallbacksHandle,
-      runBeforeAnalysis(HasNameRegex("MockAnalysisHandle"), HasName("test")))
-      .InSequence(PISequence);
-  EXPECT_CALL(
-      CallbacksHandle,
-      runAfterAnalysis(HasNameRegex("MockAnalysisHandle"), HasName("test")))
-      .InSequence(PISequence);
-  EXPECT_CALL(CallbacksHandle,
-              runAfterPass(HasNameRegex("MockPassHandle"), HasName("test"), _))
-      .InSequence(PISequence);
-  EXPECT_CALL(
-      CallbacksHandle,
-      runBeforeSkippedPass(HasNameRegex("MockPassHandle"), HasName("test")))
-      .Times(0);
-
-  EXPECT_CALL(AnalysisHandle, run(HasName("test"), _));
-  EXPECT_CALL(PassHandle, run(HasName("test"), _)).WillOnce(&getAnalysisResult);
-
-  StringRef PipelineText = "test-transform";
-  ASSERT_THAT_ERROR(PB->parsePassPipeline(MPM, PipelineText), Succeeded())
-      << "Pipeline was: " << PipelineText;
-  MPM.run(*M, MAM);
-}
-
-TEST_F(MachineFunctionCallbacksTest, InstrumentedSkippedPasses) {
-  CallbacksHandle.registerPassInstrumentation(PIC);
-  // Non-mock instrumentation run here can safely be ignored.
-  CallbacksHandle.ignoreNonMockPassInstrumentation("test");
-  CallbacksHandle.ignoreNonMockPassInstrumentation("module");
-
-  // Skip the pass by returning false.
-  EXPECT_CALL(CallbacksHandle,
-              runBeforePass(HasNameRegex("MockPassHandle"), HasName("test")))
-      .WillOnce(Return(false));
-
-  EXPECT_CALL(
-      CallbacksHandle,
-      runBeforeSkippedPass(HasNameRegex("MockPassHandle"), HasName("test")))
-      .Times(1);
-
-  EXPECT_CALL(AnalysisHandle, run(HasName("test"), _)).Times(0);
-  EXPECT_CALL(PassHandle, run(HasName("test"), _)).Times(0);
-
-  // As the pass is skipped there is no afterPass, beforeAnalysis/afterAnalysis
-  // as well.
-  EXPECT_CALL(CallbacksHandle,
-              runBeforeNonSkippedPass(HasNameRegex("MockPassHandle"), _))
-      .Times(0);
-  EXPECT_CALL(CallbacksHandle,
-              runAfterPass(HasNameRegex("MockPassHandle"), _, _))
-      .Times(0);
-  EXPECT_CALL(CallbacksHandle,
-              runAfterPassInvalidated(HasNameRegex("MockPassHandle"), _))
-      .Times(0);
-  EXPECT_CALL(CallbacksHandle,
-              runAfterPass(HasNameRegex("MockPassHandle"), _, _))
-      .Times(0);
-  EXPECT_CALL(CallbacksHandle,
-              runBeforeAnalysis(HasNameRegex("MockAnalysisHandle"), _))
-      .Times(0);
-  EXPECT_CALL(CallbacksHandle,
-              runAfterAnalysis(HasNameRegex("MockAnalysisHandle"), _))
-      .Times(0);
-
-  StringRef PipelineText = "test-transform";
-  ASSERT_THAT_ERROR(PB->parsePassPipeline(MPM, PipelineText), Succeeded())
-      << "Pipeline was: " << PipelineText;
-  MPM.run(*M, MAM);
-}
-
-// Check that the Module -> MachineFunction adaptor properly calls
-// runAfterPassInvalidated.
-TEST_F(MachineFunctionCallbacksTest, InstrumentedFreeMFPass) {
-  CallbacksHandle.registerPassInstrumentation(PIC);
-  // Non-mock instrumentation run here can safely be ignored.
-  CallbacksHandle.ignoreNonMockPassInstrumentation("test");
-  CallbacksHandle.ignoreNonMockPassInstrumentation("module");
-
-  ::testing::Sequence PISequence;
-  EXPECT_CALL(
-      CallbacksHandle,
-      runBeforePass(HasNameRegex("FreeMachineFunctionPass"), HasName("test")))
-      .InSequence(PISequence)
-      .WillOnce(Return(true));
-  EXPECT_CALL(CallbacksHandle,
-              runBeforeNonSkippedPass(HasNameRegex("FreeMachineFunctionPass"),
-                                      HasName("test")))
-      .InSequence(PISequence);
-  EXPECT_CALL(CallbacksHandle, runAfterPassInvalidated(
-                                   HasNameRegex("FreeMachineFunctionPass"), _))
-      .InSequence(PISequence);
-
-  // runAfterPass should not be called since the MachineFunction is no longer
-  // valid after FreeMachineFunctionPass.
-  EXPECT_CALL(CallbacksHandle,
-              runAfterPass(HasNameRegex("FreeMachineFunctionPass"), _, _))
-      .Times(0);
-
-  MPM.addPass(
-      createModuleToMachineFunctionPassAdaptor(FreeMachineFunctionPass()));
-  MPM.run(*M, MAM);
-}
-
-// Check that the Module -> MachineFunction adaptor and MachineFunction pass
-// manager properly call runAfterPassInvalidated.
-TEST_F(MachineFunctionCallbacksTest, InstrumentedFreeMFPass2) {
-  CallbacksHandle.registerPassInstrumentation(PIC);
-  // Non-mock instrumentation run here can safely be ignored.
-  CallbacksHandle.ignoreNonMockPassInstrumentation("test");
-  CallbacksHandle.ignoreNonMockPassInstrumentation("module");
-
-  ::testing::Sequence PISequence;
-  EXPECT_CALL(
-      CallbacksHandle,
-      runBeforePass(HasNameRegex("FreeMachineFunctionPass"), HasName("test")))
-      .InSequence(PISequence)
-      .WillOnce(Return(true));
-  EXPECT_CALL(CallbacksHandle,
-              runBeforeNonSkippedPass(HasNameRegex("FreeMachineFunctionPass"),
-                                      HasName("test")))
-      .InSequence(PISequence);
-  EXPECT_CALL(CallbacksHandle, runAfterPassInvalidated(
-                                   HasNameRegex("FreeMachineFunctionPass"), _))
-      .InSequence(PISequence);
-  EXPECT_CALL(CallbacksHandle,
-              runAfterPassInvalidated(HasNameRegex("PassManager"), _))
-      .InSequence(PISequence);
-
-  // runAfterPass should not be called since the MachineFunction is no longer
-  // valid after FreeMachineFunctionPass.
-  EXPECT_CALL(CallbacksHandle,
-              runAfterPass(HasNameRegex("FreeMachineFunctionPass"), _, _))
-      .Times(0);
-  EXPECT_CALL(CallbacksHandle, runAfterPass(HasNameRegex("PassManager"), _, _))
-      .Times(0);
-
-  MachineFunctionPassManager MFPM;
-  MFPM.addPass(FreeMachineFunctionPass());
-  MPM.addPass(createModuleToMachineFunctionPassAdaptor(std::move(MFPM)));
-  MPM.run(*M, MAM);
-}
-
-} // end anonymous namespace