[RISCV] Xqccmp Code Generation (#128815)

This adds support for Xqccmp to the following passes: - Prolog Epilog Insertion - reusing much of the existing push/pop logic, but extending it to cope with frame pointers and reorder the CFI information correctly. - Move Merger - extending it to support the `qc.` variants of the double-move instructions. - Push/Pop Optimizer - extending it to support the `qc.` variants of the pop instructions. The testing is based on existing Zcmp tests, but I have put them in separate files as some of the Zcmp tests were getting quite long.
2025-03-05 10:59:45 -08:00
parent 9f0a912a46
commit e49180d84c
14 changed files with 5641 additions and 43 deletions
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -109,8 +109,9 @@ Changes to the RISC-V Backend

 * Adds experimental assembler support for the Qualcomm uC 'Xqcilia` (Large Immediate Arithmetic)
  extension.
-* Adds experimental assembler support for the Qualcomm 'Xqccmp' extension, which
-  is a frame-pointer convention compatible version of Zcmp.
+* Adds experimental assembler and code generation support for the Qualcomm
+  'Xqccmp' extension, which is a frame-pointer convention compatible version of
+  Zcmp.
 * Added non-quadratic ``log-vrgather`` cost model for ``vrgather.vv`` instruction

 Changes to the WebAssembly Backend
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -750,6 +750,54 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
  }
 }

+static bool isPush(unsigned Opcode) {
+  switch (Opcode) {
+  case RISCV::CM_PUSH:
+  case RISCV::QC_CM_PUSH:
+  case RISCV::QC_CM_PUSHFP:
+    return true;
+  default:
+    return false;
+  }
+}
+
+static bool isPop(unsigned Opcode) {
+  // There are other pops but these are the only ones introduced during this
+  // pass.
+  switch (Opcode) {
+  case RISCV::CM_POP:
+  case RISCV::QC_CM_POP:
+    return true;
+  default:
+    return false;
+  }
+}
+
+static unsigned getPushOpcode(RISCVMachineFunctionInfo::PushPopKind Kind,
+                              bool HasFP) {
+  switch (Kind) {
+  case RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp:
+    return RISCV::CM_PUSH;
+  case RISCVMachineFunctionInfo::PushPopKind::VendorXqccmp:
+    return HasFP ? RISCV::QC_CM_PUSHFP : RISCV::QC_CM_PUSH;
+  default:
+    llvm_unreachable("Unhandled PushPopKind");
+  }
+}
+
+static unsigned getPopOpcode(RISCVMachineFunctionInfo::PushPopKind Kind) {
+  // There are other pops but they are introduced later by the Push/Pop
+  // Optimizer.
+  switch (Kind) {
+  case RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp:
+    return RISCV::CM_POP;
+  case RISCVMachineFunctionInfo::PushPopKind::VendorXqccmp:
+    return RISCV::QC_CM_POP;
+  default:
+    llvm_unreachable("Unhandled PushPopKind");
+  }
+}
+
 void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
                                      MachineBasicBlock &MBB) const {
  MachineFrameInfo &MFI = MF.getFrameInfo();
@@ -849,7 +897,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
  }

  if (RVFI->isPushable(MF) && FirstFrameSetup != MBB.end() &&
-      FirstFrameSetup->getOpcode() == RISCV::CM_PUSH) {
+      isPush(FirstFrameSetup->getOpcode())) {
    // Use available stack adjustment in push instruction to allocate additional
    // stack space. Align the stack size down to a multiple of 16. This is
    // needed for RVE.
@@ -900,9 +948,15 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
    // The frame pointer does need to be reserved from register allocation.
    assert(MF.getRegInfo().isReserved(FPReg) && "FP not reserved");

-    RI->adjustReg(MBB, MBBI, DL, FPReg, SPReg,
-                  StackOffset::getFixed(RealStackSize - RVFI->getVarArgsSaveSize()),
-                  MachineInstr::FrameSetup, getStackAlign());
+    // Xqccmp with hasFP will update FP using `qc.cm.pushfp`, so we don't need
+    // to update it again, but we do need to emit the `.cfi_def_cfa` below.
+    if (RVFI->getPushPopKind(MF) !=
+        RISCVMachineFunctionInfo::PushPopKind::VendorXqccmp) {
+      RI->adjustReg(
+          MBB, MBBI, DL, FPReg, SPReg,
+          StackOffset::getFixed(RealStackSize - RVFI->getVarArgsSaveSize()),
+          MachineInstr::FrameSetup, getStackAlign());
+    }

    // Emit ".cfi_def_cfa $fp, RVFI->getVarArgsSaveSize()"
    unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
@@ -1160,9 +1214,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
  // Recover callee-saved registers.
  emitCFIForCSI<CFIRestoreRegisterEmitter>(MBB, MBBI, getUnmanagedCSI(MF, CSI));

-  bool ApplyPop = RVFI->isPushable(MF) && MBBI != MBB.end() &&
-                  MBBI->getOpcode() == RISCV::CM_POP;
-  if (ApplyPop) {
+  if (RVFI->isPushable(MF) && MBBI != MBB.end() && isPop(MBBI->getOpcode())) {
    // Use available stack adjustment in pop instruction to deallocate stack
    // space. Align the stack size down to a multiple of 16. This is needed for
    // RVE.
@@ -1781,7 +1833,8 @@ bool RISCVFrameLowering::assignCalleeSavedSpillSlots(

      if (FII != std::end(FixedCSRFIMap)) {
        int64_t Offset;
-        if (RVFI->isPushable(MF))
+        if (RVFI->getPushPopKind(MF) ==
+            RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp)
          Offset = -int64_t(RVFI->getRVPushRegs() - RegNum) * Size;
        else
          Offset = -int64_t(RegNum + 1) * Size;
@@ -1845,9 +1898,10 @@ bool RISCVFrameLowering::spillCalleeSavedRegisters(
    unsigned PushedRegNum = RVFI->getRVPushRegs();
    if (PushedRegNum > 0) {
      // Use encoded number to represent registers to spill.
+      unsigned Opcode = getPushOpcode(RVFI->getPushPopKind(*MF), hasFP(*MF));
      unsigned RegEnc = RISCVZC::encodeRlistNumRegs(PushedRegNum);
      MachineInstrBuilder PushBuilder =
-          BuildMI(MBB, MI, DL, TII.get(RISCV::CM_PUSH))
+          BuildMI(MBB, MI, DL, TII.get(Opcode))
              .setMIFlag(MachineInstr::FrameSetup);
      PushBuilder.addImm(RegEnc);
      PushBuilder.addImm(0);
@@ -2000,9 +2054,10 @@ bool RISCVFrameLowering::restoreCalleeSavedRegisters(
  if (RVFI->isPushable(*MF)) {
    unsigned PushedRegNum = RVFI->getRVPushRegs();
    if (PushedRegNum > 0) {
+      unsigned Opcode = getPopOpcode(RVFI->getPushPopKind(*MF));
      unsigned RegEnc = RISCVZC::encodeRlistNumRegs(PushedRegNum);
      MachineInstrBuilder PopBuilder =
-          BuildMI(MBB, MI, DL, TII.get(RISCV::CM_POP))
+          BuildMI(MBB, MI, DL, TII.get(Opcode))
              .setMIFlag(MachineInstr::FrameDestroy);
      // Use encoded number to represent registers to restore.
      PopBuilder.addImm(RegEnc);
--- a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp
@@ -60,6 +60,27 @@ void yaml::RISCVMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
  MappingTraits<RISCVMachineFunctionInfo>::mapping(YamlIO, *this);
 }

+RISCVMachineFunctionInfo::PushPopKind
+RISCVMachineFunctionInfo::getPushPopKind(const MachineFunction &MF) const {
+  // We cannot use fixed locations for the callee saved spill slots if the
+  // function uses a varargs save area.
+  // TODO: Use a separate placement for vararg registers to enable Zcmp.
+  if (VarArgsSaveSize != 0)
+    return PushPopKind::None;
+
+  // Zcmp is not compatible with the frame pointer convention.
+  if (MF.getSubtarget<RISCVSubtarget>().hasStdExtZcmp() &&
+      !MF.getTarget().Options.DisableFramePointerElim(MF))
+    return PushPopKind::StdExtZcmp;
+
+  // Xqccmp is Zcmp but has a push order compatible with the frame-pointer
+  // convention.
+  if (MF.getSubtarget<RISCVSubtarget>().hasVendorXqccmp())
+    return PushPopKind::VendorXqccmp;
+
+  return PushPopKind::None;
+}
+
 void RISCVMachineFunctionInfo::initializeBaseYamlFields(
    const yaml::RISCVMachineFunctionInfo &YamlMFI) {
  VarArgsFrameIndex = YamlMFI.VarArgsFrameIndex;
--- a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
@@ -136,13 +136,12 @@ public:
  unsigned getCalleeSavedStackSize() const { return CalleeSavedStackSize; }
  void setCalleeSavedStackSize(unsigned Size) { CalleeSavedStackSize = Size; }

+  enum class PushPopKind { None = 0, StdExtZcmp, VendorXqccmp };
+
+  PushPopKind getPushPopKind(const MachineFunction &MF) const;
+
  bool isPushable(const MachineFunction &MF) const {
-    // We cannot use fixed locations for the callee saved spill slots if the
-    // function uses a varargs save area.
-    // TODO: Use a separate placement for vararg registers to enable Zcmp.
-    return MF.getSubtarget<RISCVSubtarget>().hasStdExtZcmp() &&
-           !MF.getTarget().Options.DisableFramePointerElim(MF) &&
-           VarArgsSaveSize == 0;
+    return getPushPopKind(MF) != PushPopKind::None;
  }

  unsigned getRVPushRegs() const { return RVPushRegs; }
--- a/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp
@@ -9,10 +9,12 @@
 // This file contains a pass that performs move related peephole optimizations
 // as Zcmp has specified. This pass should be run after register allocation.
 //
+// This pass also supports Xqccmp, which has identical instructions.
+//
 //===----------------------------------------------------------------------===//

 #include "RISCVInstrInfo.h"
-#include "RISCVMachineFunctionInfo.h"
+#include "RISCVSubtarget.h"

 using namespace llvm;

@@ -43,7 +45,7 @@ struct RISCVMoveMerge : public MachineFunctionPass {
  MachineBasicBlock::iterator
  findMatchingInst(MachineBasicBlock::iterator &MBBI, unsigned InstOpcode,
                   const DestSourcePair &RegPair);
-  bool mergeMoveSARegPair(MachineBasicBlock &MBB);
+  bool mergeMoveSARegPair(const RISCVSubtarget &STI, MachineBasicBlock &MBB);
  bool runOnMachineFunction(MachineFunction &Fn) override;

  StringRef getPassName() const override { return RISCV_MOVE_MERGE_NAME; }
@@ -56,6 +58,46 @@ char RISCVMoveMerge::ID = 0;
 INITIALIZE_PASS(RISCVMoveMerge, "riscv-move-merge", RISCV_MOVE_MERGE_NAME,
                false, false)

+static bool isMoveFromAToS(unsigned Opcode) {
+  switch (Opcode) {
+  case RISCV::CM_MVA01S:
+  case RISCV::QC_CM_MVA01S:
+    return true;
+  default:
+    return false;
+  }
+}
+
+static unsigned getMoveFromAToSOpcode(const RISCVSubtarget &STI) {
+  if (STI.hasStdExtZcmp())
+    return RISCV::CM_MVA01S;
+
+  if (STI.hasVendorXqccmp())
+    return RISCV::QC_CM_MVA01S;
+
+  llvm_unreachable("Unhandled subtarget with paired A to S move.");
+}
+
+static bool isMoveFromSToA(unsigned Opcode) {
+  switch (Opcode) {
+  case RISCV::CM_MVSA01:
+  case RISCV::QC_CM_MVSA01:
+    return true;
+  default:
+    return false;
+  }
+}
+
+static unsigned getMoveFromSToAOpcode(const RISCVSubtarget &STI) {
+  if (STI.hasStdExtZcmp())
+    return RISCV::CM_MVSA01;
+
+  if (STI.hasVendorXqccmp())
+    return RISCV::QC_CM_MVSA01;
+
+  llvm_unreachable("Unhandled subtarget with paired S to A move");
+}
+
 // Check if registers meet CM.MVA01S constraints.
 bool RISCVMoveMerge::isCandidateToMergeMVA01S(const DestSourcePair &RegPair) {
  Register Destination = RegPair.Destination->getReg();
@@ -87,7 +129,7 @@ RISCVMoveMerge::mergePairedInsns(MachineBasicBlock::iterator I,
  MachineBasicBlock::iterator NextI = next_nodbg(I, E);
  DestSourcePair FirstPair = TII->isCopyInstrImpl(*I).value();
  DestSourcePair PairedRegs = TII->isCopyInstrImpl(*Paired).value();
-  Register ARegInFirstPair = Opcode == RISCV::CM_MVA01S
+  Register ARegInFirstPair = isMoveFromAToS(Opcode)
                                 ? FirstPair.Destination->getReg()
                                 : FirstPair.Source->getReg();

@@ -104,7 +146,7 @@ RISCVMoveMerge::mergePairedInsns(MachineBasicBlock::iterator I,
  //   mv a0, s2
  //   mv a1, s1    =>  cm.mva01s s2,s1
  bool StartWithX10 = ARegInFirstPair == RISCV::X10;
-  if (Opcode == RISCV::CM_MVA01S) {
+  if (isMoveFromAToS(Opcode)) {
    Sreg1 = StartWithX10 ? FirstPair.Source : PairedRegs.Source;
    Sreg2 = StartWithX10 ? PairedRegs.Source : FirstPair.Source;
  } else {
@@ -139,8 +181,7 @@ RISCVMoveMerge::findMatchingInst(MachineBasicBlock::iterator &MBBI,
      Register SourceReg = SecondPair->Source->getReg();
      Register DestReg = SecondPair->Destination->getReg();

-      if (InstOpcode == RISCV::CM_MVA01S &&
-          isCandidateToMergeMVA01S(*SecondPair)) {
+      if (isMoveFromAToS(InstOpcode) && isCandidateToMergeMVA01S(*SecondPair)) {
        // If register pair is valid and destination registers are different.
        if ((RegPair.Destination->getReg() == DestReg))
          return E;
@@ -154,7 +195,7 @@ RISCVMoveMerge::findMatchingInst(MachineBasicBlock::iterator &MBBI,
          return E;

        return I;
-      } else if (InstOpcode == RISCV::CM_MVSA01 &&
+      } else if (isMoveFromSToA(InstOpcode) &&
                 isCandidateToMergeMVSA01(*SecondPair)) {
        if ((RegPair.Source->getReg() == SourceReg) ||
            (RegPair.Destination->getReg() == DestReg))
@@ -176,7 +217,8 @@ RISCVMoveMerge::findMatchingInst(MachineBasicBlock::iterator &MBBI,

 // Finds instructions, which could be represented as C.MV instructions and
 // merged into CM.MVA01S or CM.MVSA01.
-bool RISCVMoveMerge::mergeMoveSARegPair(MachineBasicBlock &MBB) {
+bool RISCVMoveMerge::mergeMoveSARegPair(const RISCVSubtarget &STI,
+                                        MachineBasicBlock &MBB) {
  bool Modified = false;

  for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
@@ -188,9 +230,9 @@ bool RISCVMoveMerge::mergeMoveSARegPair(MachineBasicBlock &MBB) {
      unsigned Opcode = 0;

      if (isCandidateToMergeMVA01S(*RegPair))
-        Opcode = RISCV::CM_MVA01S;
+        Opcode = getMoveFromAToSOpcode(STI);
      else if (isCandidateToMergeMVSA01(*RegPair))
-        Opcode = RISCV::CM_MVSA01;
+        Opcode = getMoveFromSToAOpcode(STI);
      else {
        ++MBBI;
        continue;
@@ -215,7 +257,7 @@ bool RISCVMoveMerge::runOnMachineFunction(MachineFunction &Fn) {
    return false;

  const RISCVSubtarget *Subtarget = &Fn.getSubtarget<RISCVSubtarget>();
-  if (!Subtarget->hasStdExtZcmp())
+  if (!(Subtarget->hasStdExtZcmp() || Subtarget->hasVendorXqccmp()))
    return false;

  TII = Subtarget->getInstrInfo();
@@ -227,7 +269,7 @@ bool RISCVMoveMerge::runOnMachineFunction(MachineFunction &Fn) {
  UsedRegUnits.init(*TRI);
  bool Modified = false;
  for (auto &MBB : Fn)
-    Modified |= mergeMoveSARegPair(MBB);
+    Modified |= mergeMoveSARegPair(*Subtarget, MBB);
  return Modified;
 }

--- a/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp
@@ -13,6 +13,7 @@

 #include "RISCVInstrInfo.h"
 #include "RISCVMachineFunctionInfo.h"
+#include "llvm/CodeGen/MachineInstr.h"

 using namespace llvm;

@@ -45,11 +46,34 @@ char RISCVPushPopOpt::ID = 0;
 INITIALIZE_PASS(RISCVPushPopOpt, "riscv-push-pop-opt", RISCV_PUSH_POP_OPT_NAME,
                false, false)

+static bool isPop(unsigned Opcode) {
+  switch (Opcode) {
+  case RISCV::CM_POP:
+  case RISCV::QC_CM_POP:
+    return true;
+  default:
+    return false;
+  }
+}
+
+static unsigned getPopRetOpcode(unsigned PopOpcode, bool IsReturnZero) {
+  assert(isPop(PopOpcode) && "Unexpected Pop Opcode");
+
+  switch (PopOpcode) {
+  case RISCV::CM_POP:
+    return IsReturnZero ? RISCV::CM_POPRETZ : RISCV::CM_POPRET;
+  case RISCV::QC_CM_POP:
+    return IsReturnZero ? RISCV::QC_CM_POPRETZ : RISCV::QC_CM_POPRET;
+  default:
+    llvm_unreachable("Unhandled Pop Opcode");
+  }
+}
+
 // Check if POP instruction was inserted into the MBB and return iterator to it.
 static MachineBasicBlock::iterator containsPop(MachineBasicBlock &MBB) {
  for (MachineBasicBlock::iterator MBBI = MBB.begin(); MBBI != MBB.end();
       MBBI = next_nodbg(MBBI, MBB.end()))
-    if (MBBI->getOpcode() == RISCV::CM_POP)
+    if (MBBI->getFlag(MachineInstr::FrameDestroy) && isPop(MBBI->getOpcode()))
      return MBBI;

  return MBB.end();
@@ -61,11 +85,12 @@ bool RISCVPushPopOpt::usePopRet(MachineBasicBlock::iterator &MBBI,
  // Since Pseudo instruction lowering happen later in the pipeline,
  // this will detect all ret instruction.
  DebugLoc DL = NextI->getDebugLoc();
-  unsigned Opc = IsReturnZero ? RISCV::CM_POPRETZ : RISCV::CM_POPRET;
+  unsigned Opc = getPopRetOpcode(MBBI->getOpcode(), IsReturnZero);
  MachineInstrBuilder PopRetBuilder =
      BuildMI(*NextI->getParent(), NextI, DL, TII->get(Opc))
          .add(MBBI->getOperand(0))
-          .add(MBBI->getOperand(1));
+          .add(MBBI->getOperand(1))
+          .setMIFlag(MachineInstr::FrameDestroy);

  // Copy over the variable implicit uses and defs from the CM_POP. They depend
  // on what register list has been picked during frame lowering.
@@ -120,12 +145,7 @@ bool RISCVPushPopOpt::runOnMachineFunction(MachineFunction &Fn) {

  // If Zcmp extension is not supported, abort.
  const RISCVSubtarget *Subtarget = &Fn.getSubtarget<RISCVSubtarget>();
-  if (!Subtarget->hasStdExtZcmp())
-    return false;
-
-  // If frame pointer elimination has been disabled, abort to avoid breaking the
-  // ABI.
-  if (Fn.getTarget().Options.DisableFramePointerElim(Fn))
+  if (!Subtarget->hasStdExtZcmp() && !Subtarget->hasVendorXqccmp())
    return false;

  TII = Subtarget->getInstrInfo();
--- a/llvm/test/CodeGen/RISCV/xqccmp-additional-stack.ll
+++ b/llvm/test/CodeGen/RISCV/xqccmp-additional-stack.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqccmp,+e -target-abi ilp32e -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV32
+
+define ptr @func(ptr %s, i32 %_c, ptr %incdec.ptr, i1 %0, i8 %conv14) #0 {
+; RV32-LABEL: func:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    qc.cm.push {ra, s0-s1}, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    .cfi_offset s0, -8
+; RV32-NEXT:    .cfi_offset s1, -12
+; RV32-NEXT:    addi sp, sp, -4
+; RV32-NEXT:    .cfi_def_cfa_offset 20
+; RV32-NEXT:    sw a4, 4(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw a2, 0(sp) # 4-byte Folded Spill
+; RV32-NEXT:    mv a2, a1
+; RV32-NEXT:    mv s1, a0
+; RV32-NEXT:    li a0, 1
+; RV32-NEXT:    andi a3, a3, 1
+; RV32-NEXT:  .LBB0_1: # %while.body
+; RV32-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32-NEXT:    mv s0, a0
+; RV32-NEXT:    li a0, 0
+; RV32-NEXT:    bnez a3, .LBB0_1
+; RV32-NEXT:  # %bb.2: # %while.end
+; RV32-NEXT:    lui a0, 4112
+; RV32-NEXT:    addi a1, a0, 257
+; RV32-NEXT:    mv a0, a2
+; RV32-NEXT:    call __mulsi3
+; RV32-NEXT:    sw a0, 0(zero)
+; RV32-NEXT:    andi s0, s0, 1
+; RV32-NEXT:    lw a0, 0(sp) # 4-byte Folded Reload
+; RV32-NEXT:    add s0, s0, a0
+; RV32-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
+; RV32-NEXT:    sb a0, 0(s0)
+; RV32-NEXT:    mv a0, s1
+; RV32-NEXT:    addi sp, sp, 4
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    qc.cm.popret {ra, s0-s1}, 16
+entry:
+  br label %while.body
+
+while.body:                                       ; preds = %while.body, %entry
+  %n.addr.042 = phi i32 [ 1, %entry ], [ 0, %while.body ]
+  br i1 %0, label %while.body, label %while.end
+
+while.end:                                        ; preds = %while.body
+  %or5 = mul i32 %_c, 16843009
+  store i32 %or5, ptr null, align 4
+  %1 = and i32 %n.addr.042, 1
+  %scevgep = getelementptr i8, ptr %incdec.ptr, i32 %1
+  store i8 %conv14, ptr %scevgep, align 1
+  ret ptr %s
+}
--- a/llvm/test/CodeGen/RISCV/xqccmp-callee-saved-gprs.ll
+++ b/llvm/test/CodeGen/RISCV/xqccmp-callee-saved-gprs.ll
--- a/llvm/test/CodeGen/RISCV/xqccmp-cm-popretz.mir
+++ b/llvm/test/CodeGen/RISCV/xqccmp-cm-popretz.mir
@@ -0,0 +1,66 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -mtriple=riscv32 -mattr=+experimental-xqccmp -x mir -start-before=prologepilog -stop-after=riscv-push-pop-opt -verify-machineinstrs -o - %s \
+# RUN: | FileCheck -check-prefixes=CHECK-XQCCMP32 %s
+# RUN: llc -mtriple=riscv64 -mattr=+experimental-xqccmp -x mir -start-before=prologepilog -stop-after=riscv-push-pop-opt -verify-machineinstrs -o - %s \
+# RUN: | FileCheck -check-prefixes=CHECK-XQCCMP64 %s
+---
+name: popret_rvlist5
+tracksRegLiveness: true
+body:                   |
+  bb.0:
+    ; CHECK-XQCCMP32-LABEL: name: popret_rvlist5
+    ; CHECK-XQCCMP32: liveins: $x1, $x8
+    ; CHECK-XQCCMP32-NEXT: {{  $}}
+    ; CHECK-XQCCMP32-NEXT: frame-setup QC_CM_PUSH 5, 0, implicit-def $x2, implicit $x2, implicit $x1, implicit $x8
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -4
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -8
+    ; CHECK-XQCCMP32-NEXT: $x1 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: $x8 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: frame-destroy QC_CM_POPRET 5, 0, implicit-def $x2, implicit $x2, implicit-def $x1, implicit-def $x8
+    ;
+    ; CHECK-XQCCMP64-LABEL: name: popret_rvlist5
+    ; CHECK-XQCCMP64: liveins: $x1, $x8
+    ; CHECK-XQCCMP64-NEXT: {{  $}}
+    ; CHECK-XQCCMP64-NEXT: frame-setup QC_CM_PUSH 5, 0, implicit-def $x2, implicit $x2, implicit $x1, implicit $x8
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -16
+    ; CHECK-XQCCMP64-NEXT: $x1 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: $x8 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: frame-destroy QC_CM_POPRET 5, 0, implicit-def $x2, implicit $x2, implicit-def $x1, implicit-def $x8
+    $x1 = IMPLICIT_DEF
+    $x8 = IMPLICIT_DEF
+    PseudoRET
+...
+---
+name: popretz_rvlist5
+tracksRegLiveness: true
+body:                   |
+  bb.0:
+    ; CHECK-XQCCMP32-LABEL: name: popretz_rvlist5
+    ; CHECK-XQCCMP32: liveins: $x1, $x8
+    ; CHECK-XQCCMP32-NEXT: {{  $}}
+    ; CHECK-XQCCMP32-NEXT: frame-setup QC_CM_PUSH 5, 0, implicit-def $x2, implicit $x2, implicit $x1, implicit $x8
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -4
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -8
+    ; CHECK-XQCCMP32-NEXT: $x1 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: $x8 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: frame-destroy QC_CM_POPRETZ 5, 0, implicit-def $x2, implicit-def $x10, implicit $x2, implicit-def $x1, implicit-def $x8
+    ;
+    ; CHECK-XQCCMP64-LABEL: name: popretz_rvlist5
+    ; CHECK-XQCCMP64: liveins: $x1, $x8
+    ; CHECK-XQCCMP64-NEXT: {{  $}}
+    ; CHECK-XQCCMP64-NEXT: frame-setup QC_CM_PUSH 5, 0, implicit-def $x2, implicit $x2, implicit $x1, implicit $x8
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -16
+    ; CHECK-XQCCMP64-NEXT: $x1 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: $x8 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: frame-destroy QC_CM_POPRETZ 5, 0, implicit-def $x2, implicit-def $x10, implicit $x2, implicit-def $x1, implicit-def $x8
+    $x1 = IMPLICIT_DEF
+    $x8 = IMPLICIT_DEF
+    $x10 = COPY $x0
+    PseudoRET implicit $x10
+...
--- a/llvm/test/CodeGen/RISCV/xqccmp-cm-push-pop.mir
+++ b/llvm/test/CodeGen/RISCV/xqccmp-cm-push-pop.mir
@@ -0,0 +1,92 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -mtriple=riscv32 -mattr=+experimental-xqccmp -x mir -run-pass=prologepilog -verify-machineinstrs -o - %s \
+# RUN: | FileCheck -check-prefixes=CHECK-XQCCMP32 %s
+# RUN: llc -mtriple=riscv64 -mattr=+experimental-xqccmp -x mir -run-pass=prologepilog -verify-machineinstrs -o - %s \
+# RUN: | FileCheck -check-prefixes=CHECK-XQCCMP64 %s
+---
+name: push_rvlist15
+tracksRegLiveness: true
+body:                   |
+  bb.0:
+    ; CHECK-XQCCMP32-LABEL: name: push_rvlist15
+    ; CHECK-XQCCMP32: liveins: $x1, $x8, $x9, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27
+    ; CHECK-XQCCMP32-NEXT: {{  $}}
+    ; CHECK-XQCCMP32-NEXT: frame-setup QC_CM_PUSH 15, 0, implicit-def $x2, implicit $x2, implicit $x1, implicit $x8, implicit $x9, implicit $x18, implicit $x19, implicit $x20, implicit $x21, implicit $x22, implicit $x23, implicit $x24, implicit $x25, implicit $x26, implicit $x27
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 64
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -4
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -8
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x9, -12
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x18, -16
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x19, -20
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x20, -24
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x21, -28
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x22, -32
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x23, -36
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x24, -40
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x25, -44
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x26, -48
+    ; CHECK-XQCCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x27, -52
+    ; CHECK-XQCCMP32-NEXT: $x1 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: $x8 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: $x9 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: $x18 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: $x19 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: $x20 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: $x21 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: $x22 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: $x23 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: $x24 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: $x25 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: $x26 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: $x27 = IMPLICIT_DEF
+    ; CHECK-XQCCMP32-NEXT: frame-destroy QC_CM_POP 15, 0, implicit-def $x2, implicit $x2, implicit-def $x1, implicit-def $x8, implicit-def $x9, implicit-def $x18, implicit-def $x19, implicit-def $x20, implicit-def $x21, implicit-def $x22, implicit-def $x23, implicit-def $x24, implicit-def $x25, implicit-def $x26, implicit-def $x27
+    ; CHECK-XQCCMP32-NEXT: PseudoRET
+    ;
+    ; CHECK-XQCCMP64-LABEL: name: push_rvlist15
+    ; CHECK-XQCCMP64: liveins: $x1, $x8, $x9, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27
+    ; CHECK-XQCCMP64-NEXT: {{  $}}
+    ; CHECK-XQCCMP64-NEXT: frame-setup QC_CM_PUSH 15, 0, implicit-def $x2, implicit $x2, implicit $x1, implicit $x8, implicit $x9, implicit $x18, implicit $x19, implicit $x20, implicit $x21, implicit $x22, implicit $x23, implicit $x24, implicit $x25, implicit $x26, implicit $x27
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 112
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x1, -8
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -16
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x9, -24
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x18, -32
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x19, -40
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x20, -48
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x21, -56
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x22, -64
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x23, -72
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x24, -80
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x25, -88
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x26, -96
+    ; CHECK-XQCCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x27, -104
+    ; CHECK-XQCCMP64-NEXT: $x1 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: $x8 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: $x9 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: $x18 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: $x19 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: $x20 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: $x21 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: $x22 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: $x23 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: $x24 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: $x25 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: $x26 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: $x27 = IMPLICIT_DEF
+    ; CHECK-XQCCMP64-NEXT: frame-destroy QC_CM_POP 15, 0, implicit-def $x2, implicit $x2, implicit-def $x1, implicit-def $x8, implicit-def $x9, implicit-def $x18, implicit-def $x19, implicit-def $x20, implicit-def $x21, implicit-def $x22, implicit-def $x23, implicit-def $x24, implicit-def $x25, implicit-def $x26, implicit-def $x27
+    ; CHECK-XQCCMP64-NEXT: PseudoRET
+    $x1 = IMPLICIT_DEF
+    $x8 = IMPLICIT_DEF
+    $x9 = IMPLICIT_DEF
+    $x18 = IMPLICIT_DEF
+    $x19 = IMPLICIT_DEF
+    $x20 = IMPLICIT_DEF
+    $x21 = IMPLICIT_DEF
+    $x22 = IMPLICIT_DEF
+    $x23 = IMPLICIT_DEF
+    $x24 = IMPLICIT_DEF
+    $x25 = IMPLICIT_DEF
+    $x26 = IMPLICIT_DEF
+    $x27 = IMPLICIT_DEF
+    PseudoRET
+...
--- a/llvm/test/CodeGen/RISCV/xqccmp-push-pop-popret.ll
+++ b/llvm/test/CodeGen/RISCV/xqccmp-push-pop-popret.ll
--- a/llvm/test/CodeGen/RISCV/xqccmp-with-float.ll
+++ b/llvm/test/CodeGen/RISCV/xqccmp-with-float.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=riscv32 -mattr=+f,+experimental-xqccmp -target-abi ilp32f -verify-machineinstrs < %s | FileCheck %s --check-prefix=XQCCMP32
+; RUN: llc -mtriple=riscv64 -mattr=+f,+experimental-xqccmp -target-abi lp64f -verify-machineinstrs < %s | FileCheck %s --check-prefix=XQCCMP64
+
+declare void @callee()
+
+; Test the file could be compiled successfully.
+define float @foo(float %arg) {
+; XQCCMP32-LABEL: foo:
+; XQCCMP32:       # %bb.0: # %entry
+; XQCCMP32-NEXT:    qc.cm.push {ra}, -16
+; XQCCMP32-NEXT:    .cfi_def_cfa_offset 16
+; XQCCMP32-NEXT:    .cfi_offset ra, -4
+; XQCCMP32-NEXT:    fsw fs0, 8(sp) # 4-byte Folded Spill
+; XQCCMP32-NEXT:    .cfi_offset fs0, -8
+; XQCCMP32-NEXT:    fmv.s fs0, fa0
+; XQCCMP32-NEXT:    call callee
+; XQCCMP32-NEXT:    fmv.s fa0, fs0
+; XQCCMP32-NEXT:    flw fs0, 8(sp) # 4-byte Folded Reload
+; XQCCMP32-NEXT:    .cfi_restore fs0
+; XQCCMP32-NEXT:    qc.cm.popret {ra}, 16
+;
+; XQCCMP64-LABEL: foo:
+; XQCCMP64:       # %bb.0: # %entry
+; XQCCMP64-NEXT:    qc.cm.push {ra}, -16
+; XQCCMP64-NEXT:    .cfi_def_cfa_offset 16
+; XQCCMP64-NEXT:    .cfi_offset ra, -8
+; XQCCMP64-NEXT:    fsw fs0, 4(sp) # 4-byte Folded Spill
+; XQCCMP64-NEXT:    .cfi_offset fs0, -12
+; XQCCMP64-NEXT:    fmv.s fs0, fa0
+; XQCCMP64-NEXT:    call callee
+; XQCCMP64-NEXT:    fmv.s fa0, fs0
+; XQCCMP64-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
+; XQCCMP64-NEXT:    .cfi_restore fs0
+; XQCCMP64-NEXT:    qc.cm.popret {ra}, 16
+entry:
+  call void @callee()
+  ret float %arg
+}
+
+define void @foo2(i32 %x, float %y) {
+; XQCCMP32-LABEL: foo2:
+; XQCCMP32:       # %bb.0: # %entry
+; XQCCMP32-NEXT:    qc.cm.push {ra, s0}, -16
+; XQCCMP32-NEXT:    .cfi_def_cfa_offset 16
+; XQCCMP32-NEXT:    .cfi_offset ra, -4
+; XQCCMP32-NEXT:    .cfi_offset s0, -8
+; XQCCMP32-NEXT:    fsw fs0, 4(sp) # 4-byte Folded Spill
+; XQCCMP32-NEXT:    .cfi_offset fs0, -12
+; XQCCMP32-NEXT:    fmv.s fs0, fa0
+; XQCCMP32-NEXT:    mv s0, a0
+; XQCCMP32-NEXT:    call bar
+; XQCCMP32-NEXT:    mv a0, s0
+; XQCCMP32-NEXT:    fmv.s fa0, fs0
+; XQCCMP32-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
+; XQCCMP32-NEXT:    .cfi_restore fs0
+; XQCCMP32-NEXT:    qc.cm.pop {ra, s0}, 16
+; XQCCMP32-NEXT:    .cfi_restore ra
+; XQCCMP32-NEXT:    .cfi_restore s0
+; XQCCMP32-NEXT:    .cfi_def_cfa_offset 0
+; XQCCMP32-NEXT:    tail func
+;
+; XQCCMP64-LABEL: foo2:
+; XQCCMP64:       # %bb.0: # %entry
+; XQCCMP64-NEXT:    qc.cm.push {ra, s0}, -32
+; XQCCMP64-NEXT:    .cfi_def_cfa_offset 32
+; XQCCMP64-NEXT:    .cfi_offset ra, -8
+; XQCCMP64-NEXT:    .cfi_offset s0, -16
+; XQCCMP64-NEXT:    fsw fs0, 12(sp) # 4-byte Folded Spill
+; XQCCMP64-NEXT:    .cfi_offset fs0, -20
+; XQCCMP64-NEXT:    fmv.s fs0, fa0
+; XQCCMP64-NEXT:    mv s0, a0
+; XQCCMP64-NEXT:    call bar
+; XQCCMP64-NEXT:    mv a0, s0
+; XQCCMP64-NEXT:    fmv.s fa0, fs0
+; XQCCMP64-NEXT:    flw fs0, 12(sp) # 4-byte Folded Reload
+; XQCCMP64-NEXT:    .cfi_restore fs0
+; XQCCMP64-NEXT:    qc.cm.pop {ra, s0}, 32
+; XQCCMP64-NEXT:    .cfi_restore ra
+; XQCCMP64-NEXT:    .cfi_restore s0
+; XQCCMP64-NEXT:    .cfi_def_cfa_offset 0
+; XQCCMP64-NEXT:    tail func
+entry:
+  tail call void @bar()
+  tail call void @func(i32 %x, float %y)
+  ret void
+}
+
+declare void @bar()
+declare void @func(i32, float)
--- a/llvm/test/CodeGen/RISCV/xqccmp_mvas_mvsa.mir
+++ b/llvm/test/CodeGen/RISCV/xqccmp_mvas_mvsa.mir
@@ -0,0 +1,28 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -mtriple=riscv32 -mattr=+experimental-xqccmp -verify-machineinstrs -run-pass=riscv-move-merge -simplify-mir -o - %s | FileCheck -check-prefixes=CHECK32XQCCMP %s
+# RUN: llc -mtriple=riscv64 -mattr=+experimental-xqccmp -verify-machineinstrs -run-pass=riscv-move-merge -simplify-mir -o - %s | FileCheck -check-prefixes=CHECK64XQCCMP %s
+---
+name: mv
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x11, $x10
+    ; CHECK32XQCCMP-LABEL: name: mv
+    ; CHECK32XQCCMP: liveins: $x11, $x10
+    ; CHECK32XQCCMP-NEXT: {{  $}}
+    ; CHECK32XQCCMP-NEXT: $x9, $x8 = QC_CM_MVSA01 implicit $x10, implicit $x11
+    ; CHECK32XQCCMP-NEXT: QC_CM_MVA01S killed $x9, $x8, implicit-def $x10, implicit-def $x11
+    ; CHECK32XQCCMP-NEXT: PseudoRET
+    ;
+    ; CHECK64XQCCMP-LABEL: name: mv
+    ; CHECK64XQCCMP: liveins: $x11, $x10
+    ; CHECK64XQCCMP-NEXT: {{  $}}
+    ; CHECK64XQCCMP-NEXT: $x9, $x8 = QC_CM_MVSA01 implicit $x10, implicit $x11
+    ; CHECK64XQCCMP-NEXT: QC_CM_MVA01S killed $x9, $x8, implicit-def $x10, implicit-def $x11
+    ; CHECK64XQCCMP-NEXT: PseudoRET
+    $x8 = ADDI $x11, 0
+    $x9 = ADDI $x10, 0
+    $x10 = ADDI killed $x9, 0
+    $x11 = ADDI $x8, 0
+    PseudoRET
+...
--- a/llvm/test/CodeGen/RISCV/zcmp-cm-popretz.mir
+++ b/llvm/test/CodeGen/RISCV/zcmp-cm-popretz.mir
@@ -25,7 +25,7 @@ body:                   |
    ; CHECK-ZCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -4
    ; CHECK-ZCMP32-NEXT: $x1 = IMPLICIT_DEF
    ; CHECK-ZCMP32-NEXT: $x8 = IMPLICIT_DEF
-    ; CHECK-ZCMP32-NEXT: CM_POPRET 5, 0, implicit-def $x2, implicit $x2, implicit-def $x1, implicit-def $x8
+    ; CHECK-ZCMP32-NEXT: frame-destroy CM_POPRET 5, 0, implicit-def $x2, implicit $x2, implicit-def $x1, implicit-def $x8
    ;
    ; CHECK-LIBCALL32-LABEL: name: popret_rvlist5
    ; CHECK-LIBCALL32: liveins: $x1, $x8
@@ -47,7 +47,7 @@ body:                   |
    ; CHECK-ZCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -8
    ; CHECK-ZCMP64-NEXT: $x1 = IMPLICIT_DEF
    ; CHECK-ZCMP64-NEXT: $x8 = IMPLICIT_DEF
-    ; CHECK-ZCMP64-NEXT: CM_POPRET 5, 0, implicit-def $x2, implicit $x2, implicit-def $x1, implicit-def $x8
+    ; CHECK-ZCMP64-NEXT: frame-destroy CM_POPRET 5, 0, implicit-def $x2, implicit $x2, implicit-def $x1, implicit-def $x8
    ;
    ; CHECK-LIBCALL64-LABEL: name: popret_rvlist5
    ; CHECK-LIBCALL64: liveins: $x1, $x8
@@ -115,7 +115,7 @@ body:                   |
    ; CHECK-ZCMP32-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -4
    ; CHECK-ZCMP32-NEXT: $x1 = IMPLICIT_DEF
    ; CHECK-ZCMP32-NEXT: $x8 = IMPLICIT_DEF
-    ; CHECK-ZCMP32-NEXT: CM_POPRETZ 5, 0, implicit-def $x2, implicit-def $x10, implicit $x2, implicit-def $x1, implicit-def $x8
+    ; CHECK-ZCMP32-NEXT: frame-destroy CM_POPRETZ 5, 0, implicit-def $x2, implicit-def $x10, implicit $x2, implicit-def $x1, implicit-def $x8
    ;
    ; CHECK-LIBCALL32-LABEL: name: popretz_rvlist5
    ; CHECK-LIBCALL32: liveins: $x1, $x8
@@ -138,7 +138,7 @@ body:                   |
    ; CHECK-ZCMP64-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -8
    ; CHECK-ZCMP64-NEXT: $x1 = IMPLICIT_DEF
    ; CHECK-ZCMP64-NEXT: $x8 = IMPLICIT_DEF
-    ; CHECK-ZCMP64-NEXT: CM_POPRETZ 5, 0, implicit-def $x2, implicit-def $x10, implicit $x2, implicit-def $x1, implicit-def $x8
+    ; CHECK-ZCMP64-NEXT: frame-destroy CM_POPRETZ 5, 0, implicit-def $x2, implicit-def $x10, implicit $x2, implicit-def $x1, implicit-def $x8
    ;
    ; CHECK-LIBCALL64-LABEL: name: popretz_rvlist5
    ; CHECK-LIBCALL64: liveins: $x1, $x8