4a305d40a3
The s_singleuse_vdst instruction is used to mark regions of instructions that produce values that have only one use. Certain instructions take more than one cycle to execute, resulting in regions being incorrectly marked. This patch excludes these multi-cycle instructions from being marked as either producing single use values or consuming single use values or both depending on the instruction.
246 lines
8.7 KiB
C++
246 lines
8.7 KiB
C++
//===- AMDGPUInsertSingleUseVDST.cpp - Insert s_singleuse_vdst instructions ==//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
/// \file
|
|
/// Insert s_singleuse_vdst instructions on GFX11.5+ to mark regions of VALU
|
|
/// instructions that produce single-use VGPR values. If the value is forwarded
|
|
/// to the consumer instruction prior to VGPR writeback, the hardware can
|
|
/// then skip (kill) the VGPR write.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "AMDGPU.h"
|
|
#include "AMDGPUGenSearchableTables.inc"
|
|
#include "GCNSubtarget.h"
|
|
#include "SIInstrInfo.h"
|
|
#include "SIRegisterInfo.h"
|
|
#include "llvm/ADT/DenseMap.h"
|
|
#include "llvm/ADT/STLExtras.h"
|
|
#include "llvm/ADT/SmallVector.h"
|
|
#include "llvm/ADT/StringRef.h"
|
|
#include "llvm/CodeGen/MachineBasicBlock.h"
|
|
#include "llvm/CodeGen/MachineFunction.h"
|
|
#include "llvm/CodeGen/MachineFunctionPass.h"
|
|
#include "llvm/CodeGen/MachineInstr.h"
|
|
#include "llvm/CodeGen/MachineInstrBuilder.h"
|
|
#include "llvm/CodeGen/MachineOperand.h"
|
|
#include "llvm/CodeGen/Register.h"
|
|
#include "llvm/IR/DebugLoc.h"
|
|
#include "llvm/MC/MCRegister.h"
|
|
#include "llvm/MC/MCRegisterInfo.h"
|
|
#include "llvm/Pass.h"
|
|
#include <array>
|
|
|
|
using namespace llvm;
|
|
|
|
#define DEBUG_TYPE "amdgpu-insert-single-use-vdst"
|
|
|
|
namespace {
|
|
class AMDGPUInsertSingleUseVDST : public MachineFunctionPass {
|
|
private:
|
|
const SIInstrInfo *SII;
|
|
class SingleUseInstruction {
|
|
private:
|
|
static const unsigned MaxSkipRange = 0b111;
|
|
static const unsigned MaxNumberOfSkipRegions = 2;
|
|
|
|
unsigned LastEncodedPositionEnd;
|
|
MachineInstr *ProducerInstr;
|
|
|
|
std::array<unsigned, MaxNumberOfSkipRegions + 1> SingleUseRegions;
|
|
SmallVector<unsigned, MaxNumberOfSkipRegions> SkipRegions;
|
|
|
|
// Adds a skip region into the instruction.
|
|
void skip(const unsigned ProducerPosition) {
|
|
while (LastEncodedPositionEnd + MaxSkipRange < ProducerPosition) {
|
|
SkipRegions.push_back(MaxSkipRange);
|
|
LastEncodedPositionEnd += MaxSkipRange;
|
|
}
|
|
SkipRegions.push_back(ProducerPosition - LastEncodedPositionEnd);
|
|
LastEncodedPositionEnd = ProducerPosition;
|
|
}
|
|
|
|
bool currentRegionHasSpace() {
|
|
const auto Region = SkipRegions.size();
|
|
// The first region has an extra bit of encoding space.
|
|
return SingleUseRegions[Region] <
|
|
((Region == MaxNumberOfSkipRegions) ? 0b1111U : 0b111U);
|
|
}
|
|
|
|
unsigned encodeImm() {
|
|
// Handle the first Single Use Region separately as it has an extra bit
|
|
// of encoding space.
|
|
unsigned Imm = SingleUseRegions[SkipRegions.size()];
|
|
unsigned ShiftAmount = 4;
|
|
for (unsigned i = SkipRegions.size(); i > 0; i--) {
|
|
Imm |= SkipRegions[i - 1] << ShiftAmount;
|
|
ShiftAmount += 3;
|
|
Imm |= SingleUseRegions[i - 1] << ShiftAmount;
|
|
ShiftAmount += 3;
|
|
}
|
|
return Imm;
|
|
}
|
|
|
|
public:
|
|
SingleUseInstruction(const unsigned ProducerPosition,
|
|
MachineInstr *Producer)
|
|
: LastEncodedPositionEnd(ProducerPosition + 1), ProducerInstr(Producer),
|
|
SingleUseRegions({1, 0, 0}) {}
|
|
|
|
// Returns false if adding a new single use producer failed. This happens
|
|
// because it could not be encoded, either because there is no room to
|
|
// encode another single use producer region or that this single use
|
|
// producer is too far away to encode the amount of instructions to skip.
|
|
bool tryAddProducer(const unsigned ProducerPosition, MachineInstr *MI) {
|
|
// Producer is too far away to encode into this instruction or another
|
|
// skip region is needed and SkipRegions.size() = 2 so there's no room for
|
|
// another skip region, therefore a new instruction is needed.
|
|
if (LastEncodedPositionEnd +
|
|
(MaxSkipRange * (MaxNumberOfSkipRegions - SkipRegions.size())) <
|
|
ProducerPosition)
|
|
return false;
|
|
|
|
// If a skip region is needed.
|
|
if (LastEncodedPositionEnd != ProducerPosition ||
|
|
!currentRegionHasSpace()) {
|
|
// If the current region is out of space therefore a skip region would
|
|
// be needed, but there is no room for another skip region.
|
|
if (SkipRegions.size() == MaxNumberOfSkipRegions)
|
|
return false;
|
|
skip(ProducerPosition);
|
|
}
|
|
|
|
SingleUseRegions[SkipRegions.size()]++;
|
|
LastEncodedPositionEnd = ProducerPosition + 1;
|
|
ProducerInstr = MI;
|
|
return true;
|
|
}
|
|
|
|
auto emit(const SIInstrInfo *SII) {
|
|
return BuildMI(*ProducerInstr->getParent(), ProducerInstr, DebugLoc(),
|
|
SII->get(AMDGPU::S_SINGLEUSE_VDST))
|
|
.addImm(encodeImm());
|
|
}
|
|
};
|
|
|
|
public:
|
|
static char ID;
|
|
|
|
AMDGPUInsertSingleUseVDST() : MachineFunctionPass(ID) {}
|
|
|
|
void insertSingleUseInstructions(
|
|
ArrayRef<std::pair<unsigned, MachineInstr *>> SingleUseProducers) const {
|
|
SmallVector<SingleUseInstruction> Instructions;
|
|
|
|
for (auto &[Position, MI] : SingleUseProducers) {
|
|
// Encode this position into the last single use instruction if possible.
|
|
if (Instructions.empty() ||
|
|
!Instructions.back().tryAddProducer(Position, MI)) {
|
|
// If not, add a new instruction.
|
|
Instructions.push_back(SingleUseInstruction(Position, MI));
|
|
}
|
|
}
|
|
|
|
for (auto &Instruction : Instructions)
|
|
Instruction.emit(SII);
|
|
}
|
|
|
|
bool runOnMachineFunction(MachineFunction &MF) override {
|
|
const auto &ST = MF.getSubtarget<GCNSubtarget>();
|
|
if (!ST.hasVGPRSingleUseHintInsts())
|
|
return false;
|
|
|
|
SII = ST.getInstrInfo();
|
|
const auto *TRI = &SII->getRegisterInfo();
|
|
bool InstructionEmitted = false;
|
|
|
|
for (MachineBasicBlock &MBB : MF) {
|
|
DenseMap<MCRegUnit, unsigned> RegisterUseCount;
|
|
|
|
// Handle boundaries at the end of basic block separately to avoid
|
|
// false positives. If they are live at the end of a basic block then
|
|
// assume it has more uses later on.
|
|
for (const auto &Liveout : MBB.liveouts()) {
|
|
for (MCRegUnitMaskIterator Units(Liveout.PhysReg, TRI); Units.isValid();
|
|
++Units) {
|
|
const auto [Unit, Mask] = *Units;
|
|
if ((Mask & Liveout.LaneMask).any())
|
|
RegisterUseCount[Unit] = 2;
|
|
}
|
|
}
|
|
|
|
SmallVector<std::pair<unsigned, MachineInstr *>>
|
|
SingleUseProducerPositions;
|
|
|
|
unsigned VALUInstrCount = 0;
|
|
for (MachineInstr &MI : reverse(MBB.instrs())) {
|
|
// All registers in all operands need to be single use for an
|
|
// instruction to be marked as a single use producer.
|
|
bool AllProducerOperandsAreSingleUse = true;
|
|
|
|
// Gather a list of Registers used before updating use counts to avoid
|
|
// double counting registers that appear multiple times in a single
|
|
// MachineInstr.
|
|
SmallVector<MCRegUnit> RegistersUsed;
|
|
|
|
for (const auto &Operand : MI.all_defs()) {
|
|
const auto Reg = Operand.getReg();
|
|
|
|
const auto RegUnits = TRI->regunits(Reg);
|
|
if (any_of(RegUnits, [&RegisterUseCount](const MCRegUnit Unit) {
|
|
return RegisterUseCount[Unit] > 1;
|
|
}))
|
|
AllProducerOperandsAreSingleUse = false;
|
|
|
|
// Reset uses count when a register is no longer live.
|
|
for (const MCRegUnit Unit : RegUnits)
|
|
RegisterUseCount.erase(Unit);
|
|
}
|
|
|
|
for (const auto &Operand : MI.all_uses()) {
|
|
const auto Reg = Operand.getReg();
|
|
|
|
// Count the number of times each register is read.
|
|
for (const MCRegUnit Unit : TRI->regunits(Reg)) {
|
|
if (!is_contained(RegistersUsed, Unit))
|
|
RegistersUsed.push_back(Unit);
|
|
}
|
|
}
|
|
for (const MCRegUnit Unit : RegistersUsed)
|
|
RegisterUseCount[Unit]++;
|
|
|
|
// Do not attempt to optimise across exec mask changes.
|
|
if (MI.modifiesRegister(AMDGPU::EXEC, TRI) ||
|
|
AMDGPU::isInvalidSingleUseConsumerInst(MI.getOpcode())) {
|
|
for (auto &UsedReg : RegisterUseCount)
|
|
UsedReg.second = 2;
|
|
}
|
|
|
|
if (!SIInstrInfo::isVALU(MI) ||
|
|
AMDGPU::isInvalidSingleUseProducerInst(MI.getOpcode()))
|
|
continue;
|
|
if (AllProducerOperandsAreSingleUse) {
|
|
SingleUseProducerPositions.push_back({VALUInstrCount, &MI});
|
|
InstructionEmitted = true;
|
|
}
|
|
VALUInstrCount++;
|
|
}
|
|
insertSingleUseInstructions(SingleUseProducerPositions);
|
|
}
|
|
return InstructionEmitted;
|
|
}
|
|
};
|
|
} // namespace
|
|
|
|
char AMDGPUInsertSingleUseVDST::ID = 0;
|
|
|
|
char &llvm::AMDGPUInsertSingleUseVDSTID = AMDGPUInsertSingleUseVDST::ID;
|
|
|
|
INITIALIZE_PASS(AMDGPUInsertSingleUseVDST, DEBUG_TYPE,
|
|
"AMDGPU Insert SingleUseVDST", false, false)
|