2c1ae801e1
This patch adds more precise side effects to the current ops with memory effects, allowing us to determine which OpOperand/OpResult/BlockArgument the operation reads or writes, rather than just recording the reading and writing of values. This allows for convenient use of precise side effects to achieve analysis and optimization. Related discussions: https://discourse.llvm.org/t/rfc-add-operandindex-to-sideeffect-instance/79243
730 lines
33 KiB
C++
730 lines
33 KiB
C++
//===- ScheduleOrderedAssignments.cpp -- Ordered Assignment Scheduling ----===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "ScheduleOrderedAssignments.h"
|
|
#include "flang/Optimizer/Analysis/AliasAnalysis.h"
|
|
#include "flang/Optimizer/Builder/FIRBuilder.h"
|
|
#include "flang/Optimizer/Builder/Todo.h"
|
|
#include "flang/Optimizer/Dialect/Support/FIRContext.h"
|
|
#include "llvm/ADT/SmallSet.h"
|
|
#include "llvm/Support/Debug.h"
|
|
|
|
#define DEBUG_TYPE "flang-ordered-assignment"
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Scheduling logging utilities for debug and test
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
/// Log RAW or WAW conflict.
|
|
static void LLVM_ATTRIBUTE_UNUSED logConflict(llvm::raw_ostream &os,
|
|
mlir::Value writtenOrReadVarA,
|
|
mlir::Value writtenVarB);
|
|
/// Log when an expression evaluation must be saved.
|
|
static void LLVM_ATTRIBUTE_UNUSED logSaveEvaluation(llvm::raw_ostream &os,
|
|
unsigned runid,
|
|
mlir::Region &yieldRegion,
|
|
bool anyWrite);
|
|
/// Log when an assignment is scheduled.
|
|
static void LLVM_ATTRIBUTE_UNUSED logAssignmentEvaluation(
|
|
llvm::raw_ostream &os, unsigned runid, hlfir::RegionAssignOp assign);
|
|
/// Log when starting to schedule an order assignment tree.
|
|
static void LLVM_ATTRIBUTE_UNUSED logStartScheduling(
|
|
llvm::raw_ostream &os, hlfir::OrderedAssignmentTreeOpInterface root);
|
|
/// Log op if effect value is not known.
|
|
static void LLVM_ATTRIBUTE_UNUSED logIfUnkownEffectValue(
|
|
llvm::raw_ostream &os, mlir::MemoryEffects::EffectInstance effect,
|
|
mlir::Operation &op);
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Scheduling Implementation
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
namespace {
|
|
/// Structure that is in charge of building the schedule. For each
|
|
/// hlfir.region_assign inside an ordered assignment tree, it is walked through
|
|
/// the parent operations and their "leaf" regions (that contain expression
|
|
/// evaluations). The Scheduler analyze the memory effects of these regions
|
|
/// against the effect of the current assignment, and if any conflict is found,
|
|
/// it will create an action to save the value computed by the region before the
|
|
/// assignment evaluation.
|
|
class Scheduler {
|
|
public:
|
|
Scheduler(bool tryFusingAssignments)
|
|
: tryFusingAssignments{tryFusingAssignments} {}
|
|
|
|
/// Start scheduling an assignment. Gather the write side effect from the
|
|
/// assignment.
|
|
void startSchedulingAssignment(hlfir::RegionAssignOp assign,
|
|
bool leafRegionsMayOnlyRead);
|
|
|
|
/// Start analysing a set of evaluation regions that can be evaluated in
|
|
/// any order between themselves according to Fortran rules (like the controls
|
|
/// of forall). The point of this is to avoid adding the side effects of
|
|
/// independent evaluations to a run that would save only one of the control.
|
|
void startIndependentEvaluationGroup() {
|
|
assert(independentEvaluationEffects.empty() &&
|
|
"previous group was not finished");
|
|
};
|
|
|
|
/// Analyze the memory effects of a region containing an expression
|
|
/// evaluation. If any conflict is found with the current assignment, or if
|
|
/// the expression has write effects (which is possible outside of forall),
|
|
/// create an action in the schedule to save the value in the schedule before
|
|
/// evaluating the current assignment. For expression with write effect,
|
|
/// saving them ensures they are evaluated only once. A region whose value
|
|
/// was saved in a previous run is considered to have no side effects with the
|
|
/// current assignment: the saved value will be used.
|
|
void saveEvaluationIfConflict(mlir::Region &yieldRegion,
|
|
bool leafRegionsMayOnlyRead,
|
|
bool yieldIsImplicitRead = true,
|
|
bool evaluationsMayConflict = false);
|
|
|
|
/// Finish evaluating a group of independent regions. The current independent
|
|
/// regions effects are added to the "parent" effect list since evaluating the
|
|
/// next analyzed region would require evaluating the current independent
|
|
/// regions.
|
|
void finishIndependentEvaluationGroup() {
|
|
parentEvaluationEffects.append(independentEvaluationEffects.begin(),
|
|
independentEvaluationEffects.end());
|
|
independentEvaluationEffects.clear();
|
|
}
|
|
|
|
/// After all the dependent evaluation regions have been analyzed, create the
|
|
/// action to evaluate the assignment that was being analyzed.
|
|
void finishSchedulingAssignment(hlfir::RegionAssignOp assign);
|
|
|
|
/// Once all the assignments have been analyzed and scheduled, return the
|
|
/// schedule. The scheduler object should not be used after this call.
|
|
hlfir::Schedule moveSchedule() { return std::move(schedule); }
|
|
|
|
private:
|
|
/// Save a conflicting region that is evaluating an expression that is
|
|
/// controlling or masking the current assignment, or is evaluating the
|
|
/// RHS/LHS.
|
|
void
|
|
saveEvaluation(mlir::Region &yieldRegion,
|
|
llvm::ArrayRef<mlir::MemoryEffects::EffectInstance> effects,
|
|
bool anyWrite);
|
|
|
|
/// Can the current assignment be schedule with the previous run. This is
|
|
/// only possible if the assignment and all of its dependencies have no side
|
|
/// effects conflicting with the previous run.
|
|
bool canFuseAssignmentWithPreviousRun();
|
|
|
|
/// Memory effects of the assignments being lowered.
|
|
llvm::SmallVector<mlir::MemoryEffects::EffectInstance> assignEffects;
|
|
/// Memory effects of the evaluations implied by the assignments
|
|
/// being lowered. They do not include the implicit writes
|
|
/// to the LHS of the assignments.
|
|
llvm::SmallVector<mlir::MemoryEffects::EffectInstance> assignEvaluateEffects;
|
|
/// Memory effects of the unsaved evaluation region that are controlling or
|
|
/// masking the current assignments.
|
|
llvm::SmallVector<mlir::MemoryEffects::EffectInstance>
|
|
parentEvaluationEffects;
|
|
/// Same as parentEvaluationEffects, but for the current "leaf group" being
|
|
/// analyzed scheduled.
|
|
llvm::SmallVector<mlir::MemoryEffects::EffectInstance>
|
|
independentEvaluationEffects;
|
|
|
|
/// Were any region saved for the current assignment?
|
|
bool savedAnyRegionForCurrentAssignment = false;
|
|
|
|
// Schedule being built.
|
|
hlfir::Schedule schedule;
|
|
/// Leaf regions that have been saved so far.
|
|
llvm::SmallSet<mlir::Region *, 16> savedRegions;
|
|
/// Is schedule.back() a schedule that is only saving region with read
|
|
/// effects?
|
|
bool currentRunIsReadOnly = false;
|
|
|
|
/// Option to tell if the scheduler should try fusing to assignments in the
|
|
/// same loops.
|
|
const bool tryFusingAssignments;
|
|
};
|
|
} // namespace
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Scheduling Implementation : gathering memory effects of nodes.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
/// Is \p var the result of a ForallIndexOp?
|
|
/// Read effects to forall index can be ignored since forall
|
|
/// indices cannot be assigned to.
|
|
static bool isForallIndex(mlir::Value var) {
|
|
return var &&
|
|
mlir::isa_and_nonnull<hlfir::ForallIndexOp>(var.getDefiningOp());
|
|
}
|
|
|
|
/// Gather the memory effects of the operations contained in a region.
|
|
/// \p mayOnlyRead can be given to exclude some potential write effects that
|
|
/// cannot affect the current scheduling problem because it is known that the
|
|
/// regions are evaluating pure expressions from a Fortran point of view. It is
|
|
/// useful because low level IR in the region may contain operation that lacks
|
|
/// side effect interface, or that are writing temporary variables that may be
|
|
/// hard to identify as such (one would have to prove the write is "local" to
|
|
/// the region even when the alloca may be outside of the region).
|
|
static void gatherMemoryEffects(
|
|
mlir::Region ®ion, bool mayOnlyRead,
|
|
llvm::SmallVectorImpl<mlir::MemoryEffects::EffectInstance> &effects) {
|
|
/// This analysis is a simple walk of all the operations of the region that is
|
|
/// evaluating and yielding a value. This is a lot simpler and safer than
|
|
/// trying to walk back the SSA DAG from the yielded value. But if desired,
|
|
/// this could be changed.
|
|
for (mlir::Operation &op : region.getOps()) {
|
|
if (op.hasTrait<mlir::OpTrait::HasRecursiveMemoryEffects>()) {
|
|
for (mlir::Region &subRegion : op.getRegions())
|
|
gatherMemoryEffects(subRegion, mayOnlyRead, effects);
|
|
// In MLIR, RecursiveMemoryEffects can be combined with
|
|
// MemoryEffectOpInterface to describe extra effects on top of the
|
|
// effects of the nested operations. However, the presence of
|
|
// RecursiveMemoryEffects and the absence of MemoryEffectOpInterface
|
|
// implies the operation has no other memory effects than the one of its
|
|
// nested operations.
|
|
if (!mlir::isa<mlir::MemoryEffectOpInterface>(op))
|
|
continue;
|
|
}
|
|
mlir::MemoryEffectOpInterface interface =
|
|
mlir::dyn_cast<mlir::MemoryEffectOpInterface>(op);
|
|
if (!interface) {
|
|
LLVM_DEBUG(llvm::dbgs() << "unknown effect: " << op << "\n";);
|
|
// There is no generic way to know what this operation is reading/writing
|
|
// to. Assume the worst. No need to continue analyzing the code any
|
|
// further.
|
|
effects.emplace_back(mlir::MemoryEffects::Read::get());
|
|
if (!mayOnlyRead)
|
|
effects.emplace_back(mlir::MemoryEffects::Write::get());
|
|
return;
|
|
}
|
|
// Collect read/write effects. Alloc/Free effects do not matter, they
|
|
// are either local to the evaluation region and can be repeated, or, if
|
|
// they are allocatable/pointer allocation/deallocation, they are conveyed
|
|
// via the write that is updating the descriptor/allocatable (and there
|
|
// cannot be any indirect allocatable/pointer allocation/deallocation if
|
|
// mayOnlyRead is set). When mayOnlyRead is set, local write effects are
|
|
// also ignored.
|
|
llvm::SmallVector<mlir::MemoryEffects::EffectInstance> opEffects;
|
|
interface.getEffects(opEffects);
|
|
for (auto &effect : opEffects)
|
|
if (!isForallIndex(effect.getValue())) {
|
|
if (mlir::isa<mlir::MemoryEffects::Read>(effect.getEffect())) {
|
|
LLVM_DEBUG(logIfUnkownEffectValue(llvm::dbgs(), effect, op););
|
|
effects.push_back(effect);
|
|
} else if (!mayOnlyRead &&
|
|
mlir::isa<mlir::MemoryEffects::Write>(effect.getEffect())) {
|
|
LLVM_DEBUG(logIfUnkownEffectValue(llvm::dbgs(), effect, op););
|
|
effects.push_back(effect);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Return the entity yielded by a region, or a null value if the region
|
|
/// is not terminated by a yield.
|
|
static mlir::OpOperand *getYieldedEntity(mlir::Region ®ion) {
|
|
if (region.empty() || region.back().empty())
|
|
return nullptr;
|
|
if (auto yield = mlir::dyn_cast<hlfir::YieldOp>(region.back().back()))
|
|
return &yield.getEntityMutable();
|
|
if (auto elementalAddr =
|
|
mlir::dyn_cast<hlfir::ElementalAddrOp>(region.back().back()))
|
|
return &elementalAddr.getYieldOp().getEntityMutable();
|
|
return nullptr;
|
|
}
|
|
|
|
/// Gather the effect of an assignment. This is the implicit write to the LHS
|
|
/// of an assignment. This also includes the effects of the user defined
|
|
/// assignment, if any, but this does not include the effects of evaluating the
|
|
/// RHS and LHS, which occur before the assignment effects in Fortran.
|
|
static void gatherAssignEffects(
|
|
hlfir::RegionAssignOp regionAssign,
|
|
bool userDefAssignmentMayOnlyWriteToAssignedVariable,
|
|
llvm::SmallVectorImpl<mlir::MemoryEffects::EffectInstance> &assignEffects) {
|
|
mlir::OpOperand *assignedVar = getYieldedEntity(regionAssign.getLhsRegion());
|
|
assert(assignedVar && "lhs cannot be an empty region");
|
|
assignEffects.emplace_back(mlir::MemoryEffects::Write::get(), assignedVar);
|
|
|
|
if (!regionAssign.getUserDefinedAssignment().empty()) {
|
|
// The write effect on the INTENT(OUT) LHS argument is already taken
|
|
// into account above.
|
|
// This side effects are "defensive" and could be improved.
|
|
// On top of the passed RHS argument, user defined assignments (even when
|
|
// pure) may also read host/used/common variable. Impure user defined
|
|
// assignments may write to host/used/common variables not passed via
|
|
// arguments. For now, simply assume the worst. Once fir.call side effects
|
|
// analysis is improved, it would best to let the call side effects be used
|
|
// directly.
|
|
if (userDefAssignmentMayOnlyWriteToAssignedVariable)
|
|
assignEffects.emplace_back(mlir::MemoryEffects::Read::get());
|
|
else
|
|
assignEffects.emplace_back(mlir::MemoryEffects::Write::get());
|
|
}
|
|
}
|
|
|
|
/// Gather the effects of evaluations implied by the given assignment.
|
|
/// These are the effects of operations from LHS and RHS.
|
|
static void gatherAssignEvaluationEffects(
|
|
hlfir::RegionAssignOp regionAssign,
|
|
bool userDefAssignmentMayOnlyWriteToAssignedVariable,
|
|
llvm::SmallVectorImpl<mlir::MemoryEffects::EffectInstance> &assignEffects) {
|
|
gatherMemoryEffects(regionAssign.getLhsRegion(),
|
|
userDefAssignmentMayOnlyWriteToAssignedVariable,
|
|
assignEffects);
|
|
gatherMemoryEffects(regionAssign.getRhsRegion(),
|
|
userDefAssignmentMayOnlyWriteToAssignedVariable,
|
|
assignEffects);
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Scheduling Implementation : finding conflicting memory effects.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
/// Follow addressing and declare like operation to the storage source.
|
|
/// This allows using FIR alias analysis that otherwise does not know
|
|
/// about those operations. This is correct, but ignoring the designate
|
|
/// and declare info may yield false positive regarding aliasing (e.g,
|
|
/// if it could be proved that the variable are different sub-part of
|
|
/// an array).
|
|
static mlir::Value getStorageSource(mlir::Value var) {
|
|
// TODO: define some kind of View interface for Fortran in FIR,
|
|
// and use it in the FIR alias analysis.
|
|
mlir::Value source = var;
|
|
while (auto *op = source.getDefiningOp()) {
|
|
if (auto designate = mlir::dyn_cast<hlfir::DesignateOp>(op)) {
|
|
source = designate.getMemref();
|
|
} else if (auto declare = mlir::dyn_cast<hlfir::DeclareOp>(op)) {
|
|
source = declare.getMemref();
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
return source;
|
|
}
|
|
|
|
/// Could there be any read or write in effectsA on a variable written to in
|
|
/// effectsB?
|
|
static bool
|
|
anyRAWorWAW(llvm::ArrayRef<mlir::MemoryEffects::EffectInstance> effectsA,
|
|
llvm::ArrayRef<mlir::MemoryEffects::EffectInstance> effectsB,
|
|
fir::AliasAnalysis &aliasAnalysis) {
|
|
for (const auto &effectB : effectsB)
|
|
if (mlir::isa<mlir::MemoryEffects::Write>(effectB.getEffect())) {
|
|
mlir::Value writtenVarB = effectB.getValue();
|
|
if (writtenVarB)
|
|
writtenVarB = getStorageSource(writtenVarB);
|
|
for (const auto &effectA : effectsA)
|
|
if (mlir::isa<mlir::MemoryEffects::Write, mlir::MemoryEffects::Read>(
|
|
effectA.getEffect())) {
|
|
mlir::Value writtenOrReadVarA = effectA.getValue();
|
|
if (!writtenVarB || !writtenOrReadVarA) {
|
|
LLVM_DEBUG(
|
|
logConflict(llvm::dbgs(), writtenOrReadVarA, writtenVarB););
|
|
return true; // unknown conflict.
|
|
}
|
|
writtenOrReadVarA = getStorageSource(writtenOrReadVarA);
|
|
if (!aliasAnalysis.alias(writtenOrReadVarA, writtenVarB).isNo()) {
|
|
LLVM_DEBUG(
|
|
logConflict(llvm::dbgs(), writtenOrReadVarA, writtenVarB););
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/// Could there be any read or write in effectsA on a variable written to in
|
|
/// effectsB, or any read in effectsB on a variable written to in effectsA?
|
|
static bool
|
|
conflict(llvm::ArrayRef<mlir::MemoryEffects::EffectInstance> effectsA,
|
|
llvm::ArrayRef<mlir::MemoryEffects::EffectInstance> effectsB) {
|
|
fir::AliasAnalysis aliasAnalysis;
|
|
// (RAW || WAW) || (WAR || WAW).
|
|
return anyRAWorWAW(effectsA, effectsB, aliasAnalysis) ||
|
|
anyRAWorWAW(effectsB, effectsA, aliasAnalysis);
|
|
}
|
|
|
|
/// Could there be any write effects in "effects"?
|
|
static bool
|
|
anyWrite(llvm::ArrayRef<mlir::MemoryEffects::EffectInstance> effects) {
|
|
return llvm::any_of(
|
|
effects, [](const mlir::MemoryEffects::EffectInstance &effect) {
|
|
return mlir::isa<mlir::MemoryEffects::Write>(effect.getEffect());
|
|
});
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Scheduling Implementation : Scheduler class implementation
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
void Scheduler::startSchedulingAssignment(hlfir::RegionAssignOp assign,
|
|
bool leafRegionsMayOnlyRead) {
|
|
gatherAssignEffects(assign, leafRegionsMayOnlyRead, assignEffects);
|
|
// Unconditionally collect effects of the evaluations of LHS and RHS
|
|
// in case they need to be analyzed for any parent that might be
|
|
// affected by conflicts of these evaluations.
|
|
// This collection migth be skipped, if there are no such parents,
|
|
// but for the time being we run it always.
|
|
gatherAssignEvaluationEffects(assign, leafRegionsMayOnlyRead,
|
|
assignEvaluateEffects);
|
|
}
|
|
|
|
void Scheduler::saveEvaluationIfConflict(mlir::Region &yieldRegion,
|
|
bool leafRegionsMayOnlyRead,
|
|
bool yieldIsImplicitRead,
|
|
bool evaluationsMayConflict) {
|
|
// If the region evaluation was previously executed and saved, the saved
|
|
// value will be used when evaluating the current assignment and this has
|
|
// no effects in the current assignment evaluation.
|
|
if (savedRegions.contains(&yieldRegion))
|
|
return;
|
|
llvm::SmallVector<mlir::MemoryEffects::EffectInstance> effects;
|
|
gatherMemoryEffects(yieldRegion, leafRegionsMayOnlyRead, effects);
|
|
// Yield has no effect as such, but in the context of order assignments.
|
|
// The order assignments will usually read the yielded entity (except for
|
|
// the yielded assignments LHS that is only read if this is an assignment
|
|
// with a finalizer, or a user defined assignment where the LHS is
|
|
// intent(inout)).
|
|
if (yieldIsImplicitRead) {
|
|
mlir::OpOperand *entity = getYieldedEntity(yieldRegion);
|
|
if (entity && hlfir::isFortranVariableType(entity->get().getType()))
|
|
effects.emplace_back(mlir::MemoryEffects::Read::get(), entity);
|
|
}
|
|
if (!leafRegionsMayOnlyRead && anyWrite(effects)) {
|
|
// Region with write effect must be executed only once: save it the first
|
|
// time it is encountered.
|
|
saveEvaluation(yieldRegion, effects, /*anyWrite=*/true);
|
|
} else if (conflict(effects, assignEffects)) {
|
|
// Region that conflicts with the current assignments must be fully
|
|
// evaluated and saved before doing the assignment (Note that it may
|
|
// have already have been evaluated without saving it before, but this
|
|
// implies that it never conflicted with a prior assignment, so its value
|
|
// should be the same.)
|
|
saveEvaluation(yieldRegion, effects, /*anyWrite=*/false);
|
|
} else if (evaluationsMayConflict &&
|
|
conflict(effects, assignEvaluateEffects)) {
|
|
// If evaluations of the assignment may conflict with the yield
|
|
// evaluations, we have to save yield evaluation.
|
|
// For example, a WHERE mask might be written by the masked assignment
|
|
// evaluations, and it has to be saved in this case:
|
|
// where (mask) r = f() ! function f modifies mask
|
|
saveEvaluation(yieldRegion, effects, anyWrite(effects));
|
|
} else {
|
|
// Can be executed while doing the assignment.
|
|
independentEvaluationEffects.append(effects.begin(), effects.end());
|
|
}
|
|
}
|
|
|
|
void Scheduler::saveEvaluation(
|
|
mlir::Region &yieldRegion,
|
|
llvm::ArrayRef<mlir::MemoryEffects::EffectInstance> effects,
|
|
bool anyWrite) {
|
|
savedAnyRegionForCurrentAssignment = true;
|
|
if (anyWrite) {
|
|
// Create a new run just for regions with side effect. Further analysis
|
|
// could try to prove the effects do not conflict with the previous
|
|
// schedule.
|
|
schedule.emplace_back(hlfir::Run{});
|
|
currentRunIsReadOnly = false;
|
|
} else if (!currentRunIsReadOnly) {
|
|
// For now, do not try to fuse an evaluation with a previous
|
|
// run that contains any write effects. One could try to prove
|
|
// that "effects" do not conflict with the current run assignments.
|
|
schedule.emplace_back(hlfir::Run{});
|
|
currentRunIsReadOnly = true;
|
|
}
|
|
// Otherwise, save the yielded entity in the current run, that already
|
|
// saving other read only entities.
|
|
schedule.back().actions.emplace_back(hlfir::SaveEntity{&yieldRegion});
|
|
// The run to save the yielded entity will need to evaluate all the unsaved
|
|
// parent control or masks. Note that these effects may already be in the
|
|
// current run memoryEffects, but it is just easier always add them, even if
|
|
// this may add them again.
|
|
schedule.back().memoryEffects.append(parentEvaluationEffects.begin(),
|
|
parentEvaluationEffects.end());
|
|
schedule.back().memoryEffects.append(effects.begin(), effects.end());
|
|
savedRegions.insert(&yieldRegion);
|
|
LLVM_DEBUG(
|
|
logSaveEvaluation(llvm::dbgs(), schedule.size(), yieldRegion, anyWrite););
|
|
}
|
|
|
|
bool Scheduler::canFuseAssignmentWithPreviousRun() {
|
|
// If a region was saved for the current assignment, the previous
|
|
// run is already known to conflict. Skip the analysis.
|
|
if (savedAnyRegionForCurrentAssignment || schedule.empty())
|
|
return false;
|
|
auto &previousRunEffects = schedule.back().memoryEffects;
|
|
return !conflict(previousRunEffects, assignEffects) &&
|
|
!conflict(previousRunEffects, parentEvaluationEffects) &&
|
|
!conflict(previousRunEffects, independentEvaluationEffects);
|
|
}
|
|
|
|
void Scheduler::finishSchedulingAssignment(hlfir::RegionAssignOp assign) {
|
|
// For now, always schedule each assignment in its own run. They could
|
|
// be done as part of previous assignment runs if it is proven they have
|
|
// no conflicting effects.
|
|
currentRunIsReadOnly = false;
|
|
if (!tryFusingAssignments || !canFuseAssignmentWithPreviousRun())
|
|
schedule.emplace_back(hlfir::Run{});
|
|
schedule.back().actions.emplace_back(assign);
|
|
// TODO: when fusing, it would probably be best to filter the
|
|
// parentEvaluationEffects that already in the previous run effects (since
|
|
// assignments may share the same parents), otherwise, this can make the
|
|
// conflict() calls more and more expensive.
|
|
schedule.back().memoryEffects.append(parentEvaluationEffects.begin(),
|
|
parentEvaluationEffects.end());
|
|
schedule.back().memoryEffects.append(assignEffects.begin(),
|
|
assignEffects.end());
|
|
assignEffects.clear();
|
|
assignEvaluateEffects.clear();
|
|
parentEvaluationEffects.clear();
|
|
independentEvaluationEffects.clear();
|
|
savedAnyRegionForCurrentAssignment = false;
|
|
LLVM_DEBUG(logAssignmentEvaluation(llvm::dbgs(), schedule.size(), assign));
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Scheduling Implementation : driving the Scheduler in the assignment tree.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
/// Gather the hlfir.region_assign nested directly and indirectly inside root in
|
|
/// execution order.
|
|
static void
|
|
gatherAssignments(hlfir::OrderedAssignmentTreeOpInterface root,
|
|
llvm::SmallVector<hlfir::RegionAssignOp> &assignments) {
|
|
llvm::SmallVector<mlir::Operation *> nodeStack{root.getOperation()};
|
|
while (!nodeStack.empty()) {
|
|
mlir::Operation *node = nodeStack.pop_back_val();
|
|
if (auto regionAssign = mlir::dyn_cast<hlfir::RegionAssignOp>(node)) {
|
|
assignments.push_back(regionAssign);
|
|
continue;
|
|
}
|
|
auto nodeIface =
|
|
mlir::dyn_cast<hlfir::OrderedAssignmentTreeOpInterface>(node);
|
|
if (nodeIface)
|
|
if (mlir::Block *block = nodeIface.getSubTreeBlock())
|
|
for (mlir::Operation &op : llvm::reverse(block->getOperations()))
|
|
nodeStack.push_back(&op);
|
|
}
|
|
}
|
|
|
|
/// Gather the parents of (not included) \p node in reverse execution order.
|
|
static void gatherParents(
|
|
hlfir::OrderedAssignmentTreeOpInterface node,
|
|
llvm::SmallVectorImpl<hlfir::OrderedAssignmentTreeOpInterface> &parents) {
|
|
while (node) {
|
|
auto parent =
|
|
mlir::dyn_cast_or_null<hlfir::OrderedAssignmentTreeOpInterface>(
|
|
node->getParentOp());
|
|
if (parent && parent.getSubTreeRegion() == node->getParentRegion()) {
|
|
parents.push_back(parent);
|
|
node = parent;
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Build the list of the parent nodes for this assignment. The list is built
|
|
// from the closest parent until the ordered assignment tree root (this is the
|
|
// revere of their execution order).
|
|
static void gatherAssignmentParents(
|
|
hlfir::RegionAssignOp assign,
|
|
llvm::SmallVectorImpl<hlfir::OrderedAssignmentTreeOpInterface> &parents) {
|
|
gatherParents(mlir::cast<hlfir::OrderedAssignmentTreeOpInterface>(
|
|
assign.getOperation()),
|
|
parents);
|
|
}
|
|
|
|
hlfir::Schedule
|
|
hlfir::buildEvaluationSchedule(hlfir::OrderedAssignmentTreeOpInterface root,
|
|
bool tryFusingAssignments) {
|
|
LLVM_DEBUG(logStartScheduling(llvm::dbgs(), root););
|
|
// The expressions inside an hlfir.forall must be pure (with the Fortran
|
|
// definition of pure). This is not a commitment that there are no operation
|
|
// with write effect in the regions: entities local to the region may still
|
|
// be written to (e.g., a temporary accumulator implementing SUM). This is
|
|
// a commitment that no write effect will affect the scheduling problem, and
|
|
// that all write effect caught by MLIR analysis can be ignored for the
|
|
// current problem.
|
|
const bool leafRegionsMayOnlyRead =
|
|
mlir::isa<hlfir::ForallOp>(root.getOperation());
|
|
|
|
// Loop through the assignments and schedule them.
|
|
Scheduler scheduler(tryFusingAssignments);
|
|
llvm::SmallVector<hlfir::RegionAssignOp> assignments;
|
|
gatherAssignments(root, assignments);
|
|
for (hlfir::RegionAssignOp assign : assignments) {
|
|
scheduler.startSchedulingAssignment(assign, leafRegionsMayOnlyRead);
|
|
// Go through the list of parents (not including the current
|
|
// hlfir.region_assign) in Fortran execution order so that any parent leaf
|
|
// region that must be saved is saved in order.
|
|
llvm::SmallVector<hlfir::OrderedAssignmentTreeOpInterface> parents;
|
|
gatherAssignmentParents(assign, parents);
|
|
for (hlfir::OrderedAssignmentTreeOpInterface parent :
|
|
llvm::reverse(parents)) {
|
|
scheduler.startIndependentEvaluationGroup();
|
|
llvm::SmallVector<mlir::Region *, 4> yieldRegions;
|
|
parent.getLeafRegions(yieldRegions);
|
|
// TODO: is this really limited to WHERE/ELSEWHERE?
|
|
bool evaluationsMayConflict = mlir::isa<hlfir::WhereOp>(parent) ||
|
|
mlir::isa<hlfir::ElseWhereOp>(parent);
|
|
for (mlir::Region *yieldRegion : yieldRegions)
|
|
scheduler.saveEvaluationIfConflict(*yieldRegion, leafRegionsMayOnlyRead,
|
|
/*yieldIsImplicitRead=*/true,
|
|
evaluationsMayConflict);
|
|
scheduler.finishIndependentEvaluationGroup();
|
|
}
|
|
// Look for conflicts between the RHS/LHS evaluation and the assignments.
|
|
// The LHS yield has no implicit read effect on the produced variable (the
|
|
// variable is not read before the assignment).
|
|
scheduler.startIndependentEvaluationGroup();
|
|
scheduler.saveEvaluationIfConflict(assign.getRhsRegion(),
|
|
leafRegionsMayOnlyRead);
|
|
// There is no point to save the LHS outside of Forall and assignment to a
|
|
// vector subscripted LHS because the LHS is already fully evaluated and
|
|
// saved in the resulting SSA address value (that may be a descriptor or
|
|
// descriptor address).
|
|
if (mlir::isa<hlfir::ForallOp>(root.getOperation()) ||
|
|
mlir::isa<hlfir::ElementalAddrOp>(assign.getLhsRegion().back().back()))
|
|
scheduler.saveEvaluationIfConflict(assign.getLhsRegion(),
|
|
leafRegionsMayOnlyRead,
|
|
/*yieldIsImplicitRead=*/false);
|
|
scheduler.finishIndependentEvaluationGroup();
|
|
scheduler.finishSchedulingAssignment(assign);
|
|
}
|
|
return scheduler.moveSchedule();
|
|
}
|
|
|
|
mlir::Value hlfir::SaveEntity::getSavedValue() {
|
|
mlir::OpOperand *saved = getYieldedEntity(*yieldRegion);
|
|
assert(saved && "SaveEntity must contain region terminated by YieldOp");
|
|
return saved->get();
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// Debug and test logging implementation
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
static llvm::raw_ostream &printRegionId(llvm::raw_ostream &os,
|
|
mlir::Region &yieldRegion) {
|
|
mlir::Operation *parent = yieldRegion.getParentOp();
|
|
if (auto forall = mlir::dyn_cast<hlfir::ForallOp>(parent)) {
|
|
if (&forall.getLbRegion() == &yieldRegion)
|
|
os << "lb";
|
|
else if (&forall.getUbRegion() == &yieldRegion)
|
|
os << "ub";
|
|
else if (&forall.getStepRegion() == &yieldRegion)
|
|
os << "step";
|
|
} else if (auto assign = mlir::dyn_cast<hlfir::ForallMaskOp>(parent)) {
|
|
if (&assign.getMaskRegion() == &yieldRegion)
|
|
os << "mask";
|
|
} else if (auto assign = mlir::dyn_cast<hlfir::RegionAssignOp>(parent)) {
|
|
if (&assign.getRhsRegion() == &yieldRegion)
|
|
os << "rhs";
|
|
else if (&assign.getLhsRegion() == &yieldRegion)
|
|
os << "lhs";
|
|
} else if (auto where = mlir::dyn_cast<hlfir::WhereOp>(parent)) {
|
|
if (&where.getMaskRegion() == &yieldRegion)
|
|
os << "mask";
|
|
} else if (auto elseWhereOp = mlir::dyn_cast<hlfir::ElseWhereOp>(parent)) {
|
|
if (&elseWhereOp.getMaskRegion() == &yieldRegion)
|
|
os << "mask";
|
|
} else {
|
|
os << "unknown";
|
|
}
|
|
return os;
|
|
}
|
|
|
|
static llvm::raw_ostream &
|
|
printNodeIndexInBody(llvm::raw_ostream &os,
|
|
hlfir::OrderedAssignmentTreeOpInterface node,
|
|
hlfir::OrderedAssignmentTreeOpInterface parent) {
|
|
if (!parent || !parent.getSubTreeRegion())
|
|
return os;
|
|
mlir::Operation *nodeOp = node.getOperation();
|
|
unsigned index = 1;
|
|
for (mlir::Operation &op : parent.getSubTreeRegion()->getOps())
|
|
if (nodeOp == &op) {
|
|
return os << index;
|
|
} else if (nodeOp->getName() == op.getName()) {
|
|
++index;
|
|
}
|
|
return os;
|
|
}
|
|
|
|
static llvm::raw_ostream &printNodePath(llvm::raw_ostream &os,
|
|
mlir::Operation *op) {
|
|
auto node =
|
|
mlir::dyn_cast_or_null<hlfir::OrderedAssignmentTreeOpInterface>(op);
|
|
if (!node) {
|
|
os << "unknown node";
|
|
return os;
|
|
}
|
|
llvm::SmallVector<hlfir::OrderedAssignmentTreeOpInterface> parents;
|
|
gatherParents(node, parents);
|
|
hlfir::OrderedAssignmentTreeOpInterface previousParent;
|
|
for (auto parent : llvm::reverse(parents)) {
|
|
os << parent->getName().stripDialect();
|
|
printNodeIndexInBody(os, parent, previousParent) << "/";
|
|
previousParent = parent;
|
|
}
|
|
os << node->getName().stripDialect();
|
|
return printNodeIndexInBody(os, node, previousParent);
|
|
}
|
|
|
|
static llvm::raw_ostream &printRegionPath(llvm::raw_ostream &os,
|
|
mlir::Region &yieldRegion) {
|
|
printNodePath(os, yieldRegion.getParentOp()) << "/";
|
|
return printRegionId(os, yieldRegion);
|
|
}
|
|
|
|
static void LLVM_ATTRIBUTE_UNUSED logSaveEvaluation(llvm::raw_ostream &os,
|
|
unsigned runid,
|
|
mlir::Region &yieldRegion,
|
|
bool anyWrite) {
|
|
os << "run " << runid << " save " << (anyWrite ? "(w)" : " ") << ": ";
|
|
printRegionPath(os, yieldRegion) << "\n";
|
|
}
|
|
|
|
static void LLVM_ATTRIBUTE_UNUSED logAssignmentEvaluation(
|
|
llvm::raw_ostream &os, unsigned runid, hlfir::RegionAssignOp assign) {
|
|
os << "run " << runid << " evaluate: ";
|
|
printNodePath(os, assign.getOperation()) << "\n";
|
|
}
|
|
|
|
static void LLVM_ATTRIBUTE_UNUSED logConflict(llvm::raw_ostream &os,
|
|
mlir::Value writtenOrReadVarA,
|
|
mlir::Value writtenVarB) {
|
|
auto printIfValue = [&](mlir::Value var) -> llvm::raw_ostream & {
|
|
if (!var)
|
|
return os << "<unknown>";
|
|
return os << var;
|
|
};
|
|
os << "conflict: R/W: ";
|
|
printIfValue(writtenOrReadVarA) << " W:";
|
|
printIfValue(writtenVarB) << "\n";
|
|
}
|
|
|
|
static void LLVM_ATTRIBUTE_UNUSED logStartScheduling(
|
|
llvm::raw_ostream &os, hlfir::OrderedAssignmentTreeOpInterface root) {
|
|
os << "------------ scheduling ";
|
|
printNodePath(os, root.getOperation());
|
|
if (auto funcOp = root->getParentOfType<mlir::func::FuncOp>())
|
|
os << " in " << funcOp.getSymName() << " ";
|
|
os << "------------\n";
|
|
}
|
|
|
|
static void LLVM_ATTRIBUTE_UNUSED logIfUnkownEffectValue(
|
|
llvm::raw_ostream &os, mlir::MemoryEffects::EffectInstance effect,
|
|
mlir::Operation &op) {
|
|
if (effect.getValue() != nullptr)
|
|
return;
|
|
os << "unknown effected value (";
|
|
os << (mlir::isa<mlir::MemoryEffects::Read>(effect.getEffect()) ? "R" : "W");
|
|
os << "): " << op << "\n";
|
|
}
|