fdfeea5bd6
Currently, there are some inconsistencies to how clause arguments are named in the OpenMP dialect. Additionally, the clause operand structures associated to them also diverge in certain cases. The purpose of this patch is to normalize argument names across all `OpenMP_Clause` tablegen definitions and clause operand structures. This has the benefit of providing more consistent representations for clauses in the dialect, but the main short-term advantage is that it enables the development of an OpenMP-specific tablegen backend to automatically generate the clause operand structures without breaking dependent code. The main re-naming decisions made in this patch are the following: - Variadic arguments (i.e. multiple values) have the "_vars" suffix. This and other similar suffixes are removed from array attribute arguments. - Individual required or optional value arguments do not have any suffix added to them (e.g. "val", "var", "expr", ...), except for `if` which would otherwise result in an invalid C++ variable name. - The associated clause's name is prepended to argument names that don't already contain it as part of its name. This avoids future collisions between arguments named the same way on different clauses and adding both clauses to the same operation. - Privatization and reduction related arguments that contain lists of symbols pointing to privatizer/reducer operations use the "_syms" suffix. This removes the inconsistencies between the names for "copyprivate_funcs", "[in]reductions", "privatizers", etc. - General improvements to names, replacement of camel case for snake case everywhere, etc. - Renaming of operation-associated operand structures to use the "Operands" suffix in place of "ClauseOps", to better differentiate between clause operand structures and operation operand structures. - Fields on clause operand structures are sorted according to the tablegen definition of the same clause. The assembly format for a few arguments is updated to better reflect the clause they are associated with: - `chunk_size` -> `dist_schedule_chunk_size` - `grain_size` -> `grainsize` - `simd` -> `par_level_simd`
607 lines
23 KiB
C++
607 lines
23 KiB
C++
//===-- DataSharingProcessor.cpp --------------------------------*- C++ -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "DataSharingProcessor.h"
|
|
|
|
#include "Utils.h"
|
|
#include "flang/Lower/ConvertVariable.h"
|
|
#include "flang/Lower/PFTBuilder.h"
|
|
#include "flang/Lower/SymbolMap.h"
|
|
#include "flang/Optimizer/Builder/HLFIRTools.h"
|
|
#include "flang/Optimizer/Builder/Todo.h"
|
|
#include "flang/Optimizer/HLFIR/HLFIROps.h"
|
|
#include "flang/Semantics/tools.h"
|
|
|
|
namespace Fortran {
|
|
namespace lower {
|
|
namespace omp {
|
|
bool DataSharingProcessor::OMPConstructSymbolVisitor::isSymbolDefineBy(
|
|
const semantics::Symbol *symbol, lower::pft::Evaluation &eval) const {
|
|
return eval.visit(
|
|
common::visitors{[&](const parser::OpenMPConstruct &functionParserNode) {
|
|
return symDefMap.count(symbol) &&
|
|
symDefMap.at(symbol) == &functionParserNode;
|
|
},
|
|
[](const auto &functionParserNode) { return false; }});
|
|
}
|
|
|
|
DataSharingProcessor::DataSharingProcessor(
|
|
lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
|
|
const List<Clause> &clauses, lower::pft::Evaluation &eval,
|
|
bool shouldCollectPreDeterminedSymbols, bool useDelayedPrivatization,
|
|
lower::SymMap *symTable)
|
|
: hasLastPrivateOp(false), converter(converter), semaCtx(semaCtx),
|
|
firOpBuilder(converter.getFirOpBuilder()), clauses(clauses), eval(eval),
|
|
shouldCollectPreDeterminedSymbols(shouldCollectPreDeterminedSymbols),
|
|
useDelayedPrivatization(useDelayedPrivatization), symTable(symTable),
|
|
visitor() {
|
|
eval.visit([&](const auto &functionParserNode) {
|
|
parser::Walk(functionParserNode, visitor);
|
|
});
|
|
}
|
|
|
|
void DataSharingProcessor::processStep1(
|
|
mlir::omp::PrivateClauseOps *clauseOps) {
|
|
collectSymbolsForPrivatization();
|
|
collectDefaultSymbols();
|
|
collectImplicitSymbols();
|
|
collectPreDeterminedSymbols();
|
|
|
|
privatize(clauseOps);
|
|
|
|
insertBarrier();
|
|
}
|
|
|
|
void DataSharingProcessor::processStep2(mlir::Operation *op, bool isLoop) {
|
|
// 'sections' lastprivate is handled by genOMP()
|
|
if (!mlir::isa<mlir::omp::SectionsOp>(op)) {
|
|
insPt = firOpBuilder.saveInsertionPoint();
|
|
copyLastPrivatize(op);
|
|
firOpBuilder.restoreInsertionPoint(insPt);
|
|
}
|
|
|
|
if (isLoop) {
|
|
// push deallocs out of the loop
|
|
firOpBuilder.setInsertionPointAfter(op);
|
|
insertDeallocs();
|
|
} else {
|
|
mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
|
|
insertDeallocs();
|
|
}
|
|
}
|
|
|
|
void DataSharingProcessor::insertDeallocs() {
|
|
for (const semantics::Symbol *sym : allPrivatizedSymbols)
|
|
if (semantics::IsAllocatable(sym->GetUltimate())) {
|
|
if (!useDelayedPrivatization) {
|
|
converter.createHostAssociateVarCloneDealloc(*sym);
|
|
continue;
|
|
}
|
|
|
|
lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
|
|
assert(hsb && "Host symbol box not found");
|
|
mlir::Type symType = hsb.getAddr().getType();
|
|
mlir::Location symLoc = hsb.getAddr().getLoc();
|
|
fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym);
|
|
mlir::omp::PrivateClauseOp privatizer = symToPrivatizer.at(sym);
|
|
|
|
symTable->pushScope();
|
|
|
|
mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
|
|
|
|
mlir::Region &deallocRegion = privatizer.getDeallocRegion();
|
|
fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
|
|
mlir::Block *deallocEntryBlock = firOpBuilder.createBlock(
|
|
&deallocRegion, /*insertPt=*/{}, symType, symLoc);
|
|
|
|
firOpBuilder.setInsertionPointToEnd(deallocEntryBlock);
|
|
symTable->addSymbol(*sym,
|
|
fir::substBase(symExV, deallocRegion.getArgument(0)));
|
|
|
|
converter.createHostAssociateVarCloneDealloc(*sym);
|
|
firOpBuilder.create<mlir::omp::YieldOp>(hsb.getAddr().getLoc());
|
|
|
|
symTable->popScope();
|
|
}
|
|
}
|
|
|
|
void DataSharingProcessor::cloneSymbol(const semantics::Symbol *sym) {
|
|
bool success = converter.createHostAssociateVarClone(*sym);
|
|
(void)success;
|
|
assert(success && "Privatization failed due to existing binding");
|
|
|
|
bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate);
|
|
if (!isFirstPrivate &&
|
|
Fortran::lower::hasDefaultInitialization(sym->GetUltimate()))
|
|
Fortran::lower::defaultInitializeAtRuntime(converter, *sym, *symTable);
|
|
}
|
|
|
|
void DataSharingProcessor::copyFirstPrivateSymbol(
|
|
const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *copyAssignIP) {
|
|
if (sym->test(semantics::Symbol::Flag::OmpFirstPrivate))
|
|
converter.copyHostAssociateVar(*sym, copyAssignIP);
|
|
}
|
|
|
|
void DataSharingProcessor::copyLastPrivateSymbol(
|
|
const semantics::Symbol *sym,
|
|
[[maybe_unused]] mlir::OpBuilder::InsertPoint *lastPrivIP) {
|
|
if (sym->test(semantics::Symbol::Flag::OmpLastPrivate)) {
|
|
bool allocatable = semantics::IsAllocatable(sym->GetUltimate());
|
|
if (!allocatable) {
|
|
converter.copyHostAssociateVar(*sym, lastPrivIP);
|
|
return;
|
|
}
|
|
|
|
// copyHostAssociateVar doesn't work properly if the privatised copy was
|
|
// reallocated (e.g. by assignment): it will only copy if the ultimate
|
|
// symbol was already allocated, and it only copies data so any reallocated
|
|
// lengths etc are lost
|
|
|
|
// 1) Fetch the original copy of the variable.
|
|
assert(sym->has<Fortran::semantics::HostAssocDetails>() &&
|
|
"No host-association found");
|
|
const Fortran::semantics::Symbol &hsym = sym->GetUltimate();
|
|
Fortran::lower::SymbolBox hsb = symTable->lookupOneLevelUpSymbol(hsym);
|
|
assert(hsb && "Host symbol box not found");
|
|
|
|
// 2) Fetch the copied one that will mask the original.
|
|
Fortran::lower::SymbolBox sb = symTable->shallowLookupSymbol(sym);
|
|
assert(sb && "Host-associated symbol box not found");
|
|
assert(hsb.getAddr() != sb.getAddr() &&
|
|
"Host and associated symbol boxes are the same");
|
|
|
|
// 3) Perform the assignment.
|
|
fir::FirOpBuilder &builder = converter.getFirOpBuilder();
|
|
mlir::Location loc = converter.genLocation(sym->name());
|
|
mlir::OpBuilder::InsertPoint insPt = builder.saveInsertionPoint();
|
|
if (lastPrivIP && lastPrivIP->isSet())
|
|
builder.restoreInsertionPoint(*lastPrivIP);
|
|
else
|
|
builder.setInsertionPointAfter(sb.getAddr().getDefiningOp());
|
|
|
|
hlfir::Entity dst{hsb.getAddr()};
|
|
hlfir::Entity src{sb.getAddr()};
|
|
builder.create<hlfir::AssignOp>(
|
|
loc, src, dst, /*isWholeAllocatableAssignment=*/allocatable,
|
|
/*keepLhsLengthInAllocatableAssignment=*/false,
|
|
/*temporary_lhs=*/false);
|
|
|
|
if (lastPrivIP && lastPrivIP->isSet() &&
|
|
sym->test(Fortran::semantics::Symbol::Flag::OmpLastPrivate)) {
|
|
builder.restoreInsertionPoint(insPt);
|
|
}
|
|
}
|
|
}
|
|
|
|
void DataSharingProcessor::collectOmpObjectListSymbol(
|
|
const omp::ObjectList &objects,
|
|
llvm::SetVector<const semantics::Symbol *> &symbolSet) {
|
|
for (const omp::Object &object : objects)
|
|
symbolSet.insert(object.sym());
|
|
}
|
|
|
|
void DataSharingProcessor::collectSymbolsForPrivatization() {
|
|
for (const omp::Clause &clause : clauses) {
|
|
if (const auto &privateClause =
|
|
std::get_if<omp::clause::Private>(&clause.u)) {
|
|
collectOmpObjectListSymbol(privateClause->v, explicitlyPrivatizedSymbols);
|
|
} else if (const auto &firstPrivateClause =
|
|
std::get_if<omp::clause::Firstprivate>(&clause.u)) {
|
|
collectOmpObjectListSymbol(firstPrivateClause->v,
|
|
explicitlyPrivatizedSymbols);
|
|
} else if (const auto &lastPrivateClause =
|
|
std::get_if<omp::clause::Lastprivate>(&clause.u)) {
|
|
const ObjectList &objects = std::get<ObjectList>(lastPrivateClause->t);
|
|
collectOmpObjectListSymbol(objects, explicitlyPrivatizedSymbols);
|
|
hasLastPrivateOp = true;
|
|
}
|
|
}
|
|
|
|
for (auto *sym : explicitlyPrivatizedSymbols)
|
|
allPrivatizedSymbols.insert(sym);
|
|
}
|
|
|
|
bool DataSharingProcessor::needBarrier() {
|
|
// Emit implicit barrier to synchronize threads and avoid data races on
|
|
// initialization of firstprivate variables and post-update of lastprivate
|
|
// variables.
|
|
// Emit implicit barrier for linear clause. Maybe on somewhere else.
|
|
for (const semantics::Symbol *sym : allPrivatizedSymbols) {
|
|
if (sym->test(semantics::Symbol::Flag::OmpFirstPrivate) &&
|
|
sym->test(semantics::Symbol::Flag::OmpLastPrivate))
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void DataSharingProcessor::insertBarrier() {
|
|
if (needBarrier())
|
|
firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation());
|
|
}
|
|
|
|
void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
|
|
mlir::omp::LoopNestOp loopOp;
|
|
if (auto wrapper = mlir::dyn_cast<mlir::omp::LoopWrapperInterface>(op))
|
|
loopOp = wrapper.isWrapper()
|
|
? mlir::cast<mlir::omp::LoopNestOp>(wrapper.getWrappedLoop())
|
|
: nullptr;
|
|
|
|
bool cmpCreated = false;
|
|
mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
|
|
for (const omp::Clause &clause : clauses) {
|
|
if (clause.id != llvm::omp::OMPC_lastprivate)
|
|
continue;
|
|
// TODO: Add lastprivate support for simd construct
|
|
if (mlir::isa<mlir::omp::WsloopOp>(op)) {
|
|
// Update the original variable just before exiting the worksharing
|
|
// loop. Conversion as follows:
|
|
//
|
|
// omp.wsloop { omp.wsloop {
|
|
// omp.loop_nest { omp.loop_nest {
|
|
// ... ...
|
|
// store ===> store
|
|
// omp.yield %v = arith.addi %iv, %step
|
|
// } %cmp = %step < 0 ? %v < %ub : %v > %ub
|
|
// omp.terminator fir.if %cmp {
|
|
// } fir.store %v to %loopIV
|
|
// ^%lpv_update_blk:
|
|
// }
|
|
// omp.yield
|
|
// }
|
|
// omp.terminator
|
|
// }
|
|
|
|
// Only generate the compare once in presence of multiple LastPrivate
|
|
// clauses.
|
|
if (cmpCreated)
|
|
continue;
|
|
cmpCreated = true;
|
|
|
|
mlir::Location loc = loopOp.getLoc();
|
|
mlir::Operation *lastOper = loopOp.getRegion().back().getTerminator();
|
|
firOpBuilder.setInsertionPoint(lastOper);
|
|
|
|
mlir::Value cmpOp;
|
|
llvm::SmallVector<mlir::Value> vs;
|
|
vs.reserve(loopOp.getIVs().size());
|
|
for (auto [iv, ub, step] :
|
|
llvm::zip_equal(loopOp.getIVs(), loopOp.getCollapseUpperBounds(),
|
|
loopOp.getCollapseSteps())) {
|
|
// v = iv + step
|
|
// cmp = step < 0 ? v < ub : v > ub
|
|
mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step);
|
|
vs.push_back(v);
|
|
mlir::Value zero =
|
|
firOpBuilder.createIntegerConstant(loc, step.getType(), 0);
|
|
mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>(
|
|
loc, mlir::arith::CmpIPredicate::slt, step, zero);
|
|
mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>(
|
|
loc, mlir::arith::CmpIPredicate::slt, v, ub);
|
|
mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>(
|
|
loc, mlir::arith::CmpIPredicate::sgt, v, ub);
|
|
mlir::Value icmpOp = firOpBuilder.create<mlir::arith::SelectOp>(
|
|
loc, negativeStep, vLT, vGT);
|
|
|
|
if (cmpOp) {
|
|
cmpOp = firOpBuilder.create<mlir::arith::AndIOp>(loc, cmpOp, icmpOp);
|
|
} else {
|
|
cmpOp = icmpOp;
|
|
}
|
|
}
|
|
|
|
auto ifOp = firOpBuilder.create<fir::IfOp>(loc, cmpOp, /*else*/ false);
|
|
firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
|
|
for (auto [v, loopIV] : llvm::zip_equal(vs, loopIVs)) {
|
|
assert(loopIV && "loopIV was not set");
|
|
firOpBuilder.createStoreWithConvert(loc, v, loopIV);
|
|
}
|
|
lastPrivIP = firOpBuilder.saveInsertionPoint();
|
|
} else if (mlir::isa<mlir::omp::SectionsOp>(op)) {
|
|
// Already handled by genOMP()
|
|
} else {
|
|
TODO(converter.getCurrentLocation(),
|
|
"lastprivate clause in constructs other than "
|
|
"simd/worksharing-loop");
|
|
}
|
|
}
|
|
}
|
|
|
|
static const parser::CharBlock *
|
|
getSource(const semantics::SemanticsContext &semaCtx,
|
|
const lower::pft::Evaluation &eval) {
|
|
const parser::CharBlock *source = nullptr;
|
|
|
|
auto ompConsVisit = [&](const parser::OpenMPConstruct &x) {
|
|
std::visit(common::visitors{
|
|
[&](const parser::OpenMPSectionsConstruct &x) {
|
|
source = &std::get<0>(x.t).source;
|
|
},
|
|
[&](const parser::OpenMPLoopConstruct &x) {
|
|
source = &std::get<0>(x.t).source;
|
|
},
|
|
[&](const parser::OpenMPBlockConstruct &x) {
|
|
source = &std::get<0>(x.t).source;
|
|
},
|
|
[&](const parser::OpenMPCriticalConstruct &x) {
|
|
source = &std::get<0>(x.t).source;
|
|
},
|
|
[&](const parser::OpenMPAtomicConstruct &x) {
|
|
std::visit([&](const auto &x) { source = &x.source; },
|
|
x.u);
|
|
},
|
|
[&](const auto &x) { source = &x.source; },
|
|
},
|
|
x.u);
|
|
};
|
|
|
|
eval.visit(common::visitors{
|
|
[&](const parser::OpenMPConstruct &x) { ompConsVisit(x); },
|
|
[&](const parser::OpenMPDeclarativeConstruct &x) { source = &x.source; },
|
|
[&](const parser::OmpEndLoopDirective &x) { source = &x.source; },
|
|
[&](const auto &x) {},
|
|
});
|
|
|
|
return source;
|
|
}
|
|
|
|
void DataSharingProcessor::collectSymbolsInNestedRegions(
|
|
lower::pft::Evaluation &eval, semantics::Symbol::Flag flag,
|
|
llvm::SetVector<const semantics::Symbol *> &symbolsInNestedRegions) {
|
|
for (lower::pft::Evaluation &nestedEval : eval.getNestedEvaluations()) {
|
|
if (nestedEval.hasNestedEvaluations()) {
|
|
if (nestedEval.isConstruct())
|
|
// Recursively look for OpenMP constructs within `nestedEval`'s region
|
|
collectSymbolsInNestedRegions(nestedEval, flag, symbolsInNestedRegions);
|
|
else {
|
|
converter.collectSymbolSet(nestedEval, symbolsInNestedRegions, flag,
|
|
/*collectSymbols=*/true,
|
|
/*collectHostAssociatedSymbols=*/false);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Collect symbols to be default privatized in two steps.
|
|
// In step 1, collect all symbols in `eval` that match `flag` into
|
|
// `defaultSymbols`. In step 2, for nested constructs (if any), if and only if
|
|
// the nested construct is an OpenMP construct, collect those nested
|
|
// symbols skipping host associated symbols into `symbolsInNestedRegions`.
|
|
// Later, in current context, all symbols in the set
|
|
// `defaultSymbols` - `symbolsInNestedRegions` will be privatized.
|
|
void DataSharingProcessor::collectSymbols(
|
|
semantics::Symbol::Flag flag,
|
|
llvm::SetVector<const semantics::Symbol *> &symbols) {
|
|
// Collect all scopes associated with 'eval'.
|
|
llvm::SetVector<const semantics::Scope *> clauseScopes;
|
|
std::function<void(const semantics::Scope *)> collectScopes =
|
|
[&](const semantics::Scope *scope) {
|
|
clauseScopes.insert(scope);
|
|
for (const semantics::Scope &child : scope->children())
|
|
collectScopes(&child);
|
|
};
|
|
const parser::CharBlock *source =
|
|
clauses.empty() ? getSource(semaCtx, eval) : &clauses.front().source;
|
|
const semantics::Scope *curScope = nullptr;
|
|
if (source && !source->empty()) {
|
|
curScope = &semaCtx.FindScope(*source);
|
|
collectScopes(curScope);
|
|
}
|
|
// Collect all symbols referenced in the evaluation being processed,
|
|
// that matches 'flag'.
|
|
llvm::SetVector<const semantics::Symbol *> allSymbols;
|
|
converter.collectSymbolSet(eval, allSymbols, flag,
|
|
/*collectSymbols=*/true,
|
|
/*collectHostAssociatedSymbols=*/true);
|
|
|
|
llvm::SetVector<const semantics::Symbol *> symbolsInNestedRegions;
|
|
collectSymbolsInNestedRegions(eval, flag, symbolsInNestedRegions);
|
|
|
|
for (auto *symbol : allSymbols)
|
|
if (visitor.isSymbolDefineBy(symbol, eval))
|
|
symbolsInNestedRegions.remove(symbol);
|
|
|
|
// Filter-out symbols that must not be privatized.
|
|
bool collectImplicit = flag == semantics::Symbol::Flag::OmpImplicit;
|
|
bool collectPreDetermined = flag == semantics::Symbol::Flag::OmpPreDetermined;
|
|
|
|
auto isPrivatizable = [](const semantics::Symbol &sym) -> bool {
|
|
return !semantics::IsProcedure(sym) &&
|
|
!sym.GetUltimate().has<semantics::DerivedTypeDetails>() &&
|
|
!sym.GetUltimate().has<semantics::NamelistDetails>() &&
|
|
!semantics::IsImpliedDoIndex(sym.GetUltimate());
|
|
};
|
|
|
|
auto shouldCollectSymbol = [&](const semantics::Symbol *sym) {
|
|
if (collectImplicit)
|
|
return sym->test(semantics::Symbol::Flag::OmpImplicit);
|
|
|
|
if (collectPreDetermined)
|
|
return sym->test(semantics::Symbol::Flag::OmpPreDetermined);
|
|
|
|
return !sym->test(semantics::Symbol::Flag::OmpImplicit) &&
|
|
!sym->test(semantics::Symbol::Flag::OmpPreDetermined);
|
|
};
|
|
|
|
for (const auto *sym : allSymbols) {
|
|
assert(curScope && "couldn't find current scope");
|
|
if (isPrivatizable(*sym) && !symbolsInNestedRegions.contains(sym) &&
|
|
!explicitlyPrivatizedSymbols.contains(sym) &&
|
|
shouldCollectSymbol(sym) && clauseScopes.contains(&sym->owner())) {
|
|
allPrivatizedSymbols.insert(sym);
|
|
symbols.insert(sym);
|
|
}
|
|
}
|
|
}
|
|
|
|
void DataSharingProcessor::collectDefaultSymbols() {
|
|
using DataSharingAttribute = omp::clause::Default::DataSharingAttribute;
|
|
for (const omp::Clause &clause : clauses) {
|
|
if (const auto *defaultClause =
|
|
std::get_if<omp::clause::Default>(&clause.u)) {
|
|
if (defaultClause->v == DataSharingAttribute::Private)
|
|
collectSymbols(semantics::Symbol::Flag::OmpPrivate, defaultSymbols);
|
|
else if (defaultClause->v == DataSharingAttribute::Firstprivate)
|
|
collectSymbols(semantics::Symbol::Flag::OmpFirstPrivate,
|
|
defaultSymbols);
|
|
}
|
|
}
|
|
}
|
|
|
|
void DataSharingProcessor::collectImplicitSymbols() {
|
|
// There will be no implicit symbols when a default clause is present.
|
|
if (defaultSymbols.empty())
|
|
collectSymbols(semantics::Symbol::Flag::OmpImplicit, implicitSymbols);
|
|
}
|
|
|
|
void DataSharingProcessor::collectPreDeterminedSymbols() {
|
|
if (shouldCollectPreDeterminedSymbols)
|
|
collectSymbols(semantics::Symbol::Flag::OmpPreDetermined,
|
|
preDeterminedSymbols);
|
|
}
|
|
|
|
void DataSharingProcessor::privatize(mlir::omp::PrivateClauseOps *clauseOps) {
|
|
for (const semantics::Symbol *sym : allPrivatizedSymbols) {
|
|
if (const auto *commonDet =
|
|
sym->detailsIf<semantics::CommonBlockDetails>()) {
|
|
for (const auto &mem : commonDet->objects())
|
|
doPrivatize(&*mem, clauseOps);
|
|
} else
|
|
doPrivatize(sym, clauseOps);
|
|
}
|
|
}
|
|
|
|
void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) {
|
|
insertLastPrivateCompare(op);
|
|
for (const semantics::Symbol *sym : allPrivatizedSymbols)
|
|
if (const auto *commonDet =
|
|
sym->detailsIf<semantics::CommonBlockDetails>()) {
|
|
for (const auto &mem : commonDet->objects()) {
|
|
copyLastPrivateSymbol(&*mem, &lastPrivIP);
|
|
}
|
|
} else {
|
|
copyLastPrivateSymbol(sym, &lastPrivIP);
|
|
}
|
|
}
|
|
|
|
void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym,
|
|
mlir::omp::PrivateClauseOps *clauseOps) {
|
|
if (!useDelayedPrivatization) {
|
|
cloneSymbol(sym);
|
|
copyFirstPrivateSymbol(sym);
|
|
return;
|
|
}
|
|
|
|
lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
|
|
assert(hsb && "Host symbol box not found");
|
|
|
|
mlir::Type symType = hsb.getAddr().getType();
|
|
mlir::Location symLoc = hsb.getAddr().getLoc();
|
|
std::string privatizerName = sym->name().ToString() + ".privatizer";
|
|
bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate);
|
|
|
|
mlir::omp::PrivateClauseOp privatizerOp = [&]() {
|
|
auto moduleOp = firOpBuilder.getModule();
|
|
auto uniquePrivatizerName = fir::getTypeAsString(
|
|
symType, converter.getKindMap(),
|
|
converter.mangleName(*sym) +
|
|
(isFirstPrivate ? "_firstprivate" : "_private"));
|
|
|
|
if (auto existingPrivatizer =
|
|
moduleOp.lookupSymbol<mlir::omp::PrivateClauseOp>(
|
|
uniquePrivatizerName))
|
|
return existingPrivatizer;
|
|
|
|
mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
|
|
firOpBuilder.setInsertionPoint(&moduleOp.getBodyRegion().front(),
|
|
moduleOp.getBodyRegion().front().begin());
|
|
auto result = firOpBuilder.create<mlir::omp::PrivateClauseOp>(
|
|
symLoc, uniquePrivatizerName, symType,
|
|
isFirstPrivate ? mlir::omp::DataSharingClauseType::FirstPrivate
|
|
: mlir::omp::DataSharingClauseType::Private);
|
|
fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym);
|
|
|
|
symTable->pushScope();
|
|
|
|
// Populate the `alloc` region.
|
|
{
|
|
mlir::Region &allocRegion = result.getAllocRegion();
|
|
mlir::Block *allocEntryBlock = firOpBuilder.createBlock(
|
|
&allocRegion, /*insertPt=*/{}, symType, symLoc);
|
|
|
|
firOpBuilder.setInsertionPointToEnd(allocEntryBlock);
|
|
|
|
fir::ExtendedValue localExV =
|
|
hlfir::translateToExtendedValue(
|
|
symLoc, firOpBuilder, hlfir::Entity{allocRegion.getArgument(0)},
|
|
/*contiguousHint=*/
|
|
evaluate::IsSimplyContiguous(*sym, converter.getFoldingContext()))
|
|
.first;
|
|
|
|
symTable->addSymbol(*sym, localExV);
|
|
symTable->pushScope();
|
|
cloneSymbol(sym);
|
|
firOpBuilder.create<mlir::omp::YieldOp>(
|
|
hsb.getAddr().getLoc(),
|
|
symTable->shallowLookupSymbol(*sym).getAddr());
|
|
symTable->popScope();
|
|
}
|
|
|
|
// Populate the `copy` region if this is a `firstprivate`.
|
|
if (isFirstPrivate) {
|
|
mlir::Region ©Region = result.getCopyRegion();
|
|
// First block argument corresponding to the original/host value while
|
|
// second block argument corresponding to the privatized value.
|
|
mlir::Block *copyEntryBlock = firOpBuilder.createBlock(
|
|
©Region, /*insertPt=*/{}, {symType, symType}, {symLoc, symLoc});
|
|
firOpBuilder.setInsertionPointToEnd(copyEntryBlock);
|
|
|
|
auto addSymbol = [&](unsigned argIdx, bool force = false) {
|
|
symExV.match(
|
|
[&](const fir::MutableBoxValue &box) {
|
|
symTable->addSymbol(
|
|
*sym, fir::substBase(box, copyRegion.getArgument(argIdx)),
|
|
force);
|
|
},
|
|
[&](const auto &box) {
|
|
symTable->addSymbol(*sym, copyRegion.getArgument(argIdx), force);
|
|
});
|
|
};
|
|
|
|
addSymbol(0, true);
|
|
symTable->pushScope();
|
|
addSymbol(1);
|
|
|
|
auto ip = firOpBuilder.saveInsertionPoint();
|
|
copyFirstPrivateSymbol(sym, &ip);
|
|
|
|
firOpBuilder.create<mlir::omp::YieldOp>(
|
|
hsb.getAddr().getLoc(),
|
|
symTable->shallowLookupSymbol(*sym).getAddr());
|
|
symTable->popScope();
|
|
}
|
|
|
|
symTable->popScope();
|
|
return result;
|
|
}();
|
|
|
|
if (clauseOps) {
|
|
clauseOps->privateSyms.push_back(mlir::SymbolRefAttr::get(privatizerOp));
|
|
clauseOps->privateVars.push_back(hsb.getAddr());
|
|
}
|
|
|
|
symToPrivatizer[sym] = privatizerOp;
|
|
}
|
|
|
|
} // namespace omp
|
|
} // namespace lower
|
|
} // namespace Fortran
|