b62b58d1bb
OpenMP allows duplicate mappings, i.e. in OpenMP 6.0, 7.9.6 "map Clause": Two list items of the map clauses on the same construct must not share original storage unless one of the following is true: they are the same list item [or other omitted reasons]" Duplicate mappings can arise as a result of user-defined mapper processing (which I think is a separate bug, and is not addressed here), but also in straightforward cases such as: #pragma omp target map(tofrom: s.mem[0:10]) map(tofrom: s.mem[0:10]) Both these cases cause crashes at runtime at present, due to an unfortunate interaction between reference counting behaviour and shadow pointer handling for blocks. This is what happens: 1. The member "s.mem" is copied to the target 2. A shadow pointer is created, modifying the pointer on the target 3. The member "s.mem" is copied to the target again 4. The previous shadow pointer metadata is still present, so the runtime doesn't modify the target pointer a second time. The fix is to disable step 3 if we've already done step 2 for a given block that has the "is new" flag set.
551 lines
23 KiB
C++
551 lines
23 KiB
C++
//===-- OpenMP/Mapping.cpp - OpenMP/OpenACC pointer mapping impl. ---------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "OpenMP/Mapping.h"
|
|
|
|
#include "PluginManager.h"
|
|
#include "Shared/Debug.h"
|
|
#include "Shared/Requirements.h"
|
|
#include "device.h"
|
|
|
|
/// Dump a table of all the host-target pointer pairs on failure
|
|
void dumpTargetPointerMappings(const ident_t *Loc, DeviceTy &Device,
|
|
bool toStdOut) {
|
|
MappingInfoTy::HDTTMapAccessorTy HDTTMap =
|
|
Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
|
|
if (HDTTMap->empty()) {
|
|
DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
|
|
"OpenMP Host-Device pointer mappings table empty\n");
|
|
return;
|
|
}
|
|
|
|
SourceInfo Kernel(Loc);
|
|
DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
|
|
"OpenMP Host-Device pointer mappings after block at %s:%d:%d:\n",
|
|
Kernel.getFilename(), Kernel.getLine(), Kernel.getColumn());
|
|
DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
|
|
"%-18s %-18s %s %s %s %s\n", "Host Ptr", "Target Ptr", "Size (B)",
|
|
"DynRefCount", "HoldRefCount", "Declaration");
|
|
for (const auto &It : *HDTTMap) {
|
|
HostDataToTargetTy &HDTT = *It.HDTT;
|
|
SourceInfo Info(HDTT.HstPtrName);
|
|
DUMP_INFO(toStdOut, OMP_INFOTYPE_ALL, Device.DeviceID,
|
|
DPxMOD " " DPxMOD " %-8" PRIuPTR " %-11s %-12s %s at %s:%d:%d\n",
|
|
DPxPTR(HDTT.HstPtrBegin), DPxPTR(HDTT.TgtPtrBegin),
|
|
HDTT.HstPtrEnd - HDTT.HstPtrBegin,
|
|
HDTT.dynRefCountToStr().c_str(), HDTT.holdRefCountToStr().c_str(),
|
|
Info.getName(), Info.getFilename(), Info.getLine(),
|
|
Info.getColumn());
|
|
}
|
|
}
|
|
|
|
int MappingInfoTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin,
|
|
int64_t Size) {
|
|
HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
|
|
|
|
// Check if entry exists
|
|
auto It = HDTTMap->find(HstPtrBegin);
|
|
if (It != HDTTMap->end()) {
|
|
HostDataToTargetTy &HDTT = *It->HDTT;
|
|
std::lock_guard<HostDataToTargetTy> LG(HDTT);
|
|
// Mapping already exists
|
|
bool IsValid = HDTT.HstPtrEnd == (uintptr_t)HstPtrBegin + Size &&
|
|
HDTT.TgtPtrBegin == (uintptr_t)TgtPtrBegin;
|
|
if (IsValid) {
|
|
DP("Attempt to re-associate the same device ptr+offset with the same "
|
|
"host ptr, nothing to do\n");
|
|
return OFFLOAD_SUCCESS;
|
|
}
|
|
REPORT("Not allowed to re-associate a different device ptr+offset with "
|
|
"the same host ptr\n");
|
|
return OFFLOAD_FAIL;
|
|
}
|
|
|
|
// Mapping does not exist, allocate it with refCount=INF
|
|
const HostDataToTargetTy &NewEntry =
|
|
*HDTTMap
|
|
->emplace(new HostDataToTargetTy(
|
|
/*HstPtrBase=*/(uintptr_t)HstPtrBegin,
|
|
/*HstPtrBegin=*/(uintptr_t)HstPtrBegin,
|
|
/*HstPtrEnd=*/(uintptr_t)HstPtrBegin + Size,
|
|
/*TgtAllocBegin=*/(uintptr_t)TgtPtrBegin,
|
|
/*TgtPtrBegin=*/(uintptr_t)TgtPtrBegin,
|
|
/*UseHoldRefCount=*/false, /*Name=*/nullptr,
|
|
/*IsRefCountINF=*/true))
|
|
.first->HDTT;
|
|
DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD
|
|
", HstEnd=" DPxMOD ", TgtBegin=" DPxMOD ", DynRefCount=%s, "
|
|
"HoldRefCount=%s\n",
|
|
DPxPTR(NewEntry.HstPtrBase), DPxPTR(NewEntry.HstPtrBegin),
|
|
DPxPTR(NewEntry.HstPtrEnd), DPxPTR(NewEntry.TgtPtrBegin),
|
|
NewEntry.dynRefCountToStr().c_str(), NewEntry.holdRefCountToStr().c_str());
|
|
(void)NewEntry;
|
|
|
|
// Notify the plugin about the new mapping.
|
|
return Device.notifyDataMapped(HstPtrBegin, Size);
|
|
}
|
|
|
|
int MappingInfoTy::disassociatePtr(void *HstPtrBegin) {
|
|
HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
|
|
|
|
auto It = HDTTMap->find(HstPtrBegin);
|
|
if (It == HDTTMap->end()) {
|
|
REPORT("Association not found\n");
|
|
return OFFLOAD_FAIL;
|
|
}
|
|
// Mapping exists
|
|
HostDataToTargetTy &HDTT = *It->HDTT;
|
|
std::lock_guard<HostDataToTargetTy> LG(HDTT);
|
|
|
|
if (HDTT.getHoldRefCount()) {
|
|
// This is based on OpenACC 3.1, sec 3.2.33 "acc_unmap_data", L3656-3657:
|
|
// "It is an error to call acc_unmap_data if the structured reference
|
|
// count for the pointer is not zero."
|
|
REPORT("Trying to disassociate a pointer with a non-zero hold reference "
|
|
"count\n");
|
|
return OFFLOAD_FAIL;
|
|
}
|
|
|
|
if (HDTT.isDynRefCountInf()) {
|
|
DP("Association found, removing it\n");
|
|
void *Event = HDTT.getEvent();
|
|
delete &HDTT;
|
|
if (Event)
|
|
Device.destroyEvent(Event);
|
|
HDTTMap->erase(It);
|
|
return Device.notifyDataUnmapped(HstPtrBegin);
|
|
}
|
|
|
|
REPORT("Trying to disassociate a pointer which was not mapped via "
|
|
"omp_target_associate_ptr\n");
|
|
return OFFLOAD_FAIL;
|
|
}
|
|
|
|
LookupResult MappingInfoTy::lookupMapping(HDTTMapAccessorTy &HDTTMap,
|
|
void *HstPtrBegin, int64_t Size,
|
|
HostDataToTargetTy *OwnedTPR) {
|
|
|
|
uintptr_t HP = (uintptr_t)HstPtrBegin;
|
|
LookupResult LR;
|
|
|
|
DP("Looking up mapping(HstPtrBegin=" DPxMOD ", Size=%" PRId64 ")...\n",
|
|
DPxPTR(HP), Size);
|
|
|
|
if (HDTTMap->empty())
|
|
return LR;
|
|
|
|
// HDTTMap is std::set, ordered by HstPtrBegin.
|
|
// Upper is the first element whose HstPtrBegin > HP.
|
|
auto Upper = HDTTMap->upper_bound(HP);
|
|
|
|
if (Size == 0) {
|
|
// HP satisfies
|
|
// std::prev(Upper)->HDTT.HstPtrBegin <= HP < Upper->HDTT.HstPtrBegin
|
|
if (Upper != HDTTMap->begin()) {
|
|
LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
|
|
// We know that HP >= LR.TPR.getEntry()->HstPtrBegin
|
|
LR.Flags.IsContained = HP < LR.TPR.getEntry()->HstPtrEnd;
|
|
}
|
|
|
|
if (!LR.Flags.IsContained && Upper != HDTTMap->end()) {
|
|
LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
|
|
// This is a special case: HP is not really contained in the mapped
|
|
// address range, but it's contained in the extended address range,
|
|
// which suffices to get the mapping of the base pointer.
|
|
// We know that HP < LR.TPR.getEntry()->HstPtrBegin
|
|
LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBase;
|
|
}
|
|
} else {
|
|
if (Upper != HDTTMap->begin()) {
|
|
LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
|
|
// We know that HP >= LR.TPR.getEntry()->HstPtrBegin
|
|
LR.Flags.IsContained = HP < LR.TPR.getEntry()->HstPtrEnd &&
|
|
(HP + Size) <= LR.TPR.getEntry()->HstPtrEnd;
|
|
// Does it extend beyond the mapped address range?
|
|
LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
|
|
(HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
|
|
}
|
|
|
|
if (!(LR.Flags.IsContained || LR.Flags.ExtendsAfter) &&
|
|
Upper != HDTTMap->end()) {
|
|
LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
|
|
// Does it extend into an already mapped address range?
|
|
// We know that HP < LR.TPR.getEntry()->HstPtrBegin
|
|
LR.Flags.ExtendsBefore = (HP + Size) > LR.TPR.getEntry()->HstPtrBegin;
|
|
// Does it extend beyond the mapped address range?
|
|
LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
|
|
(HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
|
|
}
|
|
|
|
if (LR.Flags.ExtendsBefore) {
|
|
DP("WARNING: Pointer is not mapped but section extends into already "
|
|
"mapped data\n");
|
|
}
|
|
if (LR.Flags.ExtendsAfter) {
|
|
DP("WARNING: Pointer is already mapped but section extends beyond mapped "
|
|
"region\n");
|
|
}
|
|
}
|
|
|
|
return LR;
|
|
}
|
|
|
|
TargetPointerResultTy MappingInfoTy::getTargetPointer(
|
|
HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
|
|
int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName, bool HasFlagTo,
|
|
bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
|
|
bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
|
|
AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR, bool ReleaseHDTTMap) {
|
|
|
|
LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size, OwnedTPR);
|
|
LR.TPR.Flags.IsPresent = true;
|
|
|
|
// Release the mapping table lock only after the entry is locked by
|
|
// attaching it to TPR. Once TPR is destroyed it will release the lock
|
|
// on entry. If it is returned the lock will move to the returned object.
|
|
// If LR.Entry is already owned/locked we avoid trying to lock it again.
|
|
|
|
// Check if the pointer is contained.
|
|
// If a variable is mapped to the device manually by the user - which would
|
|
// lead to the IsContained flag to be true - then we must ensure that the
|
|
// device address is returned even under unified memory conditions.
|
|
if (LR.Flags.IsContained ||
|
|
((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && IsImplicit)) {
|
|
const char *RefCountAction;
|
|
if (UpdateRefCount) {
|
|
// After this, reference count >= 1. If the reference count was 0 but the
|
|
// entry was still there we can reuse the data on the device and avoid a
|
|
// new submission.
|
|
LR.TPR.getEntry()->incRefCount(HasHoldModifier);
|
|
RefCountAction = " (incremented)";
|
|
} else {
|
|
// It might have been allocated with the parent, but it's still new.
|
|
LR.TPR.Flags.IsNewEntry = LR.TPR.getEntry()->getTotalRefCount() == 1;
|
|
RefCountAction = " (update suppressed)";
|
|
}
|
|
const char *DynRefCountAction = HasHoldModifier ? "" : RefCountAction;
|
|
const char *HoldRefCountAction = HasHoldModifier ? RefCountAction : "";
|
|
uintptr_t Ptr = LR.TPR.getEntry()->TgtPtrBegin +
|
|
((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
|
|
INFO(OMP_INFOTYPE_MAPPING_EXISTS, Device.DeviceID,
|
|
"Mapping exists%s with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
|
|
", Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s, Name=%s\n",
|
|
(IsImplicit ? " (implicit)" : ""), DPxPTR(HstPtrBegin), DPxPTR(Ptr),
|
|
Size, LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
|
|
LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction,
|
|
(HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
|
|
LR.TPR.TargetPointer = (void *)Ptr;
|
|
} else if ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && !IsImplicit) {
|
|
// Explicit extension of mapped data - not allowed.
|
|
MESSAGE("explicit extension not allowed: host address specified is " DPxMOD
|
|
" (%" PRId64
|
|
" bytes), but device allocation maps to host at " DPxMOD
|
|
" (%" PRId64 " bytes)",
|
|
DPxPTR(HstPtrBegin), Size, DPxPTR(LR.TPR.getEntry()->HstPtrBegin),
|
|
LR.TPR.getEntry()->HstPtrEnd - LR.TPR.getEntry()->HstPtrBegin);
|
|
if (HasPresentModifier)
|
|
MESSAGE("device mapping required by 'present' map type modifier does not "
|
|
"exist for host address " DPxMOD " (%" PRId64 " bytes)",
|
|
DPxPTR(HstPtrBegin), Size);
|
|
} else if ((PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY &&
|
|
!HasCloseModifier) ||
|
|
(PM->getRequirements() & OMPX_REQ_AUTO_ZERO_COPY)) {
|
|
|
|
// If unified shared memory is active, implicitly mapped variables that are
|
|
// not privatized use host address. Any explicitly mapped variables also use
|
|
// host address where correctness is not impeded. In all other cases maps
|
|
// are respected.
|
|
// In addition to the mapping rules above, the close map modifier forces the
|
|
// mapping of the variable to the device.
|
|
if (Size) {
|
|
INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
|
|
"Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
|
|
"memory\n",
|
|
DPxPTR((uintptr_t)HstPtrBegin), Size);
|
|
DP("Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
|
|
"memory\n",
|
|
DPxPTR((uintptr_t)HstPtrBegin), Size);
|
|
LR.TPR.Flags.IsPresent = false;
|
|
LR.TPR.Flags.IsHostPointer = true;
|
|
LR.TPR.TargetPointer = HstPtrBegin;
|
|
}
|
|
} else if (HasPresentModifier) {
|
|
DP("Mapping required by 'present' map type modifier does not exist for "
|
|
"HstPtrBegin=" DPxMOD ", Size=%" PRId64 "\n",
|
|
DPxPTR(HstPtrBegin), Size);
|
|
MESSAGE("device mapping required by 'present' map type modifier does not "
|
|
"exist for host address " DPxMOD " (%" PRId64 " bytes)",
|
|
DPxPTR(HstPtrBegin), Size);
|
|
} else if (Size) {
|
|
// If it is not contained and Size > 0, we should create a new entry for it.
|
|
LR.TPR.Flags.IsNewEntry = true;
|
|
uintptr_t TgtAllocBegin =
|
|
(uintptr_t)Device.allocData(TgtPadding + Size, HstPtrBegin);
|
|
uintptr_t TgtPtrBegin = TgtAllocBegin + TgtPadding;
|
|
// Release the mapping table lock only after the entry is locked by
|
|
// attaching it to TPR.
|
|
LR.TPR.setEntry(HDTTMap
|
|
->emplace(new HostDataToTargetTy(
|
|
(uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin,
|
|
(uintptr_t)HstPtrBegin + Size, TgtAllocBegin,
|
|
TgtPtrBegin, HasHoldModifier, HstPtrName))
|
|
.first->HDTT);
|
|
INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
|
|
"Creating new map entry with HstPtrBase=" DPxMOD
|
|
", HstPtrBegin=" DPxMOD ", TgtAllocBegin=" DPxMOD
|
|
", TgtPtrBegin=" DPxMOD
|
|
", Size=%ld, DynRefCount=%s, HoldRefCount=%s, Name=%s\n",
|
|
DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin), DPxPTR(TgtAllocBegin),
|
|
DPxPTR(TgtPtrBegin), Size,
|
|
LR.TPR.getEntry()->dynRefCountToStr().c_str(),
|
|
LR.TPR.getEntry()->holdRefCountToStr().c_str(),
|
|
(HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
|
|
LR.TPR.TargetPointer = (void *)TgtPtrBegin;
|
|
|
|
// Notify the plugin about the new mapping.
|
|
if (Device.notifyDataMapped(HstPtrBegin, Size))
|
|
return TargetPointerResultTy{};
|
|
} else {
|
|
// This entry is not present and we did not create a new entry for it.
|
|
LR.TPR.Flags.IsPresent = false;
|
|
}
|
|
|
|
// All mapping table modifications have been made. If the user requested it we
|
|
// give up the lock.
|
|
if (ReleaseHDTTMap)
|
|
HDTTMap.destroy();
|
|
|
|
// If the target pointer is valid, and we need to transfer data, issue the
|
|
// data transfer.
|
|
if (LR.TPR.TargetPointer && !LR.TPR.Flags.IsHostPointer && HasFlagTo &&
|
|
(LR.TPR.Flags.IsNewEntry || HasFlagAlways) && Size != 0) {
|
|
|
|
// If we have something like:
|
|
// #pragma omp target map(to: s.myarr[0:10]) map(to: s.myarr[0:10])
|
|
// then we see two "new" mappings of the struct member s.myarr here --
|
|
// and both have the "IsNewEntry" flag set, so trigger the copy to device
|
|
// below. But, the shadow pointer is only initialised on the target for
|
|
// the first copy, and the second copy clobbers it. So, this condition
|
|
// avoids the (second) copy here if we have already set shadow pointer info.
|
|
auto FailOnPtrFound = [HstPtrBegin, Size](ShadowPtrInfoTy &SP) {
|
|
if (SP.HstPtrAddr >= HstPtrBegin &&
|
|
SP.HstPtrAddr < (void *)((char *)HstPtrBegin + Size))
|
|
return OFFLOAD_FAIL;
|
|
return OFFLOAD_SUCCESS;
|
|
};
|
|
if (LR.TPR.getEntry()->foreachShadowPointerInfo(FailOnPtrFound) ==
|
|
OFFLOAD_FAIL) {
|
|
DP("Multiple new mappings of %" PRId64 " bytes detected (hst:" DPxMOD
|
|
") -> (tgt:" DPxMOD ")\n",
|
|
Size, DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer));
|
|
return std::move(LR.TPR);
|
|
}
|
|
|
|
DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size,
|
|
DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer));
|
|
|
|
int Ret = Device.submitData(LR.TPR.TargetPointer, HstPtrBegin, Size,
|
|
AsyncInfo, LR.TPR.getEntry());
|
|
if (Ret != OFFLOAD_SUCCESS) {
|
|
REPORT("Copying data to device failed.\n");
|
|
// We will also return nullptr if the data movement fails because that
|
|
// pointer points to a corrupted memory region so it doesn't make any
|
|
// sense to continue to use it.
|
|
LR.TPR.TargetPointer = nullptr;
|
|
} else if (LR.TPR.getEntry()->addEventIfNecessary(Device, AsyncInfo) !=
|
|
OFFLOAD_SUCCESS)
|
|
return TargetPointerResultTy{};
|
|
} else {
|
|
// If not a host pointer and no present modifier, we need to wait for the
|
|
// event if it exists.
|
|
// Note: Entry might be nullptr because of zero length array section.
|
|
if (LR.TPR.getEntry() && !LR.TPR.Flags.IsHostPointer &&
|
|
!HasPresentModifier) {
|
|
void *Event = LR.TPR.getEntry()->getEvent();
|
|
if (Event) {
|
|
int Ret = Device.waitEvent(Event, AsyncInfo);
|
|
if (Ret != OFFLOAD_SUCCESS) {
|
|
// If it fails to wait for the event, we need to return nullptr in
|
|
// case of any data race.
|
|
REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event));
|
|
return TargetPointerResultTy{};
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return std::move(LR.TPR);
|
|
}
|
|
|
|
TargetPointerResultTy MappingInfoTy::getTgtPtrBegin(
|
|
void *HstPtrBegin, int64_t Size, bool UpdateRefCount, bool UseHoldRefCount,
|
|
bool MustContain, bool ForceDelete, bool FromDataEnd) {
|
|
HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
|
|
|
|
LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
|
|
|
|
LR.TPR.Flags.IsPresent = true;
|
|
|
|
if (LR.Flags.IsContained ||
|
|
(!MustContain && (LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter))) {
|
|
LR.TPR.Flags.IsLast =
|
|
LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount, ForceDelete);
|
|
|
|
if (ForceDelete) {
|
|
LR.TPR.getEntry()->resetRefCount(UseHoldRefCount);
|
|
assert(LR.TPR.Flags.IsLast ==
|
|
LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount) &&
|
|
"expected correct IsLast prediction for reset");
|
|
}
|
|
|
|
// Increment the number of threads that is using the entry on a
|
|
// targetDataEnd, tracking the number of possible "deleters". A thread may
|
|
// come to own the entry deletion even if it was not the last one querying
|
|
// for it. Thus, we must track every query on targetDataEnds to ensure only
|
|
// the last thread that holds a reference to an entry actually deletes it.
|
|
if (FromDataEnd)
|
|
LR.TPR.getEntry()->incDataEndThreadCount();
|
|
|
|
const char *RefCountAction;
|
|
if (!UpdateRefCount) {
|
|
RefCountAction = " (update suppressed)";
|
|
} else if (LR.TPR.Flags.IsLast) {
|
|
LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
|
|
assert(LR.TPR.getEntry()->getTotalRefCount() == 0 &&
|
|
"Expected zero reference count when deletion is scheduled");
|
|
if (ForceDelete)
|
|
RefCountAction = " (reset, delayed deletion)";
|
|
else
|
|
RefCountAction = " (decremented, delayed deletion)";
|
|
} else {
|
|
LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
|
|
RefCountAction = " (decremented)";
|
|
}
|
|
const char *DynRefCountAction = UseHoldRefCount ? "" : RefCountAction;
|
|
const char *HoldRefCountAction = UseHoldRefCount ? RefCountAction : "";
|
|
uintptr_t TP = LR.TPR.getEntry()->TgtPtrBegin +
|
|
((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
|
|
INFO(OMP_INFOTYPE_MAPPING_EXISTS, Device.DeviceID,
|
|
"Mapping exists with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", "
|
|
"Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s\n",
|
|
DPxPTR(HstPtrBegin), DPxPTR(TP), Size,
|
|
LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
|
|
LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction);
|
|
LR.TPR.TargetPointer = (void *)TP;
|
|
} else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY ||
|
|
PM->getRequirements() & OMPX_REQ_AUTO_ZERO_COPY) {
|
|
// If the value isn't found in the mapping and unified shared memory
|
|
// is on then it means we have stumbled upon a value which we need to
|
|
// use directly from the host.
|
|
DP("Get HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
|
|
"memory\n",
|
|
DPxPTR((uintptr_t)HstPtrBegin), Size);
|
|
LR.TPR.Flags.IsPresent = false;
|
|
LR.TPR.Flags.IsHostPointer = true;
|
|
LR.TPR.TargetPointer = HstPtrBegin;
|
|
} else {
|
|
// OpenMP Specification v5.2: if a matching list item is not found, the
|
|
// pointer retains its original value as per firstprivate semantics.
|
|
LR.TPR.Flags.IsPresent = false;
|
|
LR.TPR.Flags.IsHostPointer = false;
|
|
LR.TPR.TargetPointer = HstPtrBegin;
|
|
}
|
|
|
|
return std::move(LR.TPR);
|
|
}
|
|
|
|
// Return the target pointer begin (where the data will be moved).
|
|
void *MappingInfoTy::getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap,
|
|
void *HstPtrBegin, int64_t Size) {
|
|
uintptr_t HP = (uintptr_t)HstPtrBegin;
|
|
LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
|
|
if (LR.Flags.IsContained || LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) {
|
|
uintptr_t TP =
|
|
LR.TPR.getEntry()->TgtPtrBegin + (HP - LR.TPR.getEntry()->HstPtrBegin);
|
|
return (void *)TP;
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
int MappingInfoTy::eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
|
|
HostDataToTargetTy *Entry, int64_t Size) {
|
|
assert(Entry && "Trying to delete a null entry from the HDTT map.");
|
|
assert(Entry->getTotalRefCount() == 0 &&
|
|
Entry->getDataEndThreadCount() == 0 &&
|
|
"Trying to delete entry that is in use or owned by another thread.");
|
|
|
|
INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
|
|
"Removing map entry with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
|
|
", Size=%" PRId64 ", Name=%s\n",
|
|
DPxPTR(Entry->HstPtrBegin), DPxPTR(Entry->TgtPtrBegin), Size,
|
|
(Entry->HstPtrName) ? getNameFromMapping(Entry->HstPtrName).c_str()
|
|
: "unknown");
|
|
|
|
if (HDTTMap->erase(Entry) == 0) {
|
|
REPORT("Trying to remove a non-existent map entry\n");
|
|
return OFFLOAD_FAIL;
|
|
}
|
|
|
|
return OFFLOAD_SUCCESS;
|
|
}
|
|
|
|
int MappingInfoTy::deallocTgtPtrAndEntry(HostDataToTargetTy *Entry,
|
|
int64_t Size) {
|
|
assert(Entry && "Trying to deallocate a null entry.");
|
|
|
|
DP("Deleting tgt data " DPxMOD " of size %" PRId64 " by freeing allocation "
|
|
"starting at " DPxMOD "\n",
|
|
DPxPTR(Entry->TgtPtrBegin), Size, DPxPTR(Entry->TgtAllocBegin));
|
|
|
|
void *Event = Entry->getEvent();
|
|
if (Event && Device.destroyEvent(Event) != OFFLOAD_SUCCESS) {
|
|
REPORT("Failed to destroy event " DPxMOD "\n", DPxPTR(Event));
|
|
return OFFLOAD_FAIL;
|
|
}
|
|
|
|
int Ret = Device.deleteData((void *)Entry->TgtAllocBegin);
|
|
|
|
// Notify the plugin about the unmapped memory.
|
|
Ret |= Device.notifyDataUnmapped((void *)Entry->HstPtrBegin);
|
|
|
|
delete Entry;
|
|
|
|
return Ret;
|
|
}
|
|
|
|
static void printCopyInfoImpl(int DeviceId, bool H2D, void *SrcPtrBegin,
|
|
void *DstPtrBegin, int64_t Size,
|
|
HostDataToTargetTy *HT) {
|
|
|
|
INFO(OMP_INFOTYPE_DATA_TRANSFER, DeviceId,
|
|
"Copying data from %s to %s, %sPtr=" DPxMOD ", %sPtr=" DPxMOD
|
|
", Size=%" PRId64 ", Name=%s\n",
|
|
H2D ? "host" : "device", H2D ? "device" : "host", H2D ? "Hst" : "Tgt",
|
|
DPxPTR(H2D ? SrcPtrBegin : DstPtrBegin), H2D ? "Tgt" : "Hst",
|
|
DPxPTR(H2D ? DstPtrBegin : SrcPtrBegin), Size,
|
|
(HT && HT->HstPtrName) ? getNameFromMapping(HT->HstPtrName).c_str()
|
|
: "unknown");
|
|
}
|
|
|
|
void MappingInfoTy::printCopyInfo(
|
|
void *TgtPtrBegin, void *HstPtrBegin, int64_t Size, bool H2D,
|
|
HostDataToTargetTy *Entry, MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
|
|
auto HDTTMap =
|
|
HostDataToTargetMap.getExclusiveAccessor(!!Entry || !!HDTTMapPtr);
|
|
LookupResult LR;
|
|
if (!Entry) {
|
|
LR = lookupMapping(HDTTMapPtr ? *HDTTMapPtr : HDTTMap, HstPtrBegin, Size);
|
|
Entry = LR.TPR.getEntry();
|
|
}
|
|
printCopyInfoImpl(Device.DeviceID, H2D, HstPtrBegin, TgtPtrBegin, Size,
|
|
Entry);
|
|
}
|