[ORC] Add a shared-memory based orc::MemoryMapper.
This is an implementation of orc::MemoryMapper that maps shared memory pages in both executor and controller process and writes directly to them avoiding transferring content over EPC. All allocations are properly deinitialized automatically on the executor side at shutdown by the ExecutorSharedMemoryMapperService. Reviewed By: lhames Differential Revision: https://reviews.llvm.org/D128544
This commit is contained in:
Anubhab Ghosh
committed by
Lang Hames
Lang Hames
parent
6e6c1efe04
commit
5acd471698
@@ -109,6 +109,47 @@ private:
|
||||
AllocationMap Allocations;
|
||||
};
|
||||
|
||||
class SharedMemoryMapper final : public MemoryMapper {
|
||||
public:
|
||||
struct SymbolAddrs {
|
||||
ExecutorAddr Instance;
|
||||
ExecutorAddr Reserve;
|
||||
ExecutorAddr Initialize;
|
||||
ExecutorAddr Deinitialize;
|
||||
ExecutorAddr Release;
|
||||
};
|
||||
|
||||
SharedMemoryMapper(ExecutorProcessControl &EPC, SymbolAddrs SAs)
|
||||
: EPC(EPC), SAs(SAs) {}
|
||||
|
||||
void reserve(size_t NumBytes, OnReservedFunction OnReserved) override;
|
||||
|
||||
char *prepare(ExecutorAddr Addr, size_t ContentSize) override;
|
||||
|
||||
void initialize(AllocInfo &AI, OnInitializedFunction OnInitialized) override;
|
||||
|
||||
void deinitialize(ArrayRef<ExecutorAddr> Allocations,
|
||||
OnDeinitializedFunction OnDeInitialized) override;
|
||||
|
||||
void release(ArrayRef<ExecutorAddr> Reservations,
|
||||
OnReleasedFunction OnRelease) override;
|
||||
|
||||
~SharedMemoryMapper() override;
|
||||
|
||||
private:
|
||||
struct Reservation {
|
||||
void *LocalAddr;
|
||||
size_t Size;
|
||||
};
|
||||
|
||||
ExecutorProcessControl &EPC;
|
||||
SymbolAddrs SAs;
|
||||
|
||||
std::mutex Mutex;
|
||||
|
||||
std::map<ExecutorAddr, Reservation> Reservations;
|
||||
};
|
||||
|
||||
} // namespace orc
|
||||
} // end namespace llvm
|
||||
|
||||
|
||||
@@ -31,6 +31,12 @@ extern const char *SimpleExecutorMemoryManagerReserveWrapperName;
|
||||
extern const char *SimpleExecutorMemoryManagerFinalizeWrapperName;
|
||||
extern const char *SimpleExecutorMemoryManagerDeallocateWrapperName;
|
||||
|
||||
extern const char *ExecutorSharedMemoryMapperServiceInstanceName;
|
||||
extern const char *ExecutorSharedMemoryMapperServiceReserveWrapperName;
|
||||
extern const char *ExecutorSharedMemoryMapperServiceInitializeWrapperName;
|
||||
extern const char *ExecutorSharedMemoryMapperServiceDeinitializeWrapperName;
|
||||
extern const char *ExecutorSharedMemoryMapperServiceReleaseWrapperName;
|
||||
|
||||
extern const char *MemoryWriteUInt8sWrapperName;
|
||||
extern const char *MemoryWriteUInt16sWrapperName;
|
||||
extern const char *MemoryWriteUInt32sWrapperName;
|
||||
@@ -58,6 +64,21 @@ using SPSSimpleExecutorMemoryManagerFinalizeSignature =
|
||||
using SPSSimpleExecutorMemoryManagerDeallocateSignature = shared::SPSError(
|
||||
shared::SPSExecutorAddr, shared::SPSSequence<shared::SPSExecutorAddr>);
|
||||
|
||||
// ExecutorSharedMemoryMapperService
|
||||
using SPSExecutorSharedMemoryMapperServiceReserveSignature =
|
||||
shared::SPSExpected<
|
||||
shared::SPSTuple<shared::SPSExecutorAddr, shared::SPSString>>(
|
||||
shared::SPSExecutorAddr, uint64_t);
|
||||
using SPSExecutorSharedMemoryMapperServiceInitializeSignature =
|
||||
shared::SPSExpected<shared::SPSExecutorAddr>(shared::SPSExecutorAddr,
|
||||
shared::SPSExecutorAddr,
|
||||
shared::SPSFinalizeRequest);
|
||||
using SPSExecutorSharedMemoryMapperServiceDeinitializeSignature =
|
||||
shared::SPSError(shared::SPSExecutorAddr,
|
||||
shared::SPSSequence<shared::SPSExecutorAddr>);
|
||||
using SPSExecutorSharedMemoryMapperServiceReleaseSignature = shared::SPSError(
|
||||
shared::SPSExecutorAddr, shared::SPSSequence<shared::SPSExecutorAddr>);
|
||||
|
||||
using SPSRunAsMainSignature = int64_t(shared::SPSExecutorAddr,
|
||||
shared::SPSSequence<shared::SPSString>);
|
||||
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
//===----------- ExecutorSharedMemoryMapperService.h ------------*- C++ -*-===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORSHAREDMEMORYMAPPERSERVICE
|
||||
#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORSHAREDMEMORYMAPPERSERVICE
|
||||
|
||||
#include "llvm/ADT/DenseMap.h"
|
||||
#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
|
||||
#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
namespace llvm {
|
||||
namespace orc {
|
||||
namespace rt_bootstrap {
|
||||
|
||||
class ExecutorSharedMemoryMapperService final
|
||||
: public ExecutorBootstrapService {
|
||||
public:
|
||||
~ExecutorSharedMemoryMapperService(){};
|
||||
|
||||
Expected<std::pair<ExecutorAddr, std::string>> reserve(uint64_t Size);
|
||||
Expected<ExecutorAddr> initialize(ExecutorAddr Reservation,
|
||||
tpctypes::FinalizeRequest &FR);
|
||||
|
||||
Error deinitialize(const std::vector<ExecutorAddr> &Bases);
|
||||
Error release(const std::vector<ExecutorAddr> &Bases);
|
||||
|
||||
Error shutdown() override;
|
||||
void addBootstrapSymbols(StringMap<ExecutorAddr> &M) override;
|
||||
|
||||
private:
|
||||
struct Allocation {
|
||||
std::vector<shared::WrapperFunctionCall> DeinitializationActions;
|
||||
};
|
||||
using AllocationMap = DenseMap<ExecutorAddr, Allocation>;
|
||||
|
||||
struct Reservation {
|
||||
size_t Size;
|
||||
std::vector<ExecutorAddr> Allocations;
|
||||
#if defined(_WIN32)
|
||||
HANDLE SharedMemoryFile;
|
||||
#endif
|
||||
};
|
||||
using ReservationMap = DenseMap<void *, Reservation>;
|
||||
|
||||
static llvm::orc::shared::CWrapperFunctionResult
|
||||
reserveWrapper(const char *ArgData, size_t ArgSize);
|
||||
|
||||
static llvm::orc::shared::CWrapperFunctionResult
|
||||
initializeWrapper(const char *ArgData, size_t ArgSize);
|
||||
|
||||
static llvm::orc::shared::CWrapperFunctionResult
|
||||
deinitializeWrapper(const char *ArgData, size_t ArgSize);
|
||||
|
||||
static llvm::orc::shared::CWrapperFunctionResult
|
||||
releaseWrapper(const char *ArgData, size_t ArgSize);
|
||||
|
||||
std::atomic<int> SharedMemoryCount{0};
|
||||
std::mutex Mutex;
|
||||
ReservationMap Reservations;
|
||||
AllocationMap Allocations;
|
||||
};
|
||||
|
||||
} // namespace rt_bootstrap
|
||||
} // namespace orc
|
||||
} // namespace llvm
|
||||
#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORSHAREDMEMORYMAPPERSERVICE
|
||||
@@ -8,6 +8,17 @@
|
||||
|
||||
#include "llvm/ExecutionEngine/Orc/MemoryMapper.h"
|
||||
|
||||
#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
|
||||
#include "llvm/Support/WindowsError.h"
|
||||
|
||||
#if defined(LLVM_ON_UNIX)
|
||||
#include <fcntl.h>
|
||||
#include <sys/mman.h>
|
||||
#include <unistd.h>
|
||||
#elif defined(_WIN32)
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
namespace llvm {
|
||||
namespace orc {
|
||||
|
||||
@@ -147,6 +158,225 @@ InProcessMemoryMapper::~InProcessMemoryMapper() {
|
||||
cantFail(F.get());
|
||||
}
|
||||
|
||||
// SharedMemoryMapper
|
||||
|
||||
void SharedMemoryMapper::reserve(size_t NumBytes,
|
||||
OnReservedFunction OnReserved) {
|
||||
#if defined(LLVM_ON_UNIX) || defined(_WIN32)
|
||||
|
||||
EPC.callSPSWrapperAsync<
|
||||
rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>(
|
||||
SAs.Reserve,
|
||||
[this, NumBytes, OnReserved = std::move(OnReserved)](
|
||||
Error SerializationErr,
|
||||
Expected<std::pair<ExecutorAddr, std::string>> Result) mutable {
|
||||
if (SerializationErr) {
|
||||
cantFail(Result.takeError());
|
||||
return OnReserved(std::move(SerializationErr));
|
||||
}
|
||||
|
||||
if (!Result)
|
||||
return OnReserved(Result.takeError());
|
||||
|
||||
ExecutorAddr RemoteAddr;
|
||||
std::string SharedMemoryName;
|
||||
std::tie(RemoteAddr, SharedMemoryName) = std::move(*Result);
|
||||
|
||||
void *LocalAddr = nullptr;
|
||||
|
||||
#if defined(LLVM_ON_UNIX)
|
||||
|
||||
int SharedMemoryFile = shm_open(SharedMemoryName.c_str(), O_RDWR, 0700);
|
||||
if (SharedMemoryFile < 0) {
|
||||
return OnReserved(errorCodeToError(
|
||||
std::error_code(errno, std::generic_category())));
|
||||
}
|
||||
|
||||
// this prevents other processes from accessing it by name
|
||||
shm_unlink(SharedMemoryName.c_str());
|
||||
|
||||
LocalAddr = mmap(nullptr, NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED,
|
||||
SharedMemoryFile, 0);
|
||||
if (LocalAddr == MAP_FAILED) {
|
||||
return OnReserved(errorCodeToError(
|
||||
std::error_code(errno, std::generic_category())));
|
||||
}
|
||||
|
||||
close(SharedMemoryFile);
|
||||
|
||||
#elif defined(_WIN32)
|
||||
|
||||
std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
|
||||
SharedMemoryName.end());
|
||||
HANDLE SharedMemoryFile = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE,
|
||||
WideSharedMemoryName.c_str());
|
||||
if (!SharedMemoryFile)
|
||||
return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
|
||||
|
||||
LocalAddr =
|
||||
MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0);
|
||||
if (!LocalAddr) {
|
||||
CloseHandle(SharedMemoryFile);
|
||||
return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
|
||||
}
|
||||
|
||||
CloseHandle(SharedMemoryFile);
|
||||
|
||||
#endif
|
||||
{
|
||||
std::lock_guard<std::mutex> Lock(Mutex);
|
||||
Reservations.insert({RemoteAddr, {LocalAddr, NumBytes}});
|
||||
}
|
||||
|
||||
OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes));
|
||||
},
|
||||
SAs.Instance, static_cast<uint64_t>(NumBytes));
|
||||
|
||||
#else
|
||||
OnReserved(make_error<StringError>(
|
||||
"SharedMemoryMapper is not supported on this platform yet",
|
||||
inconvertibleErrorCode()));
|
||||
#endif
|
||||
}
|
||||
|
||||
char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) {
|
||||
auto R = Reservations.upper_bound(Addr);
|
||||
assert(R != Reservations.begin() && "Attempt to prepare unknown range");
|
||||
R--;
|
||||
|
||||
ExecutorAddrDiff Offset = Addr - R->first;
|
||||
|
||||
return static_cast<char *>(R->second.LocalAddr) + Offset;
|
||||
}
|
||||
|
||||
void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI,
|
||||
OnInitializedFunction OnInitialized) {
|
||||
auto Reservation = Reservations.find(AI.MappingBase);
|
||||
assert(Reservation != Reservations.end() &&
|
||||
"Attempt to initialize unreserved range");
|
||||
|
||||
tpctypes::FinalizeRequest FR;
|
||||
|
||||
AI.Actions.swap(FR.Actions);
|
||||
|
||||
FR.Segments.reserve(AI.Segments.size());
|
||||
|
||||
for (auto Segment : AI.Segments) {
|
||||
char *Base =
|
||||
static_cast<char *>(Reservation->second.LocalAddr) + Segment.Offset;
|
||||
std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize);
|
||||
|
||||
tpctypes::SegFinalizeRequest SegReq;
|
||||
SegReq.Prot = tpctypes::toWireProtectionFlags(
|
||||
static_cast<sys::Memory::ProtectionFlags>(Segment.Prot));
|
||||
SegReq.Addr = AI.MappingBase + Segment.Offset;
|
||||
SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize;
|
||||
|
||||
FR.Segments.push_back(SegReq);
|
||||
}
|
||||
|
||||
EPC.callSPSWrapperAsync<
|
||||
rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>(
|
||||
SAs.Initialize,
|
||||
[OnInitialized = std::move(OnInitialized)](
|
||||
Error SerializationErr, Expected<ExecutorAddr> Result) mutable {
|
||||
if (SerializationErr) {
|
||||
cantFail(Result.takeError());
|
||||
return OnInitialized(std::move(SerializationErr));
|
||||
}
|
||||
|
||||
OnInitialized(std::move(Result));
|
||||
},
|
||||
SAs.Instance, AI.MappingBase, std::move(FR));
|
||||
}
|
||||
|
||||
void SharedMemoryMapper::deinitialize(
|
||||
ArrayRef<ExecutorAddr> Allocations,
|
||||
MemoryMapper::OnDeinitializedFunction OnDeinitialized) {
|
||||
EPC.callSPSWrapperAsync<
|
||||
rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>(
|
||||
SAs.Deinitialize,
|
||||
[OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr,
|
||||
Error Result) mutable {
|
||||
if (SerializationErr) {
|
||||
cantFail(std::move(Result));
|
||||
return OnDeinitialized(std::move(SerializationErr));
|
||||
}
|
||||
|
||||
OnDeinitialized(std::move(Result));
|
||||
},
|
||||
SAs.Instance, Allocations);
|
||||
}
|
||||
|
||||
void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
|
||||
OnReleasedFunction OnReleased) {
|
||||
#if defined(LLVM_ON_UNIX) || defined(_WIN32)
|
||||
Error Err = Error::success();
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> Lock(Mutex);
|
||||
|
||||
for (auto Base : Bases) {
|
||||
|
||||
#if defined(LLVM_ON_UNIX)
|
||||
|
||||
if (munmap(Reservations[Base].LocalAddr, Reservations[Base].Size) != 0)
|
||||
Err = joinErrors(std::move(Err), errorCodeToError(std::error_code(
|
||||
errno, std::generic_category())));
|
||||
|
||||
#elif defined(_WIN32)
|
||||
|
||||
if (!UnmapViewOfFile(Reservations[Base].LocalAddr))
|
||||
joinErrors(std::move(Err),
|
||||
errorCodeToError(mapWindowsError(GetLastError())));
|
||||
|
||||
#endif
|
||||
|
||||
Reservations.erase(Base);
|
||||
}
|
||||
}
|
||||
|
||||
EPC.callSPSWrapperAsync<
|
||||
rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>(
|
||||
SAs.Release,
|
||||
[OnReleased = std::move(OnReleased),
|
||||
Err = std::move(Err)](Error SerializationErr, Error Result) mutable {
|
||||
if (SerializationErr) {
|
||||
cantFail(std::move(Result));
|
||||
return OnReleased(
|
||||
joinErrors(std::move(Err), std::move(SerializationErr)));
|
||||
}
|
||||
|
||||
return OnReleased(joinErrors(std::move(Err), std::move(Result)));
|
||||
},
|
||||
SAs.Instance, Bases);
|
||||
#else
|
||||
OnReleased(make_error<StringError>(
|
||||
"SharedMemoryMapper is not supported on this platform yet",
|
||||
inconvertibleErrorCode()));
|
||||
#endif
|
||||
}
|
||||
|
||||
SharedMemoryMapper::~SharedMemoryMapper() {
|
||||
std::vector<ExecutorAddr> ReservationAddrs;
|
||||
if (!Reservations.empty()) {
|
||||
std::lock_guard<std::mutex> Lock(Mutex);
|
||||
{
|
||||
ReservationAddrs.reserve(Reservations.size());
|
||||
for (const auto &R : Reservations) {
|
||||
ReservationAddrs.push_back(R.first);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::promise<MSVCPError> P;
|
||||
auto F = P.get_future();
|
||||
release(ReservationAddrs, [&](Error Err) { P.set_value(std::move(Err)); });
|
||||
// FIXME: Release can actually fail. The error should be propagated.
|
||||
// Meanwhile, a better option is to explicitly call release().
|
||||
cantFail(F.get());
|
||||
}
|
||||
|
||||
} // namespace orc
|
||||
|
||||
} // namespace llvm
|
||||
|
||||
@@ -18,6 +18,7 @@ const char *SimpleExecutorDylibManagerOpenWrapperName =
|
||||
"__llvm_orc_SimpleExecutorDylibManager_open_wrapper";
|
||||
const char *SimpleExecutorDylibManagerLookupWrapperName =
|
||||
"__llvm_orc_SimpleExecutorDylibManager_lookup_wrapper";
|
||||
|
||||
const char *SimpleExecutorMemoryManagerInstanceName =
|
||||
"__llvm_orc_SimpleExecutorMemoryManager_Instance";
|
||||
const char *SimpleExecutorMemoryManagerReserveWrapperName =
|
||||
@@ -26,6 +27,18 @@ const char *SimpleExecutorMemoryManagerFinalizeWrapperName =
|
||||
"__llvm_orc_SimpleExecutorMemoryManager_finalize_wrapper";
|
||||
const char *SimpleExecutorMemoryManagerDeallocateWrapperName =
|
||||
"__llvm_orc_SimpleExecutorMemoryManager_deallocate_wrapper";
|
||||
|
||||
const char *ExecutorSharedMemoryMapperServiceInstanceName =
|
||||
"__llvm_orc_ExecutorSharedMemoryMapperService_Instance";
|
||||
const char *ExecutorSharedMemoryMapperServiceReserveWrapperName =
|
||||
"__llvm_orc_ExecutorSharedMemoryMapperService_Reserve";
|
||||
const char *ExecutorSharedMemoryMapperServiceInitializeWrapperName =
|
||||
"__llvm_orc_ExecutorSharedMemoryMapperService_Initialize";
|
||||
const char *ExecutorSharedMemoryMapperServiceDeinitializeWrapperName =
|
||||
"__llvm_orc_ExecutorSharedMemoryMapperService_Deinitialize";
|
||||
const char *ExecutorSharedMemoryMapperServiceReleaseWrapperName =
|
||||
"__llvm_orc_ExecutorSharedMemoryMapperService_Release";
|
||||
|
||||
const char *MemoryWriteUInt8sWrapperName =
|
||||
"__llvm_orc_bootstrap_mem_write_uint8s_wrapper";
|
||||
const char *MemoryWriteUInt16sWrapperName =
|
||||
@@ -36,10 +49,12 @@ const char *MemoryWriteUInt64sWrapperName =
|
||||
"__llvm_orc_bootstrap_mem_write_uint64s_wrapper";
|
||||
const char *MemoryWriteBuffersWrapperName =
|
||||
"__llvm_orc_bootstrap_mem_write_buffers_wrapper";
|
||||
|
||||
const char *RegisterEHFrameSectionWrapperName =
|
||||
"__llvm_orc_bootstrap_register_ehframe_section_wrapper";
|
||||
const char *DeregisterEHFrameSectionWrapperName =
|
||||
"__llvm_orc_bootstrap_deregister_ehframe_section_wrapper";
|
||||
|
||||
const char *RunAsMainWrapperName = "__llvm_orc_bootstrap_run_as_main_wrapper";
|
||||
|
||||
} // end namespace rt
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
add_llvm_component_library(LLVMOrcTargetProcess
|
||||
ExecutorSharedMemoryMapperService.cpp
|
||||
JITLoaderGDB.cpp
|
||||
OrcRTBootstrap.cpp
|
||||
RegisterEHFrames.cpp
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
//===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"
|
||||
|
||||
#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
|
||||
#include "llvm/Support/Process.h"
|
||||
#include "llvm/Support/WindowsError.h"
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#if defined(LLVM_ON_UNIX)
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <sys/mman.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#if defined(_WIN32)
|
||||
static DWORD getWindowsProtectionFlags(unsigned Flags) {
|
||||
switch (Flags & llvm::sys::Memory::MF_RWE_MASK) {
|
||||
case llvm::sys::Memory::MF_READ:
|
||||
return PAGE_READONLY;
|
||||
case llvm::sys::Memory::MF_WRITE:
|
||||
// Note: PAGE_WRITE is not supported by VirtualProtect
|
||||
return PAGE_READWRITE;
|
||||
case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE:
|
||||
return PAGE_READWRITE;
|
||||
case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_EXEC:
|
||||
return PAGE_EXECUTE_READ;
|
||||
case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE |
|
||||
llvm::sys::Memory::MF_EXEC:
|
||||
return PAGE_EXECUTE_READWRITE;
|
||||
case llvm::sys::Memory::MF_EXEC:
|
||||
return PAGE_EXECUTE;
|
||||
default:
|
||||
llvm_unreachable("Illegal memory protection flag specified!");
|
||||
}
|
||||
// Provide a default return value as required by some compilers.
|
||||
return PAGE_NOACCESS;
|
||||
}
|
||||
#endif
|
||||
|
||||
namespace llvm {
|
||||
namespace orc {
|
||||
namespace rt_bootstrap {
|
||||
|
||||
Expected<std::pair<ExecutorAddr, std::string>>
|
||||
ExecutorSharedMemoryMapperService::reserve(uint64_t Size) {
|
||||
#if defined(LLVM_ON_UNIX) || defined(_WIN32)
|
||||
|
||||
#if defined(LLVM_ON_UNIX)
|
||||
|
||||
std::string SharedMemoryName;
|
||||
{
|
||||
std::stringstream SharedMemoryNameStream;
|
||||
SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_'
|
||||
<< (++SharedMemoryCount);
|
||||
SharedMemoryName = SharedMemoryNameStream.str();
|
||||
}
|
||||
|
||||
int SharedMemoryFile =
|
||||
shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, 0700);
|
||||
if (SharedMemoryFile < 0)
|
||||
return errorCodeToError(std::error_code(errno, std::generic_category()));
|
||||
|
||||
// by default size is 0
|
||||
if (ftruncate(SharedMemoryFile, Size) < 0)
|
||||
return errorCodeToError(std::error_code(errno, std::generic_category()));
|
||||
|
||||
void *Addr = mmap(nullptr, Size, PROT_NONE, MAP_SHARED, SharedMemoryFile, 0);
|
||||
if (Addr == MAP_FAILED)
|
||||
return errorCodeToError(std::error_code(errno, std::generic_category()));
|
||||
|
||||
close(SharedMemoryFile);
|
||||
|
||||
#elif defined(_WIN32)
|
||||
|
||||
std::string SharedMemoryName;
|
||||
{
|
||||
std::stringstream SharedMemoryNameStream;
|
||||
SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_'
|
||||
<< (++SharedMemoryCount);
|
||||
SharedMemoryName = SharedMemoryNameStream.str();
|
||||
}
|
||||
|
||||
std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
|
||||
SharedMemoryName.end());
|
||||
HANDLE SharedMemoryFile = CreateFileMapping(
|
||||
INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32,
|
||||
Size & 0xffffffff, WideSharedMemoryName.c_str());
|
||||
if (!SharedMemoryFile)
|
||||
return errorCodeToError(mapWindowsError(GetLastError()));
|
||||
|
||||
void *Addr = MapViewOfFile(SharedMemoryFile,
|
||||
FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0);
|
||||
if (!Addr) {
|
||||
CloseHandle(SharedMemoryFile);
|
||||
return errorCodeToError(mapWindowsError(GetLastError()));
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> Lock(Mutex);
|
||||
Reservations[Addr].Size = Size;
|
||||
#if defined(_WIN32)
|
||||
Reservations[Addr].SharedMemoryFile = SharedMemoryFile;
|
||||
#endif
|
||||
}
|
||||
|
||||
return std::make_pair(ExecutorAddr::fromPtr(Addr),
|
||||
std::move(SharedMemoryName));
|
||||
#else
|
||||
return make_error<StringError>(
|
||||
"SharedMemoryMapper is not supported on this platform yet",
|
||||
inconvertibleErrorCode());
|
||||
#endif
|
||||
}
|
||||
|
||||
Expected<ExecutorAddr>
|
||||
ExecutorSharedMemoryMapperService::initialize(ExecutorAddr Reservation,
|
||||
tpctypes::FinalizeRequest &FR) {
|
||||
#if defined(LLVM_ON_UNIX) || defined(_WIN32)
|
||||
|
||||
ExecutorAddr MinAddr(~0ULL);
|
||||
|
||||
// Contents are already in place
|
||||
for (auto &Segment : FR.Segments) {
|
||||
if (Segment.Addr < MinAddr)
|
||||
MinAddr = Segment.Addr;
|
||||
|
||||
#if defined(LLVM_ON_UNIX)
|
||||
|
||||
int NativeProt = 0;
|
||||
if (Segment.Prot & tpctypes::WPF_Read)
|
||||
NativeProt |= PROT_READ;
|
||||
if (Segment.Prot & tpctypes::WPF_Write)
|
||||
NativeProt |= PROT_WRITE;
|
||||
if (Segment.Prot & tpctypes::WPF_Exec)
|
||||
NativeProt |= PROT_EXEC;
|
||||
|
||||
if (mprotect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt))
|
||||
return errorCodeToError(std::error_code(errno, std::generic_category()));
|
||||
|
||||
#elif defined(_WIN32)
|
||||
|
||||
DWORD NativeProt =
|
||||
getWindowsProtectionFlags(fromWireProtectionFlags(Segment.Prot));
|
||||
|
||||
if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt,
|
||||
&NativeProt))
|
||||
return errorCodeToError(mapWindowsError(GetLastError()));
|
||||
|
||||
#endif
|
||||
|
||||
if (Segment.Prot & tpctypes::WPF_Exec)
|
||||
sys::Memory::InvalidateInstructionCache(Segment.Addr.toPtr<void *>(),
|
||||
Segment.Size);
|
||||
}
|
||||
|
||||
// Run finalization actions and get deinitlization action list.
|
||||
auto DeinitializeActions = shared::runFinalizeActions(FR.Actions);
|
||||
if (!DeinitializeActions) {
|
||||
return DeinitializeActions.takeError();
|
||||
}
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> Lock(Mutex);
|
||||
Allocations[MinAddr].DeinitializationActions =
|
||||
std::move(*DeinitializeActions);
|
||||
Reservations[Reservation.toPtr<void *>()].Allocations.push_back(MinAddr);
|
||||
}
|
||||
|
||||
return MinAddr;
|
||||
|
||||
#else
|
||||
return make_error<StringError>(
|
||||
"SharedMemoryMapper is not supported on this platform yet",
|
||||
inconvertibleErrorCode());
|
||||
#endif
|
||||
}
|
||||
|
||||
Error ExecutorSharedMemoryMapperService::deinitialize(
|
||||
const std::vector<ExecutorAddr> &Bases) {
|
||||
Error AllErr = Error::success();
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> Lock(Mutex);
|
||||
|
||||
for (auto Base : Bases) {
|
||||
if (Error Err = shared::runDeallocActions(
|
||||
Allocations[Base].DeinitializationActions)) {
|
||||
AllErr = joinErrors(std::move(AllErr), std::move(Err));
|
||||
}
|
||||
|
||||
Allocations.erase(Base);
|
||||
}
|
||||
}
|
||||
|
||||
return AllErr;
|
||||
}
|
||||
|
||||
Error ExecutorSharedMemoryMapperService::release(
|
||||
const std::vector<ExecutorAddr> &Bases) {
|
||||
#if defined(LLVM_ON_UNIX) || defined(_WIN32)
|
||||
Error Err = Error::success();
|
||||
|
||||
for (auto Base : Bases) {
|
||||
std::vector<ExecutorAddr> AllocAddrs;
|
||||
size_t Size;
|
||||
|
||||
#if defined(_WIN32)
|
||||
HANDLE SharedMemoryFile;
|
||||
#endif
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> Lock(Mutex);
|
||||
auto &R = Reservations[Base.toPtr<void *>()];
|
||||
Size = R.Size;
|
||||
|
||||
#if defined(_WIN32)
|
||||
SharedMemoryFile = R.SharedMemoryFile;
|
||||
#endif
|
||||
|
||||
AllocAddrs.swap(R.Allocations);
|
||||
}
|
||||
|
||||
// deinitialize sub allocations
|
||||
if (Error E = deinitialize(AllocAddrs))
|
||||
Err = joinErrors(std::move(Err), std::move(E));
|
||||
|
||||
#if defined(LLVM_ON_UNIX)
|
||||
|
||||
if (munmap(Base.toPtr<void *>(), Size) != 0)
|
||||
Err = joinErrors(std::move(Err), errorCodeToError(std::error_code(
|
||||
errno, std::generic_category())));
|
||||
|
||||
#elif defined(_WIN32)
|
||||
|
||||
if (!UnmapViewOfFile(Base.toPtr<void *>()))
|
||||
Err = joinErrors(std::move(Err),
|
||||
errorCodeToError(mapWindowsError(GetLastError())));
|
||||
|
||||
CloseHandle(SharedMemoryFile);
|
||||
|
||||
#endif
|
||||
|
||||
std::lock_guard<std::mutex> Lock(Mutex);
|
||||
Reservations.erase(Base.toPtr<void *>());
|
||||
}
|
||||
|
||||
return Err;
|
||||
#else
|
||||
return make_error<StringError>(
|
||||
"SharedMemoryMapper is not supported on this platform yet",
|
||||
inconvertibleErrorCode());
|
||||
#endif
|
||||
}
|
||||
|
||||
Error ExecutorSharedMemoryMapperService::shutdown() {
|
||||
std::vector<ExecutorAddr> ReservationAddrs;
|
||||
if (!Reservations.empty()) {
|
||||
std::lock_guard<std::mutex> Lock(Mutex);
|
||||
{
|
||||
ReservationAddrs.reserve(Reservations.size());
|
||||
for (const auto &R : Reservations) {
|
||||
ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst()));
|
||||
}
|
||||
}
|
||||
}
|
||||
return release(ReservationAddrs);
|
||||
|
||||
return Error::success();
|
||||
}
|
||||
|
||||
void ExecutorSharedMemoryMapperService::addBootstrapSymbols(
|
||||
StringMap<ExecutorAddr> &M) {
|
||||
M[rt::ExecutorSharedMemoryMapperServiceInstanceName] =
|
||||
ExecutorAddr::fromPtr(this);
|
||||
M[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName] =
|
||||
ExecutorAddr::fromPtr(&reserveWrapper);
|
||||
M[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName] =
|
||||
ExecutorAddr::fromPtr(&initializeWrapper);
|
||||
M[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName] =
|
||||
ExecutorAddr::fromPtr(&deinitializeWrapper);
|
||||
M[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName] =
|
||||
ExecutorAddr::fromPtr(&releaseWrapper);
|
||||
}
|
||||
|
||||
llvm::orc::shared::CWrapperFunctionResult
|
||||
ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData,
|
||||
size_t ArgSize) {
|
||||
return shared::WrapperFunction<
|
||||
rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>::
|
||||
handle(ArgData, ArgSize,
|
||||
shared::makeMethodWrapperHandler(
|
||||
&ExecutorSharedMemoryMapperService::reserve))
|
||||
.release();
|
||||
}
|
||||
|
||||
llvm::orc::shared::CWrapperFunctionResult
|
||||
ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData,
|
||||
size_t ArgSize) {
|
||||
return shared::WrapperFunction<
|
||||
rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>::
|
||||
handle(ArgData, ArgSize,
|
||||
shared::makeMethodWrapperHandler(
|
||||
&ExecutorSharedMemoryMapperService::initialize))
|
||||
.release();
|
||||
}
|
||||
|
||||
llvm::orc::shared::CWrapperFunctionResult
|
||||
ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData,
|
||||
size_t ArgSize) {
|
||||
return shared::WrapperFunction<
|
||||
rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>::
|
||||
handle(ArgData, ArgSize,
|
||||
shared::makeMethodWrapperHandler(
|
||||
&ExecutorSharedMemoryMapperService::deinitialize))
|
||||
.release();
|
||||
}
|
||||
|
||||
llvm::orc::shared::CWrapperFunctionResult
|
||||
ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData,
|
||||
size_t ArgSize) {
|
||||
return shared::WrapperFunction<
|
||||
rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>::
|
||||
handle(ArgData, ArgSize,
|
||||
shared::makeMethodWrapperHandler(
|
||||
&ExecutorSharedMemoryMapperService::release))
|
||||
.release();
|
||||
}
|
||||
|
||||
} // namespace rt_bootstrap
|
||||
} // end namespace orc
|
||||
} // end namespace llvm
|
||||
@@ -30,6 +30,7 @@ add_llvm_unittest(OrcJITTests
|
||||
OrcTestCommon.cpp
|
||||
ResourceTrackerTest.cpp
|
||||
RTDyldObjectLinkingLayerTest.cpp
|
||||
SharedMemoryMapperTest.cpp
|
||||
SimpleExecutorMemoryManagerTest.cpp
|
||||
SimplePackedSerializationTest.cpp
|
||||
SymbolStringPoolTest.cpp
|
||||
|
||||
129
llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp
Normal file
129
llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp
Normal file
@@ -0,0 +1,129 @@
|
||||
//===- SharedMemoryMapperTest.cpp -- Tests for SharedMemoryMapper ---------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "OrcTestCommon.h"
|
||||
|
||||
#include "llvm/ExecutionEngine/Orc/MemoryMapper.h"
|
||||
#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
|
||||
#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"
|
||||
#include "llvm/Testing/Support/Error.h"
|
||||
|
||||
using namespace llvm;
|
||||
using namespace llvm::orc;
|
||||
using namespace llvm::orc::shared;
|
||||
using namespace llvm::orc::rt_bootstrap;
|
||||
|
||||
// A basic function to be used as both initializer/deinitializer
|
||||
orc::shared::CWrapperFunctionResult incrementWrapper(const char *ArgData,
|
||||
size_t ArgSize) {
|
||||
return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
|
||||
ArgData, ArgSize,
|
||||
[](ExecutorAddr A) -> Error {
|
||||
*A.toPtr<int *>() += 1;
|
||||
return Error::success();
|
||||
})
|
||||
.release();
|
||||
}
|
||||
|
||||
TEST(SharedMemoryMapperTest, MemReserveInitializeDeinitializeRelease) {
|
||||
// These counters are used to track how many times the initializer and
|
||||
// deinitializer functions are called
|
||||
int InitializeCounter = 0;
|
||||
int DeinitializeCounter = 0;
|
||||
|
||||
auto SelfEPC = cantFail(SelfExecutorProcessControl::Create());
|
||||
|
||||
ExecutorSharedMemoryMapperService MapperService;
|
||||
|
||||
SharedMemoryMapper::SymbolAddrs SAs;
|
||||
{
|
||||
StringMap<ExecutorAddr> Map;
|
||||
MapperService.addBootstrapSymbols(Map);
|
||||
SAs.Instance = Map[rt::ExecutorSharedMemoryMapperServiceInstanceName];
|
||||
SAs.Reserve = Map[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName];
|
||||
SAs.Initialize =
|
||||
Map[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName];
|
||||
SAs.Deinitialize =
|
||||
Map[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName];
|
||||
SAs.Release = Map[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName];
|
||||
}
|
||||
|
||||
std::string TestString = "Hello, World!";
|
||||
|
||||
// barrier
|
||||
std::promise<void> P;
|
||||
auto F = P.get_future();
|
||||
|
||||
{
|
||||
auto PageSize = cantFail(sys::Process::getPageSize());
|
||||
size_t ReqSize = PageSize;
|
||||
|
||||
std::unique_ptr<MemoryMapper> Mapper =
|
||||
std::make_unique<SharedMemoryMapper>(*SelfEPC, SAs);
|
||||
|
||||
Mapper->reserve(ReqSize, [&](Expected<ExecutorAddrRange> Result) {
|
||||
EXPECT_THAT_ERROR(Result.takeError(), Succeeded());
|
||||
auto Reservation = std::move(*Result);
|
||||
{
|
||||
char *Addr = Mapper->prepare(Reservation.Start, TestString.size() + 1);
|
||||
std::strcpy(Addr, TestString.c_str());
|
||||
}
|
||||
MemoryMapper::AllocInfo AI;
|
||||
{
|
||||
MemoryMapper::AllocInfo::SegInfo SI;
|
||||
SI.Offset = 0;
|
||||
SI.ContentSize = TestString.size() + 1;
|
||||
SI.ZeroFillSize = PageSize - SI.ContentSize;
|
||||
SI.Prot = sys::Memory::MF_READ | sys::Memory::MF_WRITE;
|
||||
|
||||
AI.MappingBase = Reservation.Start;
|
||||
AI.Segments.push_back(SI);
|
||||
AI.Actions.push_back(
|
||||
{cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
|
||||
ExecutorAddr::fromPtr(incrementWrapper),
|
||||
ExecutorAddr::fromPtr(&InitializeCounter))),
|
||||
cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
|
||||
ExecutorAddr::fromPtr(incrementWrapper),
|
||||
ExecutorAddr::fromPtr(&DeinitializeCounter)))});
|
||||
}
|
||||
|
||||
EXPECT_EQ(InitializeCounter, 0);
|
||||
EXPECT_EQ(DeinitializeCounter, 0);
|
||||
|
||||
Mapper->initialize(AI, [&, Reservation](Expected<ExecutorAddr> Result) {
|
||||
EXPECT_THAT_ERROR(Result.takeError(), Succeeded());
|
||||
|
||||
EXPECT_EQ(TestString, std::string(static_cast<char *>(
|
||||
Reservation.Start.toPtr<char *>())));
|
||||
|
||||
EXPECT_EQ(InitializeCounter, 1);
|
||||
EXPECT_EQ(DeinitializeCounter, 0);
|
||||
|
||||
Mapper->deinitialize({*Result}, [&, Reservation](Error Err) {
|
||||
EXPECT_THAT_ERROR(std::move(Err), Succeeded());
|
||||
|
||||
EXPECT_EQ(InitializeCounter, 1);
|
||||
EXPECT_EQ(DeinitializeCounter, 1);
|
||||
|
||||
Mapper->release({Reservation.Start}, [&](Error Err) {
|
||||
EXPECT_THAT_ERROR(std::move(Err), Succeeded());
|
||||
|
||||
P.set_value();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// This will block the test if any of the above callbacks are not executed
|
||||
F.wait();
|
||||
// Mapper must be destructed before calling shutdown to avoid double free
|
||||
}
|
||||
|
||||
EXPECT_THAT_ERROR(MapperService.shutdown(), Succeeded());
|
||||
cantFail(SelfEPC->disconnect());
|
||||
}
|
||||
Reference in New Issue
Block a user