Media
/
compute-runtime
mirror of https://github.com/intel/compute-runtime.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

L0 Virtual Memory Reservation support 

- Enable support for L0 Virtual Memory reservation on Linux and Windows.
- Excludes support for Linux to allow pStart option

Related-To: LOCI-3397, LOCI-1543

Signed-off-by: Spruit, Neil R <neil.r.spruit@intel.com>
This commit is contained in:
Spruit, Neil R 2022-09-17 00:38:06 +00:00 committed by Compute-Runtime-Automation
parent 467119931c
commit d81b0b14a1
25 changed files with 627 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
level_zero/core/source/context/context_imp.cpp
View File

@ -766,18 +766,58 @@ ze_result_t ContextImp::activateMetricGroups(zet_device_handle_t hDevice,
ze_result_t ContextImp::reserveVirtualMem(const void *pStart,
size_t size,
void **pptr) {
return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
if ((getPageSizeRequired(size) != size)) {
return ZE_RESULT_ERROR_UNSUPPORTED_SIZE;
}
NEO::VirtualMemoryReservation *virtualMemoryReservation = new NEO::VirtualMemoryReservation;
virtualMemoryReservation->virtualAddressRange = this->driverHandle->getMemoryManager()->reserveGpuAddress(pStart, size, this->driverHandle->rootDeviceIndices, &virtualMemoryReservation->rootDeviceIndex);
if (pStart != 0x0) {
if (virtualMemoryReservation->virtualAddressRange.address != reinterpret_cast<uint64_t>(pStart)) {
delete virtualMemoryReservation;
return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY;
}
}
if (virtualMemoryReservation->virtualAddressRange.address == 0) {
delete virtualMemoryReservation;
return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY;
}
virtualMemoryReservation->flags.readWrite = false;
virtualMemoryReservation->flags.readOnly = false;
virtualMemoryReservation->flags.noAccess = true;
virtualMemoryReservation->mapped = false;
auto lock = this->driverHandle->getMemoryManager()->lockVirtualMemoryReservationMap();
this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().insert(std::pair<void *, NEO::VirtualMemoryReservation *>(reinterpret_cast<void *>(virtualMemoryReservation->virtualAddressRange.address), virtualMemoryReservation));
*pptr = reinterpret_cast<void *>(virtualMemoryReservation->virtualAddressRange.address);
return ZE_RESULT_SUCCESS;
}
ze_result_t ContextImp::freeVirtualMem(const void *ptr,
size_t size) {
return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
std::map<void *, NEO::VirtualMemoryReservation *>::iterator it;
auto lock = this->driverHandle->getMemoryManager()->lockVirtualMemoryReservationMap();
it = this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().find(const_cast<void *>(ptr));
if (it != this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().end()) {
NEO::VirtualMemoryReservation *virtualMemoryReservation = it->second;
if (virtualMemoryReservation->virtualAddressRange.size != size) {
return ZE_RESULT_ERROR_INVALID_ARGUMENT;
}
this->driverHandle->getMemoryManager()->freeGpuAddress(virtualMemoryReservation->virtualAddressRange, virtualMemoryReservation->rootDeviceIndex);
delete virtualMemoryReservation;
this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().erase(it);
return ZE_RESULT_SUCCESS;
} else {
return ZE_RESULT_ERROR_INVALID_ARGUMENT;
}
}
size_t ContextImp::getPageSizeRequired(size_t size) {
return std::max(Math::prevPowerOfTwo(size), MemoryConstants::pageSize64k);
}
ze_result_t ContextImp::queryVirtualMemPageSize(ze_device_handle_t hDevice,
size_t size,
size_t *pagesize) {
*pagesize = std::max(Math::prevPowerOfTwo(size), MemoryConstants::pageSize64k);
*pagesize = getPageSizeRequired(size);
return ZE_RESULT_SUCCESS;
}
@ -807,14 +847,57 @@ ze_result_t ContextImp::unMapVirtualMem(const void *ptr,
ze_result_t ContextImp::setVirtualMemAccessAttribute(const void *ptr,
size_t size,
ze_memory_access_attribute_t access) {
return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
std::map<void *, NEO::VirtualMemoryReservation *>::iterator it;
auto lock = this->driverHandle->getMemoryManager()->lockVirtualMemoryReservationMap();
it = this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().find(const_cast<void *>(ptr));
if (it != this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().end()) {
NEO::VirtualMemoryReservation *virtualMemoryReservation = this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().at(const_cast<void *>(ptr));
switch (access) {
case ZE_MEMORY_ACCESS_ATTRIBUTE_NONE:
virtualMemoryReservation->flags.readOnly = false;
virtualMemoryReservation->flags.noAccess = true;
virtualMemoryReservation->flags.readWrite = false;
break;
case ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE:
virtualMemoryReservation->flags.readOnly = false;
virtualMemoryReservation->flags.noAccess = false;
virtualMemoryReservation->flags.readWrite = true;
break;
case ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY:
virtualMemoryReservation->flags.readWrite = false;
virtualMemoryReservation->flags.noAccess = false;
virtualMemoryReservation->flags.readOnly = true;
break;
default:
return ZE_RESULT_ERROR_INVALID_ENUMERATION;
}
return ZE_RESULT_SUCCESS;
} else {
return ZE_RESULT_ERROR_INVALID_ARGUMENT;
}
}
ze_result_t ContextImp::getVirtualMemAccessAttribute(const void *ptr,
size_t size,
ze_memory_access_attribute_t *access,
size_t *outSize) {
return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
std::map<void *, NEO::VirtualMemoryReservation *>::iterator it;
auto lock = this->driverHandle->getMemoryManager()->lockVirtualMemoryReservationMap();
it = this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().find(const_cast<void *>(ptr));
if (it != this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().end()) {
NEO::VirtualMemoryReservation *virtualMemoryReservation = this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().at(const_cast<void *>(ptr));
if (virtualMemoryReservation->flags.readWrite) {
*access = ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE;
} else if (virtualMemoryReservation->flags.readOnly) {
*access = ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY;
} else {
*access = ZE_MEMORY_ACCESS_ATTRIBUTE_NONE;
}
*outSize = virtualMemoryReservation->virtualAddressRange.size;
return ZE_RESULT_SUCCESS;
} else {
return ZE_RESULT_ERROR_INVALID_ARGUMENT;
}
}
ze_result_t ContextImp::createEventPool(const ze_event_pool_desc_t *desc,

1
level_zero/core/source/context/context_imp.h
View File

@ -162,6 +162,7 @@ struct ContextImp : Context {
protected:
bool isAllocationSuitableForCompression(const StructuresLookupTable &structuresLookupTable, Device &device, size_t allocSize);
size_t getPageSizeRequired(size_t size);
std::map<uint32_t, ze_device_handle_t> devices;
std::vector<ze_device_handle_t> deviceHandles;

4
level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h
View File

@ -401,7 +401,7 @@ class MemoryManagerIpcMock : public NEO::MemoryManager {
uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override { return 0; };
uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override { return 0; };
double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override { return 0; }
AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override {
AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override {
return {};
}
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{};
@ -608,7 +608,7 @@ class MemoryManagerIpcImplicitScalingMock : public NEO::MemoryManager {
uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override { return 0; };
uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override { return 0; };
double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override { return 0; }
AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override {
AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override {
return {};
}
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{};

316
level_zero/core/test/unit_tests/sources/context/test_context.cpp
View File

@ -788,7 +788,7 @@ TEST_F(ContextTest, whenGettingDriverThenDriverIsRetrievedSuccessfully) {
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
}
TEST_F(ContextTest, whenCallingVirtualMemInterfacesThenUnsupportedIsReturned) {
TEST_F(ContextTest, whenCallingVirtualMemInterfacesThenSuccessIsReturned) {
ze_context_handle_t hContext;
ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0};
@ -798,13 +798,17 @@ TEST_F(ContextTest, whenCallingVirtualMemInterfacesThenUnsupportedIsReturned) {
ContextImp *contextImp = static_cast<ContextImp *>(L0::Context::fromHandle(hContext));
void *pStart = 0x0;
size_t size = 0u;
size_t size = 4096u;
void *ptr = nullptr;
res = contextImp->reserveVirtualMem(pStart, size, &ptr);
EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res);
size_t pagesize = 0u;
res = contextImp->queryVirtualMemPageSize(device, size, &pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0);
res = contextImp->freeVirtualMem(ptr, size);
EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res);
res = contextImp->freeVirtualMem(ptr, pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
res = contextImp->destroy();
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
@ -871,6 +875,16 @@ TEST_F(ContextTest, whenCallingMappingVirtualInterfacesThenUnsupportedIsReturned
ContextImp *contextImp = static_cast<ContextImp *>(L0::Context::fromHandle(hContext));
void *pStart = 0x0;
size_t size = 4096u;
void *ptr = nullptr;
size_t pagesize = 0u;
res = contextImp->queryVirtualMemPageSize(device, size, &pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0);
ze_physical_mem_desc_t descMem = {};
ze_physical_mem_handle_t mem = {};
res = contextImp->createPhysicalMem(device, &descMem, &mem);
@ -878,25 +892,299 @@ TEST_F(ContextTest, whenCallingMappingVirtualInterfacesThenUnsupportedIsReturned
ze_memory_access_attribute_t access = {};
size_t offset = 0;
void *ptr = nullptr;
size_t size = 0;
res = contextImp->mapVirtualMem(ptr, size, mem, offset, access);
res = contextImp->mapVirtualMem(ptr, pagesize, mem, offset, access);
EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res);
res = contextImp->setVirtualMemAccessAttribute(ptr, size, access);
EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res);
res = contextImp->setVirtualMemAccessAttribute(ptr, pagesize, access);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
ze_memory_access_attribute_t outAccess = {};
size_t outSize = 0;
res = contextImp->getVirtualMemAccessAttribute(ptr, size, &outAccess, &outSize);
EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res);
res = contextImp->getVirtualMemAccessAttribute(ptr, pagesize, &outAccess, &outSize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
EXPECT_EQ(pagesize, outSize);
res = contextImp->unMapVirtualMem(ptr, size);
res = contextImp->unMapVirtualMem(ptr, pagesize);
EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res);
res = contextImp->destroyPhysicalMem(mem);
EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res);
res = contextImp->freeVirtualMem(ptr, pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
res = contextImp->destroy();
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
}
TEST_F(ContextTest, whenCallingVirtualMemorySetAttributeWithValidEnumerationsThenSuccessisReturned) {
ze_context_handle_t hContext;
ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0};
ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
ContextImp *contextImp = static_cast<ContextImp *>(L0::Context::fromHandle(hContext));
void *pStart = 0x0;
size_t size = 4096;
void *ptr = nullptr;
size_t pagesize = 0u;
res = contextImp->queryVirtualMemPageSize(device, size, &pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0);
std::vector<ze_memory_access_attribute_t> memoryAccessFlags = {
ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE, ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY,
ZE_MEMORY_ACCESS_ATTRIBUTE_NONE};
ze_memory_access_attribute_t access = {ZE_MEMORY_ACCESS_ATTRIBUTE_NONE};
ze_memory_access_attribute_t previousAccess = {};
size_t outSize = 0;
for (auto accessFlags : memoryAccessFlags) {
previousAccess = access;
res = contextImp->setVirtualMemAccessAttribute(ptr, pagesize, accessFlags);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
res = contextImp->getVirtualMemAccessAttribute(ptr, pagesize, &access, &outSize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
EXPECT_EQ(outSize, pagesize);
EXPECT_NE(previousAccess, access);
EXPECT_EQ(accessFlags, access);
}
res = contextImp->getVirtualMemAccessAttribute(ptr, pagesize, &access, &outSize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
res = contextImp->freeVirtualMem(ptr, pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
res = contextImp->destroy();
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
}
TEST_F(ContextTest, whenCallingVirtualMemorySetAttributeWithInvalidValuesThenFailureReturned) {
ze_context_handle_t hContext;
ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0};
ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
ContextImp *contextImp = static_cast<ContextImp *>(L0::Context::fromHandle(hContext));
void *pStart = 0x0;
size_t size = 4096u;
void *ptr = nullptr;
size_t pagesize = 0u;
res = contextImp->queryVirtualMemPageSize(device, size, &pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0);
ze_memory_access_attribute_t access = {ZE_MEMORY_ACCESS_ATTRIBUTE_FORCE_UINT32};
res = contextImp->setVirtualMemAccessAttribute(ptr, pagesize, access);
EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ENUMERATION, res);
res = contextImp->setVirtualMemAccessAttribute(pStart, pagesize, access);
EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res);
res = contextImp->freeVirtualMem(ptr, pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
res = contextImp->destroy();
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
}
TEST_F(ContextTest, whenCallingVirtualMemoryGetAttributeWithInvalidValuesThenFailureReturned) {
ze_context_handle_t hContext;
ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0};
ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
ContextImp *contextImp = static_cast<ContextImp *>(L0::Context::fromHandle(hContext));
void *pStart = 0x0;
size_t size = 4096u;
void *ptr = nullptr;
size_t pagesize = 0u;
res = contextImp->queryVirtualMemPageSize(device, size, &pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0);
size_t outSize = 0;
ze_memory_access_attribute_t access = {ZE_MEMORY_ACCESS_ATTRIBUTE_NONE};
res = contextImp->getVirtualMemAccessAttribute(pStart, pagesize, &access, &outSize);
EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res);
res = contextImp->freeVirtualMem(ptr, pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
res = contextImp->destroy();
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
}
TEST_F(ContextTest, whenCallingVirtualMemoryFreeWithInvalidValuesThenFailuresReturned) {
ze_context_handle_t hContext;
ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0};
ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
ContextImp *contextImp = static_cast<ContextImp *>(L0::Context::fromHandle(hContext));
void *pStart = 0x0;
size_t size = 4096u;
size_t invalidSize = 10u;
void *ptr = nullptr;
size_t pagesize = 0u;
res = contextImp->queryVirtualMemPageSize(device, size, &pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0);
res = contextImp->freeVirtualMem(ptr, invalidSize);
EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res);
res = contextImp->freeVirtualMem(pStart, pagesize);
EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res);
res = contextImp->freeVirtualMem(ptr, pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
res = contextImp->destroy();
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
}
TEST_F(ContextTest, whenCallingVirtualMemoryReservationWithInvalidArgumentsThenFailureReturned) {
ze_context_handle_t hContext;
ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0};
ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
ContextImp *contextImp = static_cast<ContextImp *>(L0::Context::fromHandle(hContext));
void *pStart = 0x0;
size_t size = 64u;
void *ptr = nullptr;
res = contextImp->reserveVirtualMem(pStart, size, &ptr);
EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_SIZE, res);
size_t pagesize = 0u;
res = contextImp->queryVirtualMemPageSize(device, size, &pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
pStart = reinterpret_cast<void *>(0x1234);
res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr);
EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY, res);
res = contextImp->destroy();
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
}
class ReserveMemoryManagerMock : public NEO::MemoryManager {
public:
ReserveMemoryManagerMock(NEO::ExecutionEnvironment &executionEnvironment) : NEO::MemoryManager(executionEnvironment) {}
NEO::GraphicsAllocation *createGraphicsAllocationFromMultipleSharedHandles(const std::vector<osHandle> &handles, AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override { return nullptr; }
NEO::GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override { return nullptr; }
void addAllocationToHostPtrManager(NEO::GraphicsAllocation *memory) override{};
void removeAllocationFromHostPtrManager(NEO::GraphicsAllocation *memory) override{};
NEO::GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex, AllocationType allocType) override { return nullptr; };
AllocationStatus populateOsHandles(NEO::OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override { return AllocationStatus::Success; };
void cleanOsHandles(NEO::OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override{};
void freeGraphicsMemoryImpl(NEO::GraphicsAllocation *gfxAllocation) override{};
void freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation, bool isImportedAllocation) override{};
uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override { return 0; };
uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override { return 0; };
double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override { return 0; }
AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override {
if (failReserveGpuAddress) {
return {};
}
return AddressRange{reinterpret_cast<uint64_t>(requiredStartAddress), size};
}
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{};
NEO::GraphicsAllocation *createGraphicsAllocation(OsHandleStorage &handleStorage, const NEO::AllocationData &allocationData) override { return nullptr; };
NEO::GraphicsAllocation *allocateGraphicsMemoryForNonSvmHostPtr(const NEO::AllocationData &allocationData) override { return nullptr; };
NEO::GraphicsAllocation *allocateGraphicsMemoryWithAlignment(const NEO::AllocationData &allocationData) override { return nullptr; };
NEO::GraphicsAllocation *allocateUSMHostGraphicsMemory(const NEO::AllocationData &allocationData) override { return nullptr; };
NEO::GraphicsAllocation *allocateGraphicsMemory64kb(const NEO::AllocationData &allocationData) override { return nullptr; };
NEO::GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const NEO::AllocationData &allocationData, bool useLocalMemory) override { return nullptr; };
NEO::GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const NEO::AllocationData &allocationData, AllocationStatus &status) override { return nullptr; };
NEO::GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const NEO::AllocationData &allocationData) override { return nullptr; };
NEO::GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const NEO::AllocationData &allocationData, std::unique_ptr<Gmm> gmm) override { return nullptr; };
NEO::GraphicsAllocation *allocateMemoryByKMD(const NEO::AllocationData &allocationData) override { return nullptr; };
void *lockResourceImpl(NEO::GraphicsAllocation &graphicsAllocation) override { return nullptr; };
void unlockResourceImpl(NEO::GraphicsAllocation &graphicsAllocation) override{};
bool failReserveGpuAddress = true;
};
TEST_F(ContextTest, whenCallingVirtualMemReserveWithPStartWithSuccessfulAllocationThenSuccessReturned) {
ze_context_handle_t hContext;
ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0};
ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
ContextImp *contextImp = static_cast<ContextImp *>(L0::Context::fromHandle(hContext));
void *pStart = reinterpret_cast<void *>(0x1234);
size_t size = 4096u;
void *ptr = nullptr;
size_t pagesize = 0u;
res = contextImp->queryVirtualMemPageSize(device, size, &pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
ReserveMemoryManagerMock *ReserveMemoryManager = new ReserveMemoryManagerMock(*neoDevice->executionEnvironment);
auto memoryManager = driverHandle->getMemoryManager();
ReserveMemoryManager->failReserveGpuAddress = false;
NEO::MemoryManager *mockingMemoryManager = (NEO::MemoryManager *)ReserveMemoryManager;
driverHandle->setMemoryManager(mockingMemoryManager);
res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
EXPECT_GT((int)mockingMemoryManager->getVirtualMemoryReservationMap().size(), 0);
res = contextImp->freeVirtualMem(ptr, pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
driverHandle->setMemoryManager(memoryManager);
delete ReserveMemoryManager;
res = contextImp->destroy();
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
}
TEST_F(ContextTest, whenCallingVirtualMemoryReservationWhenOutOfMemoryThenOutOfMemoryReturned) {
ze_context_handle_t hContext;
ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0};
ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
ContextImp *contextImp = static_cast<ContextImp *>(L0::Context::fromHandle(hContext));
void *pStart = 0x0;
size_t size = 0u;
void *ptr = nullptr;
size_t pagesize = 0u;
res = contextImp->queryVirtualMemPageSize(device, size, &pagesize);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
NEO::MemoryManager *failingReserveMemoryManager = new ReserveMemoryManagerMock(*neoDevice->executionEnvironment);
auto memoryManager = driverHandle->getMemoryManager();
driverHandle->setMemoryManager(failingReserveMemoryManager);
res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr);
EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY, res);
pStart = reinterpret_cast<void *>(0x1234);
res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr);
EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY, res);
driverHandle->setMemoryManager(memoryManager);
delete failingReserveMemoryManager;
res = contextImp->destroy();
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
}

2
level_zero/core/test/unit_tests/sources/event/test_event.cpp
View File

@ -63,7 +63,7 @@ class MemoryManagerEventPoolFailMock : public NEO::MemoryManager {
uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override { return 0; };
uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override { return 0; };
double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override { return 0; }
AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override {
AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override {
return {};
}
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{};

4
level_zero/core/test/unit_tests/sources/memory/linux/test_memory_linux.cpp
View File

@ -65,7 +65,7 @@ class MemoryManagerIpcImplicitScalingObtainFdMock : public NEO::DrmMemoryManager
uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override { return 0; };
uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override { return 0; };
double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override { return 0; }
AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override {
AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override {
return {};
}
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{};
@ -497,7 +497,7 @@ class MemoryManagerIpcObtainFdMock : public NEO::DrmMemoryManager {
uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override { return 0; };
uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override { return 0; };
double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override { return 0; }
AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override {
AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override {
return {};
}
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{};

32
opencl/test/unit_test/memory_manager/memory_manager_tests.cpp
View File

@ -1628,15 +1628,43 @@ TEST(OsAgnosticMemoryManager, givenOsAgnosticMemoryManagerAndFreeMemoryDisabledW
TEST(OsAgnosticMemoryManager, givenOsAgnosticMemoryManagerWhenGpuAddressIsReservedAndFreedThenAddressFromGfxPartitionIsUsed) {
MockExecutionEnvironment executionEnvironment;
OsAgnosticMemoryManager memoryManager(executionEnvironment);
auto addressRange = memoryManager.reserveGpuAddress(MemoryConstants::pageSize, 0);
RootDeviceIndicesContainer rootDevices;
rootDevices.push_back(0);
uint32_t rootDeviceIndexReserved = 10;
auto addressRange = memoryManager.reserveGpuAddress(nullptr, MemoryConstants::pageSize, rootDevices, &rootDeviceIndexReserved);
auto gmmHelper = memoryManager.getGmmHelper(0);
EXPECT_EQ(0u, rootDeviceIndexReserved);
EXPECT_LE(memoryManager.getGfxPartition(0)->getHeapBase(HeapIndex::HEAP_STANDARD), gmmHelper->decanonize(addressRange.address));
EXPECT_GT(memoryManager.getGfxPartition(0)->getHeapLimit(HeapIndex::HEAP_STANDARD), gmmHelper->decanonize(addressRange.address));
memoryManager.freeGpuAddress(addressRange, 0);
}
TEST(OsAgnosticMemoryManager, givenOsAgnosticMemoryManagerWhenGpuAddressReservationIsAttemptedWihtInvalidSizeThenFailureReturnsNullAddressRange) {
MockExecutionEnvironment executionEnvironment;
OsAgnosticMemoryManager memoryManager(executionEnvironment);
RootDeviceIndicesContainer rootDevices;
rootDevices.push_back(0);
uint32_t rootDeviceIndexReserved = 10;
// emulate GPU address space exhaust
memoryManager.getGfxPartition(0)->heapInit(HeapIndex::HEAP_STANDARD, 0x0, 0x10000);
auto addressRange = memoryManager.reserveGpuAddress(nullptr, (size_t)(memoryManager.getGfxPartition(0)->getHeapLimit(HeapIndex::HEAP_STANDARD) * 2), rootDevices, &rootDeviceIndexReserved);
EXPECT_EQ(static_cast<int>(addressRange.address), 0);
}
TEST(OsAgnosticMemoryManager, givenOsAgnosticMemoryManagerWhenGpuAddressReservationIsAttemptedWithARequiredPtrThenNullRangeReturned) {
MockExecutionEnvironment executionEnvironment;
OsAgnosticMemoryManager memoryManager(executionEnvironment);
RootDeviceIndicesContainer rootDevices;
rootDevices.push_back(0);
uint32_t rootDeviceIndexReserved = 10;
void *requiredPtr = reinterpret_cast<void *>(0x1234);
auto addressRange = memoryManager.reserveGpuAddress(requiredPtr, MemoryConstants::pageSize, rootDevices, &rootDeviceIndexReserved);
EXPECT_EQ(0u, rootDeviceIndexReserved);
EXPECT_EQ(static_cast<int>(addressRange.address), 0);
EXPECT_EQ(static_cast<int>(addressRange.size), 0);
}
TEST(OsAgnosticMemoryManager, givenOsAgnosticMemoryManagerWhenCheckedForIndirectAllocationsAsPackSupportThenFalseIsReturned) {
MockExecutionEnvironment executionEnvironment;
OsAgnosticMemoryManager memoryManager(executionEnvironment);

2
opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp
View File

@ -283,7 +283,7 @@ TEST_F(Wddm20WithMockGdiDllTests, givenAllocationSmallerUnderlyingThanAlignedSiz
TEST_F(Wddm20WithMockGdiDllTests, givenReserveCallWhenItIsCalledWithProperParamtersThenAddressInRangeIsReturend) {
auto sizeAlignedTo64Kb = 64 * KB;
auto reservationAddress = wddm->reserveGpuVirtualAddress(wddm->getGfxPartition().Heap32[0].Base,
auto reservationAddress = wddm->reserveGpuVirtualAddress(0ull, wddm->getGfxPartition().Heap32[0].Base,
wddm->getGfxPartition().Heap32[0].Limit,
sizeAlignedTo64Kb);

13
opencl/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp
View File

@ -564,12 +564,15 @@ TEST_F(WddmMemoryManagerSimpleTest, givenNonZeroFenceValueOnSomeOfMultipleEngine
memoryManager->freeGraphicsMemory(allocation);
}
TEST_F(WddmMemoryManagerSimpleTest, givenWddmMemoryManagerWhenGpuAddressIsReservedAndFreedThenAddressRangeIsZero) {
auto addressRange = memoryManager->reserveGpuAddress(MemoryConstants::pageSize, 0);
TEST_F(WddmMemoryManagerSimpleTest, givenWddmMemoryManagerWhenGpuAddressIsReservedAndFreedThenAddressRangeIsNonZero) {
RootDeviceIndicesContainer rootDevices;
rootDevices.push_back(0);
uint32_t rootDeviceIndexReserved = 1;
auto addressRange = memoryManager->reserveGpuAddress(nullptr, MemoryConstants::pageSize64k, rootDevices, &rootDeviceIndexReserved);
auto gmmHelper = memoryManager->getGmmHelper(0);
EXPECT_EQ(0u, gmmHelper->decanonize(addressRange.address));
EXPECT_EQ(0u, addressRange.size);
EXPECT_EQ(0u, rootDeviceIndexReserved);
EXPECT_NE(0u, gmmHelper->decanonize(addressRange.address));
EXPECT_EQ(MemoryConstants::pageSize64k, addressRange.size);
memoryManager->freeGpuAddress(addressRange, 0);
}

5
shared/source/debugger/debugger_l0.cpp
View File

@ -51,7 +51,10 @@ void DebuggerL0::initialize() {
device->getDeviceBitfield()};
if (!singleAddressSpaceSbaTracking) {
sbaTrackingGpuVa = device->getMemoryManager()->reserveGpuAddress(MemoryConstants::pageSize, device->getRootDeviceIndex());
RootDeviceIndicesContainer rootDevices;
rootDevices.push_back(device->getRootDeviceIndex());
uint32_t rootDeviceIndexReserved = 0;
sbaTrackingGpuVa = device->getMemoryManager()->reserveGpuAddress(nullptr, MemoryConstants::pageSize, rootDevices, &rootDeviceIndexReserved);
properties.gpuAddress = sbaTrackingGpuVa.address;
}

13
shared/source/memory_manager/memory_manager.h
View File

@ -52,6 +52,13 @@ struct AddressRange {
size_t size;
};
struct VirtualMemoryReservation {
AddressRange virtualAddressRange;
MemoryFlags flags;
bool mapped;
uint32_t rootDeviceIndex;
};
constexpr size_t paddingBufferSize = 2 * MemoryConstants::megaByte;
namespace MemoryTransferHelper {
@ -211,7 +218,7 @@ class MemoryManager {
GmmHelper *getGmmHelper(uint32_t rootDeviceIndex) {
return executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->getGmmHelper();
}
virtual AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) = 0;
virtual AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) = 0;
virtual void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) = 0;
static HeapIndex selectInternalHeap(bool useLocalMemory) { return useLocalMemory ? HeapIndex::HEAP_INTERNAL_DEVICE_MEMORY : HeapIndex::HEAP_INTERNAL; }
static HeapIndex selectExternalHeap(bool useLocalMemory) { return useLocalMemory ? HeapIndex::HEAP_EXTERNAL_DEVICE_MEMORY : HeapIndex::HEAP_EXTERNAL; }
@ -264,6 +271,8 @@ class MemoryManager {
std::unordered_map<std::string, KernelAllocationInfo> &getKernelAllocationMap() { return this->kernelAllocationMap; };
[[nodiscard]] std::unique_lock<std::mutex> lockKernelAllocationMap() { return std::unique_lock<std::mutex>(this->kernelAllocationMutex); };
std::map<void *, VirtualMemoryReservation *> &getVirtualMemoryReservationMap() { return this->virtualMemoryReservationMap; };
[[nodiscard]] std::unique_lock<std::mutex> lockVirtualMemoryReservationMap() { return std::unique_lock<std::mutex>(this->virtualMemoryReservationMapMutex); };
protected:
bool getAllocationData(AllocationData &allocationData, const AllocationProperties &properties, const void *hostPtr, const StorageInfo &storageInfo);
@ -329,6 +338,8 @@ class MemoryManager {
std::vector<bool> isaInLocalMemory;
std::unordered_map<std::string, KernelAllocationInfo> kernelAllocationMap;
std::mutex kernelAllocationMutex;
std::map<void *, VirtualMemoryReservation *> virtualMemoryReservationMap;
std::mutex virtualMemoryReservationMapMutex;
};
std::unique_ptr<DeferredDeleter> createDeferredDeleter();

19
shared/source/memory_manager/os_agnostic_memory_manager.cpp
View File

@ -453,10 +453,21 @@ MemoryAllocation *OsAgnosticMemoryManager::createMemoryAllocation(AllocationType
return memoryAllocation;
}
AddressRange OsAgnosticMemoryManager::reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) {
auto gfxPartition = getGfxPartition(rootDeviceIndex);
auto gmmHelper = getGmmHelper(rootDeviceIndex);
auto gpuVa = gmmHelper->canonize(gfxPartition->heapAllocate(HeapIndex::HEAP_STANDARD, size));
AddressRange OsAgnosticMemoryManager::reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) {
uint64_t gpuVa = 0u;
*reservedOnRootDeviceIndex = 0;
if (requiredStartAddress) {
return AddressRange{0, 0};
}
for (uint32_t rootDeviceIndex = 0; rootDeviceIndex < rootDeviceIndices.size(); rootDeviceIndex++) {
auto gfxPartition = getGfxPartition(rootDeviceIndex);
auto gmmHelper = getGmmHelper(rootDeviceIndex);
gpuVa = gmmHelper->canonize(gfxPartition->heapAllocate(HeapIndex::HEAP_STANDARD, size));
if (gpuVa != 0u) {
*reservedOnRootDeviceIndex = rootDeviceIndex;
break;
}
}
return AddressRange{gpuVa, size};
}

2
shared/source/memory_manager/os_agnostic_memory_manager.h
View File

@ -90,7 +90,7 @@ class OsAgnosticMemoryManager : public MemoryManager {
void *reserveCpuAddressRange(size_t size, uint32_t rootDeviceIndex) override;
void releaseReservedCpuAddressRange(void *reserved, size_t size, uint32_t rootDeviceIndex) override;
AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override;
AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override;
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override;
bool is64kbPagesEnabled(const HardwareInfo *hwInfo);

15
shared/source/os_interface/linux/drm_memory_manager.cpp
View File

@ -1156,8 +1156,19 @@ uint32_t DrmMemoryManager::getRootDeviceIndex(const Drm *drm) {
return CommonConstants::unspecifiedDeviceIndex;
}
AddressRange DrmMemoryManager::reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) {
auto gpuVa = acquireGpuRange(size, rootDeviceIndex, HeapIndex::HEAP_STANDARD);
AddressRange DrmMemoryManager::reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) {
uint64_t gpuVa = 0u;
*reservedOnRootDeviceIndex = 0;
if (requiredStartAddress) {
return AddressRange{0, 0};
}
for (uint32_t rootDeviceIndex = 0; rootDeviceIndex < rootDeviceIndices.size(); rootDeviceIndex++) {
gpuVa = acquireGpuRange(size, rootDeviceIndex, HeapIndex::HEAP_STANDARD);
if (gpuVa != 0u) {
*reservedOnRootDeviceIndex = rootDeviceIndex;
break;
}
}
return AddressRange{gpuVa, size};
}

2
shared/source/os_interface/linux/drm_memory_manager.h
View File

@ -62,7 +62,7 @@ class DrmMemoryManager : public MemoryManager {
bool copyMemoryToAllocationBanks(GraphicsAllocation *graphicsAllocation, size_t destinationOffset, const void *memoryToCopy, size_t sizeToCopy, DeviceBitfield handleMask) override;
MOCKABLE_VIRTUAL int obtainFdFromHandle(int boHandle, uint32_t rootDeviceindex);
AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override;
AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override;
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override;
MOCKABLE_VIRTUAL BufferObject *createBufferObjectInMemoryRegion(Drm *drm, Gmm *gmm, AllocationType allocationType, uint64_t gpuAddress, size_t size,
uint32_t memoryBanks, size_t maxOsContextCount, int32_t pairHandle);

4
shared/source/os_interface/windows/wddm/wddm.cpp
View File

@ -558,11 +558,13 @@ bool Wddm::mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_A
return ret;
}
D3DGPU_VIRTUAL_ADDRESS Wddm::reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS minimumAddress,
D3DGPU_VIRTUAL_ADDRESS Wddm::reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS baseAddress,
D3DGPU_VIRTUAL_ADDRESS minimumAddress,
D3DGPU_VIRTUAL_ADDRESS maximumAddress,
D3DGPU_SIZE_T size) {
UNRECOVERABLE_IF(size % MemoryConstants::pageSize64k);
D3DDDI_RESERVEGPUVIRTUALADDRESS reserveGpuVirtualAddress = {};
reserveGpuVirtualAddress.BaseAddress = baseAddress;
reserveGpuVirtualAddress.MinimumAddress = minimumAddress;
reserveGpuVirtualAddress.MaximumAddress = maximumAddress;
reserveGpuVirtualAddress.hPagingQueue = this->pagingQueue;

2
shared/source/os_interface/windows/wddm/wddm.h
View File

@ -64,7 +64,7 @@ class Wddm : public DriverModel {
MOCKABLE_VIRTUAL bool makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t totalSize);
MOCKABLE_VIRTUAL bool mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_VIRTUAL_ADDRESS preferredAddress, D3DGPU_VIRTUAL_ADDRESS &gpuPtr);
bool mapGpuVirtualAddress(AllocationStorageData *allocationStorageData);
MOCKABLE_VIRTUAL D3DGPU_VIRTUAL_ADDRESS reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size);
MOCKABLE_VIRTUAL D3DGPU_VIRTUAL_ADDRESS reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS baseAddress, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size);
MOCKABLE_VIRTUAL bool createContext(OsContextWin &osContext);
MOCKABLE_VIRTUAL void applyAdditionalContextFlags(CREATECONTEXT_PVTDATA &privateData, OsContextWin &osContext, const HardwareInfo &hwInfo);
MOCKABLE_VIRTUAL void applyAdditionalMapGPUVAFields(D3DDDI_MAPGPUVIRTUALADDRESS &mapGPUVA, Gmm *gmm);

22
shared/source/os_interface/windows/wddm_memory_manager.cpp
View File

@ -749,6 +749,26 @@ bool WddmMemoryManager::createWddmAllocation(WddmAllocation *allocation, void *r
return mapGpuVirtualAddress(allocation, requiredGpuPtr);
}
AddressRange WddmMemoryManager::reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) {
uint64_t gpuVa = 0u;
*reservedOnRootDeviceIndex = 0;
size_t reservedSize = 0;
for (uint32_t rootDeviceIndex = 0; rootDeviceIndex < rootDeviceIndices.size(); rootDeviceIndex++) {
auto gfxPartition = getGfxPartition(rootDeviceIndex);
gpuVa = getWddm(rootDeviceIndex).reserveGpuVirtualAddress(reinterpret_cast<uint64_t>(requiredStartAddress), gfxPartition->getHeapMinimalAddress(HeapIndex::HEAP_STANDARD64KB), gfxPartition->getHeapLimit(HeapIndex::HEAP_STANDARD64KB), size);
if (gpuVa != 0u) {
*reservedOnRootDeviceIndex = rootDeviceIndex;
reservedSize = size;
break;
}
}
return AddressRange{gpuVa, reservedSize};
}
void WddmMemoryManager::freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) {
getWddm(rootDeviceIndex).freeGpuVirtualAddress(addressRange.address, addressRange.size);
}
bool WddmMemoryManager::mapGpuVaForOneHandleAllocation(WddmAllocation *allocation, const void *preferredGpuVirtualAddress) {
D3DGPU_VIRTUAL_ADDRESS addressToMap = castToUint64(preferredGpuVirtualAddress);
auto heapIndex = selectHeap(allocation, preferredGpuVirtualAddress != nullptr, is32bit || executionEnvironment.rootDeviceEnvironments[allocation->getRootDeviceIndex()]->isFullRangeSvm(), allocation->allocInFrontWindowPool);
@ -795,7 +815,7 @@ bool WddmMemoryManager::mapMultiHandleAllocationWithRetry(WddmAllocation *alloca
allocation->getGpuAddressToModify() = addressToMap;
} else {
allocation->reservedSizeForGpuVirtualAddress = alignUp(alignedSize, MemoryConstants::pageSize64k);
allocation->reservedGpuVirtualAddress = wddm.reserveGpuVirtualAddress(gfxPartition->getHeapMinimalAddress(heapIndex), gfxPartition->getHeapLimit(heapIndex),
allocation->reservedGpuVirtualAddress = wddm.reserveGpuVirtualAddress(0ull, gfxPartition->getHeapMinimalAddress(heapIndex), gfxPartition->getHeapLimit(heapIndex),
allocation->reservedSizeForGpuVirtualAddress);
auto gmmHelper = getGmmHelper(allocation->getRootDeviceIndex());
allocation->getGpuAddressToModify() = gmmHelper->canonize(allocation->reservedGpuVirtualAddress);

4
shared/source/os_interface/windows/wddm_memory_manager.h
View File

@ -73,8 +73,8 @@ class WddmMemoryManager : public MemoryManager {
bool isCpuCopyRequired(const void *ptr) override;
bool isWCMemory(const void *ptr) override;
AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override { return AddressRange{0, 0}; };
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{};
AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override;
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override;
bool verifyHandle(osHandle handle, uint32_t rootDeviceIndex, bool ntHandle) override;
bool isNTHandle(osHandle handle, uint32_t rootDeviceIndex) override;
void releaseDeviceSpecificMemResources(uint32_t rootDeviceIndex) override{};

8
shared/test/common/mocks/mock_wddm.cpp
View File

@ -292,9 +292,13 @@ VOID *WddmMock::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback
return Wddm::registerTrimCallback(callback, residencyController);
}
D3DGPU_VIRTUAL_ADDRESS WddmMock::reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size) {
D3DGPU_VIRTUAL_ADDRESS WddmMock::reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS baseAddress, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size) {
reserveGpuVirtualAddressResult.called++;
return Wddm::reserveGpuVirtualAddress(minimumAddress, maximumAddress, size);
if (failReserveGpuVirtualAddress) {
D3DGPU_VIRTUAL_ADDRESS nullAddress = {};
return nullAddress;
}
return Wddm::reserveGpuVirtualAddress(baseAddress, minimumAddress, maximumAddress, size);
}
uint64_t *WddmMock::getPagingFenceAddress() {

3
shared/test/common/mocks/mock_wddm.h
View File

@ -89,7 +89,7 @@ class WddmMock : public Wddm {
void virtualFree(void *ptr, size_t size) override;
void releaseReservedAddress(void *reservedAddress) override;
VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController) override;
D3DGPU_VIRTUAL_ADDRESS reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size) override;
D3DGPU_VIRTUAL_ADDRESS reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS baseAddress, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size) override;
bool reserveValidAddressRange(size_t size, void *&reservedMem) override;
PLATFORM *getGfxPlatform() { return gfxPlatform.get(); }
uint64_t *getPagingFenceAddress() override;
@ -185,5 +185,6 @@ class WddmMock : public Wddm {
bool shutdownStatus = false;
bool callBaseSetAllocationPriority = true;
bool callBaseWaitFromCpu = true;
bool failReserveGpuVirtualAddress = false;
};
} // namespace NEO

2
shared/test/unit_test/device/neo_device_tests.cpp
View File

@ -352,7 +352,7 @@ TEST_F(DeviceGetCapsTest, givenFlagEnabled64kbPagesWhenCallConstructorMemoryMana
};
uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override { return 0; };
double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override { return 0; }
AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override {
AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override {
return {};
}
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{};

29
shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp
View File

@ -234,7 +234,10 @@ TEST_F(DrmMemoryManagerTest, GivenGraphicsAllocationWhenAddAndRemoveAllocationTo
TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDrmMemoryManagerWhenGpuAddressIsReservedAndFreedThenAddressFromGfxPartitionIsUsed) {
auto memoryManager = std::make_unique<TestedDrmMemoryManager>(false, true, false, *executionEnvironment);
auto addressRange = memoryManager->reserveGpuAddress(MemoryConstants::pageSize, 0);
RootDeviceIndicesContainer rootDevices;
rootDevices.push_back(0);
uint32_t rootDeviceIndexReserved = 1;
auto addressRange = memoryManager->reserveGpuAddress(nullptr, MemoryConstants::pageSize, rootDevices, &rootDeviceIndexReserved);
auto gmmHelper = memoryManager->getGmmHelper(0);
EXPECT_LE(memoryManager->getGfxPartition(0)->getHeapBase(HeapIndex::HEAP_STANDARD), gmmHelper->decanonize(addressRange.address));
@ -242,6 +245,30 @@ TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDrmMemoryManagerWhenGp
memoryManager->freeGpuAddress(addressRange, 0);
}
TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDrmMemoryManagerWhenGpuAddressReservationIsAttemptedWithARequiredPtrThenNullRangeReturned) {
auto memoryManager = std::make_unique<TestedDrmMemoryManager>(false, true, false, *executionEnvironment);
void *requiredPtr = reinterpret_cast<void *>(0x1234);
RootDeviceIndicesContainer rootDevices;
rootDevices.push_back(0);
uint32_t rootDeviceIndexReserved = 1;
auto addressRange = memoryManager->reserveGpuAddress(requiredPtr, MemoryConstants::pageSize, rootDevices, &rootDeviceIndexReserved);
EXPECT_EQ(static_cast<int>(addressRange.address), 0);
EXPECT_EQ(static_cast<int>(addressRange.size), 0);
}
TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDrmMemoryManagerWhenGpuAddressReservationIsAttemptedWhichFailsThenNullRangeReturned) {
auto memoryManager = std::make_unique<TestedDrmMemoryManager>(false, true, false, *executionEnvironment);
RootDeviceIndicesContainer rootDevices;
rootDevices.push_back(0);
uint32_t rootDeviceIndexReserved = 1;
// emulate GPU address space exhaust
memoryManager->forceLimitedRangeAllocator(0xFFFFFFFFF);
memoryManager->getGfxPartition(0)->heapInit(HeapIndex::HEAP_STANDARD, 0x0, 0x10000);
size_t invalidSize = (size_t)memoryManager->getGfxPartition(0)->getHeapLimit(HeapIndex::HEAP_STANDARD) + MemoryConstants::pageSize;
auto addressRange = memoryManager->reserveGpuAddress(nullptr, invalidSize, rootDevices, &rootDeviceIndexReserved);
EXPECT_EQ(static_cast<int>(addressRange.address), 0);
}
TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenSmallSizeAndGpuAddressSetWhenGraphicsMemoryIsAllocatedThenAllocationWithSpecifiedGpuAddressInSystemMemoryIsCreated) {
auto memoryManager = std::make_unique<TestedDrmMemoryManager>(false, true, false, *executionEnvironment);
auto osContext = device->getDefaultEngine().osContext;

1
shared/test/unit_test/os_interface/windows/CMakeLists.txt
View File

@ -28,6 +28,7 @@ if(WIN32)
${CMAKE_CURRENT_SOURCE_DIR}/wddm_shared_allocations_test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/wddm_special_heap_test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/wddm_tests.cpp
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_reservation_tests.cpp
)
endif()

81
shared/test/unit_test/os_interface/windows/wddm_memory_reservation_tests.cpp Normal file
View File

@ -0,0 +1,81 @@
/*
* Copyright (C) 2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/execution_environment/root_device_environment.h"
#include "shared/source/gmm_helper/gmm_helper.h"
#include "shared/source/os_interface/device_factory.h"
#include "shared/source/os_interface/os_interface.h"
#include "shared/test/common/fixtures/device_fixture.h"
#include "shared/test/common/helpers/debug_manager_state_restore.h"
#include "shared/test/common/mocks/mock_wddm.h"
#include "shared/test/common/os_interface/windows/mock_wddm_memory_manager.h"
#include "shared/test/common/test_macros/hw_test.h"
namespace NEO {
struct MemoryReservationMock : public MockWddmMemoryManager {
using MemoryManager::freeGpuAddress;
using MemoryManager::reserveGpuAddress;
MemoryReservationMock(NEO::ExecutionEnvironment &executionEnvironment) : MockWddmMemoryManager(executionEnvironment) {}
};
class WddmMemManagerFixture {
public:
void setUp() {
executionEnvironment = std::make_unique<ExecutionEnvironment>();
executionEnvironment->prepareRootDeviceEnvironments(1);
executionEnvironment->rootDeviceEnvironments[0]->setHwInfo(defaultHwInfo.get());
executionEnvironment->rootDeviceEnvironments[0]->initGmm();
DeviceFactory::prepareDeviceEnvironments(*executionEnvironment);
auto wddm = static_cast<WddmMock *>(executionEnvironment->rootDeviceEnvironments[0]->osInterface->getDriverModel()->as<Wddm>());
wddm->callBaseMapGpuVa = false;
memManager = std::unique_ptr<MemoryReservationMock>(new MemoryReservationMock(*executionEnvironment));
}
void tearDown() {
}
std::unique_ptr<MemoryReservationMock> memManager;
std::unique_ptr<ExecutionEnvironment> executionEnvironment;
};
typedef ::Test<WddmMemManagerFixture> WddmMemoryReservationTests;
TEST_F(WddmMemoryReservationTests, givenWddmMemoryManagerWhenGpuAddressIsReservedAndFreedThenAddressFromGfxPartitionIsUsed) {
RootDeviceIndicesContainer rootDevices;
rootDevices.push_back(0);
uint32_t rootDeviceIndexReserved = 1;
auto addressRange = memManager->reserveGpuAddress(nullptr, MemoryConstants::pageSize64k, rootDevices, &rootDeviceIndexReserved);
auto gmmHelper = memManager->getGmmHelper(0);
EXPECT_LE(memManager->getGfxPartition(0)->getHeapBase(HeapIndex::HEAP_STANDARD64KB), gmmHelper->decanonize(addressRange.address));
EXPECT_GT(memManager->getGfxPartition(0)->getHeapLimit(HeapIndex::HEAP_STANDARD64KB), gmmHelper->decanonize(addressRange.address));
memManager->freeGpuAddress(addressRange, 0);
}
TEST(WddmMemoryReservationFailTest, givenWddmMemoryManagerWhenGpuAddressReservationIsAttemptedThenNullPtrAndSizeReturned) {
std::unique_ptr<MemoryReservationMock> memManager;
std::unique_ptr<ExecutionEnvironment> executionEnvironment;
executionEnvironment = std::make_unique<ExecutionEnvironment>();
executionEnvironment->prepareRootDeviceEnvironments(1);
executionEnvironment->rootDeviceEnvironments[0]->setHwInfo(defaultHwInfo.get());
executionEnvironment->rootDeviceEnvironments[0]->initGmm();
DeviceFactory::prepareDeviceEnvironments(*executionEnvironment);
auto wddm = static_cast<WddmMock *>(executionEnvironment->rootDeviceEnvironments[0]->osInterface->getDriverModel()->as<Wddm>());
wddm->callBaseMapGpuVa = false;
wddm->failReserveGpuVirtualAddress = true;
memManager = std::unique_ptr<MemoryReservationMock>(new MemoryReservationMock(*executionEnvironment));
RootDeviceIndicesContainer rootDevices;
rootDevices.push_back(0);
uint32_t rootDeviceIndexReserved = 1;
auto addressRange = memManager->reserveGpuAddress(nullptr, MemoryConstants::pageSize64k, rootDevices, &rootDeviceIndexReserved);
EXPECT_EQ(addressRange.address, 0ull);
EXPECT_EQ(addressRange.size, 0u);
}
} // namespace NEO