From d81b0b14a1ab7a1644b34d452632e330db4bf89c Mon Sep 17 00:00:00 2001 From: "Spruit, Neil R" Date: Sat, 17 Sep 2022 00:38:06 +0000 Subject: [PATCH] L0 Virtual Memory Reservation support - Enable support for L0 Virtual Memory reservation on Linux and Windows. - Excludes support for Linux to allow pStart option Related-To: LOCI-3397, LOCI-1543 Signed-off-by: Spruit, Neil R --- .../core/source/context/context_imp.cpp | 93 +++++- level_zero/core/source/context/context_imp.h | 1 + .../unit_tests/fixtures/memory_ipc_fixture.h | 4 +- .../sources/context/test_context.cpp | 316 +++++++++++++++++- .../unit_tests/sources/event/test_event.cpp | 2 +- .../memory/linux/test_memory_linux.cpp | 4 +- .../memory_manager/memory_manager_tests.cpp | 32 +- .../os_interface/windows/wddm20_tests.cpp | 2 +- .../windows/wddm_memory_manager_tests.cpp | 13 +- shared/source/debugger/debugger_l0.cpp | 5 +- shared/source/memory_manager/memory_manager.h | 13 +- .../os_agnostic_memory_manager.cpp | 19 +- .../os_agnostic_memory_manager.h | 2 +- .../os_interface/linux/drm_memory_manager.cpp | 15 +- .../os_interface/linux/drm_memory_manager.h | 2 +- .../source/os_interface/windows/wddm/wddm.cpp | 4 +- .../source/os_interface/windows/wddm/wddm.h | 2 +- .../windows/wddm_memory_manager.cpp | 22 +- .../windows/wddm_memory_manager.h | 4 +- shared/test/common/mocks/mock_wddm.cpp | 8 +- shared/test/common/mocks/mock_wddm.h | 3 +- .../unit_test/device/neo_device_tests.cpp | 2 +- .../linux/drm_memory_manager_tests.cpp | 29 +- .../os_interface/windows/CMakeLists.txt | 1 + .../windows/wddm_memory_reservation_tests.cpp | 81 +++++ 25 files changed, 627 insertions(+), 52 deletions(-) create mode 100644 shared/test/unit_test/os_interface/windows/wddm_memory_reservation_tests.cpp diff --git a/level_zero/core/source/context/context_imp.cpp b/level_zero/core/source/context/context_imp.cpp index 8faa6a68ee..014c535cba 100644 --- a/level_zero/core/source/context/context_imp.cpp +++ b/level_zero/core/source/context/context_imp.cpp @@ -766,18 +766,58 @@ ze_result_t ContextImp::activateMetricGroups(zet_device_handle_t hDevice, ze_result_t ContextImp::reserveVirtualMem(const void *pStart, size_t size, void **pptr) { - return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; + if ((getPageSizeRequired(size) != size)) { + return ZE_RESULT_ERROR_UNSUPPORTED_SIZE; + } + NEO::VirtualMemoryReservation *virtualMemoryReservation = new NEO::VirtualMemoryReservation; + virtualMemoryReservation->virtualAddressRange = this->driverHandle->getMemoryManager()->reserveGpuAddress(pStart, size, this->driverHandle->rootDeviceIndices, &virtualMemoryReservation->rootDeviceIndex); + if (pStart != 0x0) { + if (virtualMemoryReservation->virtualAddressRange.address != reinterpret_cast(pStart)) { + delete virtualMemoryReservation; + return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY; + } + } + if (virtualMemoryReservation->virtualAddressRange.address == 0) { + delete virtualMemoryReservation; + return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY; + } + virtualMemoryReservation->flags.readWrite = false; + virtualMemoryReservation->flags.readOnly = false; + virtualMemoryReservation->flags.noAccess = true; + virtualMemoryReservation->mapped = false; + auto lock = this->driverHandle->getMemoryManager()->lockVirtualMemoryReservationMap(); + this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().insert(std::pair(reinterpret_cast(virtualMemoryReservation->virtualAddressRange.address), virtualMemoryReservation)); + *pptr = reinterpret_cast(virtualMemoryReservation->virtualAddressRange.address); + return ZE_RESULT_SUCCESS; } ze_result_t ContextImp::freeVirtualMem(const void *ptr, size_t size) { - return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; + std::map::iterator it; + auto lock = this->driverHandle->getMemoryManager()->lockVirtualMemoryReservationMap(); + it = this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().find(const_cast(ptr)); + if (it != this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().end()) { + NEO::VirtualMemoryReservation *virtualMemoryReservation = it->second; + if (virtualMemoryReservation->virtualAddressRange.size != size) { + return ZE_RESULT_ERROR_INVALID_ARGUMENT; + } + this->driverHandle->getMemoryManager()->freeGpuAddress(virtualMemoryReservation->virtualAddressRange, virtualMemoryReservation->rootDeviceIndex); + delete virtualMemoryReservation; + this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().erase(it); + return ZE_RESULT_SUCCESS; + } else { + return ZE_RESULT_ERROR_INVALID_ARGUMENT; + } +} + +size_t ContextImp::getPageSizeRequired(size_t size) { + return std::max(Math::prevPowerOfTwo(size), MemoryConstants::pageSize64k); } ze_result_t ContextImp::queryVirtualMemPageSize(ze_device_handle_t hDevice, size_t size, size_t *pagesize) { - *pagesize = std::max(Math::prevPowerOfTwo(size), MemoryConstants::pageSize64k); + *pagesize = getPageSizeRequired(size); return ZE_RESULT_SUCCESS; } @@ -807,14 +847,57 @@ ze_result_t ContextImp::unMapVirtualMem(const void *ptr, ze_result_t ContextImp::setVirtualMemAccessAttribute(const void *ptr, size_t size, ze_memory_access_attribute_t access) { - return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; + std::map::iterator it; + auto lock = this->driverHandle->getMemoryManager()->lockVirtualMemoryReservationMap(); + it = this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().find(const_cast(ptr)); + if (it != this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().end()) { + NEO::VirtualMemoryReservation *virtualMemoryReservation = this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().at(const_cast(ptr)); + switch (access) { + case ZE_MEMORY_ACCESS_ATTRIBUTE_NONE: + virtualMemoryReservation->flags.readOnly = false; + virtualMemoryReservation->flags.noAccess = true; + virtualMemoryReservation->flags.readWrite = false; + break; + case ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE: + virtualMemoryReservation->flags.readOnly = false; + virtualMemoryReservation->flags.noAccess = false; + virtualMemoryReservation->flags.readWrite = true; + break; + case ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY: + virtualMemoryReservation->flags.readWrite = false; + virtualMemoryReservation->flags.noAccess = false; + virtualMemoryReservation->flags.readOnly = true; + break; + default: + return ZE_RESULT_ERROR_INVALID_ENUMERATION; + } + return ZE_RESULT_SUCCESS; + } else { + return ZE_RESULT_ERROR_INVALID_ARGUMENT; + } } ze_result_t ContextImp::getVirtualMemAccessAttribute(const void *ptr, size_t size, ze_memory_access_attribute_t *access, size_t *outSize) { - return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; + std::map::iterator it; + auto lock = this->driverHandle->getMemoryManager()->lockVirtualMemoryReservationMap(); + it = this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().find(const_cast(ptr)); + if (it != this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().end()) { + NEO::VirtualMemoryReservation *virtualMemoryReservation = this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().at(const_cast(ptr)); + if (virtualMemoryReservation->flags.readWrite) { + *access = ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE; + } else if (virtualMemoryReservation->flags.readOnly) { + *access = ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY; + } else { + *access = ZE_MEMORY_ACCESS_ATTRIBUTE_NONE; + } + *outSize = virtualMemoryReservation->virtualAddressRange.size; + return ZE_RESULT_SUCCESS; + } else { + return ZE_RESULT_ERROR_INVALID_ARGUMENT; + } } ze_result_t ContextImp::createEventPool(const ze_event_pool_desc_t *desc, diff --git a/level_zero/core/source/context/context_imp.h b/level_zero/core/source/context/context_imp.h index 4f4c0da00d..276b8fdb21 100644 --- a/level_zero/core/source/context/context_imp.h +++ b/level_zero/core/source/context/context_imp.h @@ -162,6 +162,7 @@ struct ContextImp : Context { protected: bool isAllocationSuitableForCompression(const StructuresLookupTable &structuresLookupTable, Device &device, size_t allocSize); + size_t getPageSizeRequired(size_t size); std::map devices; std::vector deviceHandles; diff --git a/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h b/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h index 113181888e..d1c542ebbb 100644 --- a/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h +++ b/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h @@ -401,7 +401,7 @@ class MemoryManagerIpcMock : public NEO::MemoryManager { uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override { return 0; }; uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override { return 0; }; double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override { return 0; } - AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override { + AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override { return {}; } void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{}; @@ -608,7 +608,7 @@ class MemoryManagerIpcImplicitScalingMock : public NEO::MemoryManager { uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override { return 0; }; uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override { return 0; }; double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override { return 0; } - AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override { + AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override { return {}; } void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{}; diff --git a/level_zero/core/test/unit_tests/sources/context/test_context.cpp b/level_zero/core/test/unit_tests/sources/context/test_context.cpp index a19819bbbb..a300645713 100644 --- a/level_zero/core/test/unit_tests/sources/context/test_context.cpp +++ b/level_zero/core/test/unit_tests/sources/context/test_context.cpp @@ -788,7 +788,7 @@ TEST_F(ContextTest, whenGettingDriverThenDriverIsRetrievedSuccessfully) { EXPECT_EQ(ZE_RESULT_SUCCESS, res); } -TEST_F(ContextTest, whenCallingVirtualMemInterfacesThenUnsupportedIsReturned) { +TEST_F(ContextTest, whenCallingVirtualMemInterfacesThenSuccessIsReturned) { ze_context_handle_t hContext; ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; @@ -798,13 +798,17 @@ TEST_F(ContextTest, whenCallingVirtualMemInterfacesThenUnsupportedIsReturned) { ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); void *pStart = 0x0; - size_t size = 0u; + size_t size = 4096u; void *ptr = nullptr; - res = contextImp->reserveVirtualMem(pStart, size, &ptr); - EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res); + size_t pagesize = 0u; + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0); - res = contextImp->freeVirtualMem(ptr, size); - EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res); + res = contextImp->freeVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = contextImp->destroy(); EXPECT_EQ(ZE_RESULT_SUCCESS, res); @@ -871,6 +875,16 @@ TEST_F(ContextTest, whenCallingMappingVirtualInterfacesThenUnsupportedIsReturned ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + void *pStart = 0x0; + size_t size = 4096u; + void *ptr = nullptr; + size_t pagesize = 0u; + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0); + ze_physical_mem_desc_t descMem = {}; ze_physical_mem_handle_t mem = {}; res = contextImp->createPhysicalMem(device, &descMem, &mem); @@ -878,25 +892,299 @@ TEST_F(ContextTest, whenCallingMappingVirtualInterfacesThenUnsupportedIsReturned ze_memory_access_attribute_t access = {}; size_t offset = 0; - void *ptr = nullptr; - size_t size = 0; - res = contextImp->mapVirtualMem(ptr, size, mem, offset, access); + res = contextImp->mapVirtualMem(ptr, pagesize, mem, offset, access); EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res); - res = contextImp->setVirtualMemAccessAttribute(ptr, size, access); - EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res); + res = contextImp->setVirtualMemAccessAttribute(ptr, pagesize, access); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); ze_memory_access_attribute_t outAccess = {}; size_t outSize = 0; - res = contextImp->getVirtualMemAccessAttribute(ptr, size, &outAccess, &outSize); - EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res); + res = contextImp->getVirtualMemAccessAttribute(ptr, pagesize, &outAccess, &outSize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_EQ(pagesize, outSize); - res = contextImp->unMapVirtualMem(ptr, size); + res = contextImp->unMapVirtualMem(ptr, pagesize); EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res); res = contextImp->destroyPhysicalMem(mem); EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res); + res = contextImp->freeVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + +TEST_F(ContextTest, whenCallingVirtualMemorySetAttributeWithValidEnumerationsThenSuccessisReturned) { + ze_context_handle_t hContext; + ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + + ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + + void *pStart = 0x0; + size_t size = 4096; + void *ptr = nullptr; + size_t pagesize = 0u; + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0); + + std::vector memoryAccessFlags = { + ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE, ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY, + ZE_MEMORY_ACCESS_ATTRIBUTE_NONE}; + + ze_memory_access_attribute_t access = {ZE_MEMORY_ACCESS_ATTRIBUTE_NONE}; + ze_memory_access_attribute_t previousAccess = {}; + size_t outSize = 0; + for (auto accessFlags : memoryAccessFlags) { + previousAccess = access; + res = contextImp->setVirtualMemAccessAttribute(ptr, pagesize, accessFlags); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + res = contextImp->getVirtualMemAccessAttribute(ptr, pagesize, &access, &outSize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_EQ(outSize, pagesize); + EXPECT_NE(previousAccess, access); + EXPECT_EQ(accessFlags, access); + } + + res = contextImp->getVirtualMemAccessAttribute(ptr, pagesize, &access, &outSize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->freeVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + +TEST_F(ContextTest, whenCallingVirtualMemorySetAttributeWithInvalidValuesThenFailureReturned) { + ze_context_handle_t hContext; + ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + + ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + + void *pStart = 0x0; + size_t size = 4096u; + void *ptr = nullptr; + size_t pagesize = 0u; + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0); + + ze_memory_access_attribute_t access = {ZE_MEMORY_ACCESS_ATTRIBUTE_FORCE_UINT32}; + res = contextImp->setVirtualMemAccessAttribute(ptr, pagesize, access); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ENUMERATION, res); + + res = contextImp->setVirtualMemAccessAttribute(pStart, pagesize, access); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res); + + res = contextImp->freeVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + +TEST_F(ContextTest, whenCallingVirtualMemoryGetAttributeWithInvalidValuesThenFailureReturned) { + ze_context_handle_t hContext; + ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + + ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + + void *pStart = 0x0; + size_t size = 4096u; + void *ptr = nullptr; + size_t pagesize = 0u; + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0); + + size_t outSize = 0; + ze_memory_access_attribute_t access = {ZE_MEMORY_ACCESS_ATTRIBUTE_NONE}; + res = contextImp->getVirtualMemAccessAttribute(pStart, pagesize, &access, &outSize); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res); + + res = contextImp->freeVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + +TEST_F(ContextTest, whenCallingVirtualMemoryFreeWithInvalidValuesThenFailuresReturned) { + ze_context_handle_t hContext; + ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + + ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + + void *pStart = 0x0; + size_t size = 4096u; + size_t invalidSize = 10u; + void *ptr = nullptr; + size_t pagesize = 0u; + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0); + + res = contextImp->freeVirtualMem(ptr, invalidSize); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res); + + res = contextImp->freeVirtualMem(pStart, pagesize); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res); + + res = contextImp->freeVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + +TEST_F(ContextTest, whenCallingVirtualMemoryReservationWithInvalidArgumentsThenFailureReturned) { + ze_context_handle_t hContext; + ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + + ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + + void *pStart = 0x0; + size_t size = 64u; + void *ptr = nullptr; + res = contextImp->reserveVirtualMem(pStart, size, &ptr); + EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_SIZE, res); + size_t pagesize = 0u; + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + pStart = reinterpret_cast(0x1234); + res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); + EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY, res); + + res = contextImp->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + +class ReserveMemoryManagerMock : public NEO::MemoryManager { + public: + ReserveMemoryManagerMock(NEO::ExecutionEnvironment &executionEnvironment) : NEO::MemoryManager(executionEnvironment) {} + NEO::GraphicsAllocation *createGraphicsAllocationFromMultipleSharedHandles(const std::vector &handles, AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override { return nullptr; } + NEO::GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override { return nullptr; } + void addAllocationToHostPtrManager(NEO::GraphicsAllocation *memory) override{}; + void removeAllocationFromHostPtrManager(NEO::GraphicsAllocation *memory) override{}; + NEO::GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex, AllocationType allocType) override { return nullptr; }; + AllocationStatus populateOsHandles(NEO::OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override { return AllocationStatus::Success; }; + void cleanOsHandles(NEO::OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override{}; + void freeGraphicsMemoryImpl(NEO::GraphicsAllocation *gfxAllocation) override{}; + void freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation, bool isImportedAllocation) override{}; + uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override { return 0; }; + uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override { return 0; }; + double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override { return 0; } + AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override { + if (failReserveGpuAddress) { + return {}; + } + return AddressRange{reinterpret_cast(requiredStartAddress), size}; + } + void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{}; + NEO::GraphicsAllocation *createGraphicsAllocation(OsHandleStorage &handleStorage, const NEO::AllocationData &allocationData) override { return nullptr; }; + NEO::GraphicsAllocation *allocateGraphicsMemoryForNonSvmHostPtr(const NEO::AllocationData &allocationData) override { return nullptr; }; + NEO::GraphicsAllocation *allocateGraphicsMemoryWithAlignment(const NEO::AllocationData &allocationData) override { return nullptr; }; + NEO::GraphicsAllocation *allocateUSMHostGraphicsMemory(const NEO::AllocationData &allocationData) override { return nullptr; }; + NEO::GraphicsAllocation *allocateGraphicsMemory64kb(const NEO::AllocationData &allocationData) override { return nullptr; }; + NEO::GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const NEO::AllocationData &allocationData, bool useLocalMemory) override { return nullptr; }; + NEO::GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const NEO::AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + NEO::GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const NEO::AllocationData &allocationData) override { return nullptr; }; + + NEO::GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const NEO::AllocationData &allocationData, std::unique_ptr gmm) override { return nullptr; }; + NEO::GraphicsAllocation *allocateMemoryByKMD(const NEO::AllocationData &allocationData) override { return nullptr; }; + void *lockResourceImpl(NEO::GraphicsAllocation &graphicsAllocation) override { return nullptr; }; + void unlockResourceImpl(NEO::GraphicsAllocation &graphicsAllocation) override{}; + + bool failReserveGpuAddress = true; +}; + +TEST_F(ContextTest, whenCallingVirtualMemReserveWithPStartWithSuccessfulAllocationThenSuccessReturned) { + ze_context_handle_t hContext; + ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + + ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + + void *pStart = reinterpret_cast(0x1234); + size_t size = 4096u; + void *ptr = nullptr; + size_t pagesize = 0u; + + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + ReserveMemoryManagerMock *ReserveMemoryManager = new ReserveMemoryManagerMock(*neoDevice->executionEnvironment); + auto memoryManager = driverHandle->getMemoryManager(); + ReserveMemoryManager->failReserveGpuAddress = false; + NEO::MemoryManager *mockingMemoryManager = (NEO::MemoryManager *)ReserveMemoryManager; + driverHandle->setMemoryManager(mockingMemoryManager); + res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_GT((int)mockingMemoryManager->getVirtualMemoryReservationMap().size(), 0); + res = contextImp->freeVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + driverHandle->setMemoryManager(memoryManager); + delete ReserveMemoryManager; + + res = contextImp->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + +TEST_F(ContextTest, whenCallingVirtualMemoryReservationWhenOutOfMemoryThenOutOfMemoryReturned) { + ze_context_handle_t hContext; + ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + + ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + + void *pStart = 0x0; + size_t size = 0u; + void *ptr = nullptr; + size_t pagesize = 0u; + + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + NEO::MemoryManager *failingReserveMemoryManager = new ReserveMemoryManagerMock(*neoDevice->executionEnvironment); + auto memoryManager = driverHandle->getMemoryManager(); + driverHandle->setMemoryManager(failingReserveMemoryManager); + res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); + EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY, res); + pStart = reinterpret_cast(0x1234); + res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); + EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY, res); + driverHandle->setMemoryManager(memoryManager); + delete failingReserveMemoryManager; + res = contextImp->destroy(); EXPECT_EQ(ZE_RESULT_SUCCESS, res); } diff --git a/level_zero/core/test/unit_tests/sources/event/test_event.cpp b/level_zero/core/test/unit_tests/sources/event/test_event.cpp index 26b3eccb65..0a289daefe 100644 --- a/level_zero/core/test/unit_tests/sources/event/test_event.cpp +++ b/level_zero/core/test/unit_tests/sources/event/test_event.cpp @@ -63,7 +63,7 @@ class MemoryManagerEventPoolFailMock : public NEO::MemoryManager { uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override { return 0; }; uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override { return 0; }; double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override { return 0; } - AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override { + AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override { return {}; } void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{}; diff --git a/level_zero/core/test/unit_tests/sources/memory/linux/test_memory_linux.cpp b/level_zero/core/test/unit_tests/sources/memory/linux/test_memory_linux.cpp index 1f6e7fa6dd..8f5289a345 100644 --- a/level_zero/core/test/unit_tests/sources/memory/linux/test_memory_linux.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/linux/test_memory_linux.cpp @@ -65,7 +65,7 @@ class MemoryManagerIpcImplicitScalingObtainFdMock : public NEO::DrmMemoryManager uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override { return 0; }; uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override { return 0; }; double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override { return 0; } - AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override { + AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override { return {}; } void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{}; @@ -497,7 +497,7 @@ class MemoryManagerIpcObtainFdMock : public NEO::DrmMemoryManager { uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override { return 0; }; uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override { return 0; }; double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override { return 0; } - AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override { + AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override { return {}; } void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{}; diff --git a/opencl/test/unit_test/memory_manager/memory_manager_tests.cpp b/opencl/test/unit_test/memory_manager/memory_manager_tests.cpp index 0e4e4fc315..fdacdd3562 100644 --- a/opencl/test/unit_test/memory_manager/memory_manager_tests.cpp +++ b/opencl/test/unit_test/memory_manager/memory_manager_tests.cpp @@ -1628,15 +1628,43 @@ TEST(OsAgnosticMemoryManager, givenOsAgnosticMemoryManagerAndFreeMemoryDisabledW TEST(OsAgnosticMemoryManager, givenOsAgnosticMemoryManagerWhenGpuAddressIsReservedAndFreedThenAddressFromGfxPartitionIsUsed) { MockExecutionEnvironment executionEnvironment; OsAgnosticMemoryManager memoryManager(executionEnvironment); - - auto addressRange = memoryManager.reserveGpuAddress(MemoryConstants::pageSize, 0); + RootDeviceIndicesContainer rootDevices; + rootDevices.push_back(0); + uint32_t rootDeviceIndexReserved = 10; + auto addressRange = memoryManager.reserveGpuAddress(nullptr, MemoryConstants::pageSize, rootDevices, &rootDeviceIndexReserved); auto gmmHelper = memoryManager.getGmmHelper(0); + EXPECT_EQ(0u, rootDeviceIndexReserved); EXPECT_LE(memoryManager.getGfxPartition(0)->getHeapBase(HeapIndex::HEAP_STANDARD), gmmHelper->decanonize(addressRange.address)); EXPECT_GT(memoryManager.getGfxPartition(0)->getHeapLimit(HeapIndex::HEAP_STANDARD), gmmHelper->decanonize(addressRange.address)); memoryManager.freeGpuAddress(addressRange, 0); } +TEST(OsAgnosticMemoryManager, givenOsAgnosticMemoryManagerWhenGpuAddressReservationIsAttemptedWihtInvalidSizeThenFailureReturnsNullAddressRange) { + MockExecutionEnvironment executionEnvironment; + OsAgnosticMemoryManager memoryManager(executionEnvironment); + RootDeviceIndicesContainer rootDevices; + rootDevices.push_back(0); + uint32_t rootDeviceIndexReserved = 10; + // emulate GPU address space exhaust + memoryManager.getGfxPartition(0)->heapInit(HeapIndex::HEAP_STANDARD, 0x0, 0x10000); + auto addressRange = memoryManager.reserveGpuAddress(nullptr, (size_t)(memoryManager.getGfxPartition(0)->getHeapLimit(HeapIndex::HEAP_STANDARD) * 2), rootDevices, &rootDeviceIndexReserved); + EXPECT_EQ(static_cast(addressRange.address), 0); +} + +TEST(OsAgnosticMemoryManager, givenOsAgnosticMemoryManagerWhenGpuAddressReservationIsAttemptedWithARequiredPtrThenNullRangeReturned) { + MockExecutionEnvironment executionEnvironment; + OsAgnosticMemoryManager memoryManager(executionEnvironment); + RootDeviceIndicesContainer rootDevices; + rootDevices.push_back(0); + uint32_t rootDeviceIndexReserved = 10; + void *requiredPtr = reinterpret_cast(0x1234); + auto addressRange = memoryManager.reserveGpuAddress(requiredPtr, MemoryConstants::pageSize, rootDevices, &rootDeviceIndexReserved); + EXPECT_EQ(0u, rootDeviceIndexReserved); + EXPECT_EQ(static_cast(addressRange.address), 0); + EXPECT_EQ(static_cast(addressRange.size), 0); +} + TEST(OsAgnosticMemoryManager, givenOsAgnosticMemoryManagerWhenCheckedForIndirectAllocationsAsPackSupportThenFalseIsReturned) { MockExecutionEnvironment executionEnvironment; OsAgnosticMemoryManager memoryManager(executionEnvironment); diff --git a/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp index 32ec5f14c2..8d0dfc882c 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp @@ -283,7 +283,7 @@ TEST_F(Wddm20WithMockGdiDllTests, givenAllocationSmallerUnderlyingThanAlignedSiz TEST_F(Wddm20WithMockGdiDllTests, givenReserveCallWhenItIsCalledWithProperParamtersThenAddressInRangeIsReturend) { auto sizeAlignedTo64Kb = 64 * KB; - auto reservationAddress = wddm->reserveGpuVirtualAddress(wddm->getGfxPartition().Heap32[0].Base, + auto reservationAddress = wddm->reserveGpuVirtualAddress(0ull, wddm->getGfxPartition().Heap32[0].Base, wddm->getGfxPartition().Heap32[0].Limit, sizeAlignedTo64Kb); diff --git a/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp index a93a4f4df5..b0f3804775 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp @@ -564,12 +564,15 @@ TEST_F(WddmMemoryManagerSimpleTest, givenNonZeroFenceValueOnSomeOfMultipleEngine memoryManager->freeGraphicsMemory(allocation); } -TEST_F(WddmMemoryManagerSimpleTest, givenWddmMemoryManagerWhenGpuAddressIsReservedAndFreedThenAddressRangeIsZero) { - auto addressRange = memoryManager->reserveGpuAddress(MemoryConstants::pageSize, 0); +TEST_F(WddmMemoryManagerSimpleTest, givenWddmMemoryManagerWhenGpuAddressIsReservedAndFreedThenAddressRangeIsNonZero) { + RootDeviceIndicesContainer rootDevices; + rootDevices.push_back(0); + uint32_t rootDeviceIndexReserved = 1; + auto addressRange = memoryManager->reserveGpuAddress(nullptr, MemoryConstants::pageSize64k, rootDevices, &rootDeviceIndexReserved); auto gmmHelper = memoryManager->getGmmHelper(0); - - EXPECT_EQ(0u, gmmHelper->decanonize(addressRange.address)); - EXPECT_EQ(0u, addressRange.size); + EXPECT_EQ(0u, rootDeviceIndexReserved); + EXPECT_NE(0u, gmmHelper->decanonize(addressRange.address)); + EXPECT_EQ(MemoryConstants::pageSize64k, addressRange.size); memoryManager->freeGpuAddress(addressRange, 0); } diff --git a/shared/source/debugger/debugger_l0.cpp b/shared/source/debugger/debugger_l0.cpp index 484b2655fa..dcab8866b4 100644 --- a/shared/source/debugger/debugger_l0.cpp +++ b/shared/source/debugger/debugger_l0.cpp @@ -51,7 +51,10 @@ void DebuggerL0::initialize() { device->getDeviceBitfield()}; if (!singleAddressSpaceSbaTracking) { - sbaTrackingGpuVa = device->getMemoryManager()->reserveGpuAddress(MemoryConstants::pageSize, device->getRootDeviceIndex()); + RootDeviceIndicesContainer rootDevices; + rootDevices.push_back(device->getRootDeviceIndex()); + uint32_t rootDeviceIndexReserved = 0; + sbaTrackingGpuVa = device->getMemoryManager()->reserveGpuAddress(nullptr, MemoryConstants::pageSize, rootDevices, &rootDeviceIndexReserved); properties.gpuAddress = sbaTrackingGpuVa.address; } diff --git a/shared/source/memory_manager/memory_manager.h b/shared/source/memory_manager/memory_manager.h index c2bf23e7f3..4f6930513a 100644 --- a/shared/source/memory_manager/memory_manager.h +++ b/shared/source/memory_manager/memory_manager.h @@ -52,6 +52,13 @@ struct AddressRange { size_t size; }; +struct VirtualMemoryReservation { + AddressRange virtualAddressRange; + MemoryFlags flags; + bool mapped; + uint32_t rootDeviceIndex; +}; + constexpr size_t paddingBufferSize = 2 * MemoryConstants::megaByte; namespace MemoryTransferHelper { @@ -211,7 +218,7 @@ class MemoryManager { GmmHelper *getGmmHelper(uint32_t rootDeviceIndex) { return executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->getGmmHelper(); } - virtual AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) = 0; + virtual AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) = 0; virtual void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) = 0; static HeapIndex selectInternalHeap(bool useLocalMemory) { return useLocalMemory ? HeapIndex::HEAP_INTERNAL_DEVICE_MEMORY : HeapIndex::HEAP_INTERNAL; } static HeapIndex selectExternalHeap(bool useLocalMemory) { return useLocalMemory ? HeapIndex::HEAP_EXTERNAL_DEVICE_MEMORY : HeapIndex::HEAP_EXTERNAL; } @@ -264,6 +271,8 @@ class MemoryManager { std::unordered_map &getKernelAllocationMap() { return this->kernelAllocationMap; }; [[nodiscard]] std::unique_lock lockKernelAllocationMap() { return std::unique_lock(this->kernelAllocationMutex); }; + std::map &getVirtualMemoryReservationMap() { return this->virtualMemoryReservationMap; }; + [[nodiscard]] std::unique_lock lockVirtualMemoryReservationMap() { return std::unique_lock(this->virtualMemoryReservationMapMutex); }; protected: bool getAllocationData(AllocationData &allocationData, const AllocationProperties &properties, const void *hostPtr, const StorageInfo &storageInfo); @@ -329,6 +338,8 @@ class MemoryManager { std::vector isaInLocalMemory; std::unordered_map kernelAllocationMap; std::mutex kernelAllocationMutex; + std::map virtualMemoryReservationMap; + std::mutex virtualMemoryReservationMapMutex; }; std::unique_ptr createDeferredDeleter(); diff --git a/shared/source/memory_manager/os_agnostic_memory_manager.cpp b/shared/source/memory_manager/os_agnostic_memory_manager.cpp index 925bbfe277..f3b5ed0f99 100644 --- a/shared/source/memory_manager/os_agnostic_memory_manager.cpp +++ b/shared/source/memory_manager/os_agnostic_memory_manager.cpp @@ -453,10 +453,21 @@ MemoryAllocation *OsAgnosticMemoryManager::createMemoryAllocation(AllocationType return memoryAllocation; } -AddressRange OsAgnosticMemoryManager::reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) { - auto gfxPartition = getGfxPartition(rootDeviceIndex); - auto gmmHelper = getGmmHelper(rootDeviceIndex); - auto gpuVa = gmmHelper->canonize(gfxPartition->heapAllocate(HeapIndex::HEAP_STANDARD, size)); +AddressRange OsAgnosticMemoryManager::reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) { + uint64_t gpuVa = 0u; + *reservedOnRootDeviceIndex = 0; + if (requiredStartAddress) { + return AddressRange{0, 0}; + } + for (uint32_t rootDeviceIndex = 0; rootDeviceIndex < rootDeviceIndices.size(); rootDeviceIndex++) { + auto gfxPartition = getGfxPartition(rootDeviceIndex); + auto gmmHelper = getGmmHelper(rootDeviceIndex); + gpuVa = gmmHelper->canonize(gfxPartition->heapAllocate(HeapIndex::HEAP_STANDARD, size)); + if (gpuVa != 0u) { + *reservedOnRootDeviceIndex = rootDeviceIndex; + break; + } + } return AddressRange{gpuVa, size}; } diff --git a/shared/source/memory_manager/os_agnostic_memory_manager.h b/shared/source/memory_manager/os_agnostic_memory_manager.h index 1ef7712bf2..cfaafaf134 100644 --- a/shared/source/memory_manager/os_agnostic_memory_manager.h +++ b/shared/source/memory_manager/os_agnostic_memory_manager.h @@ -90,7 +90,7 @@ class OsAgnosticMemoryManager : public MemoryManager { void *reserveCpuAddressRange(size_t size, uint32_t rootDeviceIndex) override; void releaseReservedCpuAddressRange(void *reserved, size_t size, uint32_t rootDeviceIndex) override; - AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override; + AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override; void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override; bool is64kbPagesEnabled(const HardwareInfo *hwInfo); diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index d4a2287675..15e0b95fd9 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -1156,8 +1156,19 @@ uint32_t DrmMemoryManager::getRootDeviceIndex(const Drm *drm) { return CommonConstants::unspecifiedDeviceIndex; } -AddressRange DrmMemoryManager::reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) { - auto gpuVa = acquireGpuRange(size, rootDeviceIndex, HeapIndex::HEAP_STANDARD); +AddressRange DrmMemoryManager::reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) { + uint64_t gpuVa = 0u; + *reservedOnRootDeviceIndex = 0; + if (requiredStartAddress) { + return AddressRange{0, 0}; + } + for (uint32_t rootDeviceIndex = 0; rootDeviceIndex < rootDeviceIndices.size(); rootDeviceIndex++) { + gpuVa = acquireGpuRange(size, rootDeviceIndex, HeapIndex::HEAP_STANDARD); + if (gpuVa != 0u) { + *reservedOnRootDeviceIndex = rootDeviceIndex; + break; + } + } return AddressRange{gpuVa, size}; } diff --git a/shared/source/os_interface/linux/drm_memory_manager.h b/shared/source/os_interface/linux/drm_memory_manager.h index 19108dbd6d..0076aedc4c 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.h +++ b/shared/source/os_interface/linux/drm_memory_manager.h @@ -62,7 +62,7 @@ class DrmMemoryManager : public MemoryManager { bool copyMemoryToAllocationBanks(GraphicsAllocation *graphicsAllocation, size_t destinationOffset, const void *memoryToCopy, size_t sizeToCopy, DeviceBitfield handleMask) override; MOCKABLE_VIRTUAL int obtainFdFromHandle(int boHandle, uint32_t rootDeviceindex); - AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override; + AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override; void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override; MOCKABLE_VIRTUAL BufferObject *createBufferObjectInMemoryRegion(Drm *drm, Gmm *gmm, AllocationType allocationType, uint64_t gpuAddress, size_t size, uint32_t memoryBanks, size_t maxOsContextCount, int32_t pairHandle); diff --git a/shared/source/os_interface/windows/wddm/wddm.cpp b/shared/source/os_interface/windows/wddm/wddm.cpp index d74ad7defb..87bad1e9f0 100644 --- a/shared/source/os_interface/windows/wddm/wddm.cpp +++ b/shared/source/os_interface/windows/wddm/wddm.cpp @@ -558,11 +558,13 @@ bool Wddm::mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_A return ret; } -D3DGPU_VIRTUAL_ADDRESS Wddm::reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS minimumAddress, +D3DGPU_VIRTUAL_ADDRESS Wddm::reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS baseAddress, + D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size) { UNRECOVERABLE_IF(size % MemoryConstants::pageSize64k); D3DDDI_RESERVEGPUVIRTUALADDRESS reserveGpuVirtualAddress = {}; + reserveGpuVirtualAddress.BaseAddress = baseAddress; reserveGpuVirtualAddress.MinimumAddress = minimumAddress; reserveGpuVirtualAddress.MaximumAddress = maximumAddress; reserveGpuVirtualAddress.hPagingQueue = this->pagingQueue; diff --git a/shared/source/os_interface/windows/wddm/wddm.h b/shared/source/os_interface/windows/wddm/wddm.h index 2aba5fc363..6089557474 100644 --- a/shared/source/os_interface/windows/wddm/wddm.h +++ b/shared/source/os_interface/windows/wddm/wddm.h @@ -64,7 +64,7 @@ class Wddm : public DriverModel { MOCKABLE_VIRTUAL bool makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t totalSize); MOCKABLE_VIRTUAL bool mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_VIRTUAL_ADDRESS preferredAddress, D3DGPU_VIRTUAL_ADDRESS &gpuPtr); bool mapGpuVirtualAddress(AllocationStorageData *allocationStorageData); - MOCKABLE_VIRTUAL D3DGPU_VIRTUAL_ADDRESS reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size); + MOCKABLE_VIRTUAL D3DGPU_VIRTUAL_ADDRESS reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS baseAddress, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size); MOCKABLE_VIRTUAL bool createContext(OsContextWin &osContext); MOCKABLE_VIRTUAL void applyAdditionalContextFlags(CREATECONTEXT_PVTDATA &privateData, OsContextWin &osContext, const HardwareInfo &hwInfo); MOCKABLE_VIRTUAL void applyAdditionalMapGPUVAFields(D3DDDI_MAPGPUVIRTUALADDRESS &mapGPUVA, Gmm *gmm); diff --git a/shared/source/os_interface/windows/wddm_memory_manager.cpp b/shared/source/os_interface/windows/wddm_memory_manager.cpp index f33fadfdd7..6882276028 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.cpp +++ b/shared/source/os_interface/windows/wddm_memory_manager.cpp @@ -749,6 +749,26 @@ bool WddmMemoryManager::createWddmAllocation(WddmAllocation *allocation, void *r return mapGpuVirtualAddress(allocation, requiredGpuPtr); } +AddressRange WddmMemoryManager::reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) { + uint64_t gpuVa = 0u; + *reservedOnRootDeviceIndex = 0; + size_t reservedSize = 0; + for (uint32_t rootDeviceIndex = 0; rootDeviceIndex < rootDeviceIndices.size(); rootDeviceIndex++) { + auto gfxPartition = getGfxPartition(rootDeviceIndex); + gpuVa = getWddm(rootDeviceIndex).reserveGpuVirtualAddress(reinterpret_cast(requiredStartAddress), gfxPartition->getHeapMinimalAddress(HeapIndex::HEAP_STANDARD64KB), gfxPartition->getHeapLimit(HeapIndex::HEAP_STANDARD64KB), size); + if (gpuVa != 0u) { + *reservedOnRootDeviceIndex = rootDeviceIndex; + reservedSize = size; + break; + } + } + return AddressRange{gpuVa, reservedSize}; +} + +void WddmMemoryManager::freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) { + getWddm(rootDeviceIndex).freeGpuVirtualAddress(addressRange.address, addressRange.size); +} + bool WddmMemoryManager::mapGpuVaForOneHandleAllocation(WddmAllocation *allocation, const void *preferredGpuVirtualAddress) { D3DGPU_VIRTUAL_ADDRESS addressToMap = castToUint64(preferredGpuVirtualAddress); auto heapIndex = selectHeap(allocation, preferredGpuVirtualAddress != nullptr, is32bit || executionEnvironment.rootDeviceEnvironments[allocation->getRootDeviceIndex()]->isFullRangeSvm(), allocation->allocInFrontWindowPool); @@ -795,7 +815,7 @@ bool WddmMemoryManager::mapMultiHandleAllocationWithRetry(WddmAllocation *alloca allocation->getGpuAddressToModify() = addressToMap; } else { allocation->reservedSizeForGpuVirtualAddress = alignUp(alignedSize, MemoryConstants::pageSize64k); - allocation->reservedGpuVirtualAddress = wddm.reserveGpuVirtualAddress(gfxPartition->getHeapMinimalAddress(heapIndex), gfxPartition->getHeapLimit(heapIndex), + allocation->reservedGpuVirtualAddress = wddm.reserveGpuVirtualAddress(0ull, gfxPartition->getHeapMinimalAddress(heapIndex), gfxPartition->getHeapLimit(heapIndex), allocation->reservedSizeForGpuVirtualAddress); auto gmmHelper = getGmmHelper(allocation->getRootDeviceIndex()); allocation->getGpuAddressToModify() = gmmHelper->canonize(allocation->reservedGpuVirtualAddress); diff --git a/shared/source/os_interface/windows/wddm_memory_manager.h b/shared/source/os_interface/windows/wddm_memory_manager.h index aec55d759f..5ae959ad5a 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.h +++ b/shared/source/os_interface/windows/wddm_memory_manager.h @@ -73,8 +73,8 @@ class WddmMemoryManager : public MemoryManager { bool isCpuCopyRequired(const void *ptr) override; bool isWCMemory(const void *ptr) override; - AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override { return AddressRange{0, 0}; }; - void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{}; + AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override; + void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override; bool verifyHandle(osHandle handle, uint32_t rootDeviceIndex, bool ntHandle) override; bool isNTHandle(osHandle handle, uint32_t rootDeviceIndex) override; void releaseDeviceSpecificMemResources(uint32_t rootDeviceIndex) override{}; diff --git a/shared/test/common/mocks/mock_wddm.cpp b/shared/test/common/mocks/mock_wddm.cpp index b0b335bd2a..5d6b06e233 100644 --- a/shared/test/common/mocks/mock_wddm.cpp +++ b/shared/test/common/mocks/mock_wddm.cpp @@ -292,9 +292,13 @@ VOID *WddmMock::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback return Wddm::registerTrimCallback(callback, residencyController); } -D3DGPU_VIRTUAL_ADDRESS WddmMock::reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size) { +D3DGPU_VIRTUAL_ADDRESS WddmMock::reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS baseAddress, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size) { reserveGpuVirtualAddressResult.called++; - return Wddm::reserveGpuVirtualAddress(minimumAddress, maximumAddress, size); + if (failReserveGpuVirtualAddress) { + D3DGPU_VIRTUAL_ADDRESS nullAddress = {}; + return nullAddress; + } + return Wddm::reserveGpuVirtualAddress(baseAddress, minimumAddress, maximumAddress, size); } uint64_t *WddmMock::getPagingFenceAddress() { diff --git a/shared/test/common/mocks/mock_wddm.h b/shared/test/common/mocks/mock_wddm.h index fcd0a9405b..4f582afa4f 100644 --- a/shared/test/common/mocks/mock_wddm.h +++ b/shared/test/common/mocks/mock_wddm.h @@ -89,7 +89,7 @@ class WddmMock : public Wddm { void virtualFree(void *ptr, size_t size) override; void releaseReservedAddress(void *reservedAddress) override; VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController) override; - D3DGPU_VIRTUAL_ADDRESS reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size) override; + D3DGPU_VIRTUAL_ADDRESS reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS baseAddress, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size) override; bool reserveValidAddressRange(size_t size, void *&reservedMem) override; PLATFORM *getGfxPlatform() { return gfxPlatform.get(); } uint64_t *getPagingFenceAddress() override; @@ -185,5 +185,6 @@ class WddmMock : public Wddm { bool shutdownStatus = false; bool callBaseSetAllocationPriority = true; bool callBaseWaitFromCpu = true; + bool failReserveGpuVirtualAddress = false; }; } // namespace NEO diff --git a/shared/test/unit_test/device/neo_device_tests.cpp b/shared/test/unit_test/device/neo_device_tests.cpp index c045b3ca66..558371251c 100644 --- a/shared/test/unit_test/device/neo_device_tests.cpp +++ b/shared/test/unit_test/device/neo_device_tests.cpp @@ -352,7 +352,7 @@ TEST_F(DeviceGetCapsTest, givenFlagEnabled64kbPagesWhenCallConstructorMemoryMana }; uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override { return 0; }; double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override { return 0; } - AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override { + AddressRange reserveGpuAddress(const void *requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override { return {}; } void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override{}; diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index 328fbff916..c1e8e93c79 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -234,7 +234,10 @@ TEST_F(DrmMemoryManagerTest, GivenGraphicsAllocationWhenAddAndRemoveAllocationTo TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDrmMemoryManagerWhenGpuAddressIsReservedAndFreedThenAddressFromGfxPartitionIsUsed) { auto memoryManager = std::make_unique(false, true, false, *executionEnvironment); - auto addressRange = memoryManager->reserveGpuAddress(MemoryConstants::pageSize, 0); + RootDeviceIndicesContainer rootDevices; + rootDevices.push_back(0); + uint32_t rootDeviceIndexReserved = 1; + auto addressRange = memoryManager->reserveGpuAddress(nullptr, MemoryConstants::pageSize, rootDevices, &rootDeviceIndexReserved); auto gmmHelper = memoryManager->getGmmHelper(0); EXPECT_LE(memoryManager->getGfxPartition(0)->getHeapBase(HeapIndex::HEAP_STANDARD), gmmHelper->decanonize(addressRange.address)); @@ -242,6 +245,30 @@ TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDrmMemoryManagerWhenGp memoryManager->freeGpuAddress(addressRange, 0); } +TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDrmMemoryManagerWhenGpuAddressReservationIsAttemptedWithARequiredPtrThenNullRangeReturned) { + auto memoryManager = std::make_unique(false, true, false, *executionEnvironment); + void *requiredPtr = reinterpret_cast(0x1234); + RootDeviceIndicesContainer rootDevices; + rootDevices.push_back(0); + uint32_t rootDeviceIndexReserved = 1; + auto addressRange = memoryManager->reserveGpuAddress(requiredPtr, MemoryConstants::pageSize, rootDevices, &rootDeviceIndexReserved); + EXPECT_EQ(static_cast(addressRange.address), 0); + EXPECT_EQ(static_cast(addressRange.size), 0); +} + +TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDrmMemoryManagerWhenGpuAddressReservationIsAttemptedWhichFailsThenNullRangeReturned) { + auto memoryManager = std::make_unique(false, true, false, *executionEnvironment); + RootDeviceIndicesContainer rootDevices; + rootDevices.push_back(0); + uint32_t rootDeviceIndexReserved = 1; + // emulate GPU address space exhaust + memoryManager->forceLimitedRangeAllocator(0xFFFFFFFFF); + memoryManager->getGfxPartition(0)->heapInit(HeapIndex::HEAP_STANDARD, 0x0, 0x10000); + size_t invalidSize = (size_t)memoryManager->getGfxPartition(0)->getHeapLimit(HeapIndex::HEAP_STANDARD) + MemoryConstants::pageSize; + auto addressRange = memoryManager->reserveGpuAddress(nullptr, invalidSize, rootDevices, &rootDeviceIndexReserved); + EXPECT_EQ(static_cast(addressRange.address), 0); +} + TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenSmallSizeAndGpuAddressSetWhenGraphicsMemoryIsAllocatedThenAllocationWithSpecifiedGpuAddressInSystemMemoryIsCreated) { auto memoryManager = std::make_unique(false, true, false, *executionEnvironment); auto osContext = device->getDefaultEngine().osContext; diff --git a/shared/test/unit_test/os_interface/windows/CMakeLists.txt b/shared/test/unit_test/os_interface/windows/CMakeLists.txt index c0a2fa8133..3b1eb31e5d 100644 --- a/shared/test/unit_test/os_interface/windows/CMakeLists.txt +++ b/shared/test/unit_test/os_interface/windows/CMakeLists.txt @@ -28,6 +28,7 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/wddm_shared_allocations_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wddm_special_heap_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wddm_tests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_reservation_tests.cpp ) endif() diff --git a/shared/test/unit_test/os_interface/windows/wddm_memory_reservation_tests.cpp b/shared/test/unit_test/os_interface/windows/wddm_memory_reservation_tests.cpp new file mode 100644 index 0000000000..ae33a7639b --- /dev/null +++ b/shared/test/unit_test/os_interface/windows/wddm_memory_reservation_tests.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/execution_environment/root_device_environment.h" +#include "shared/source/gmm_helper/gmm_helper.h" +#include "shared/source/os_interface/device_factory.h" +#include "shared/source/os_interface/os_interface.h" +#include "shared/test/common/fixtures/device_fixture.h" +#include "shared/test/common/helpers/debug_manager_state_restore.h" +#include "shared/test/common/mocks/mock_wddm.h" +#include "shared/test/common/os_interface/windows/mock_wddm_memory_manager.h" +#include "shared/test/common/test_macros/hw_test.h" + +namespace NEO { + +struct MemoryReservationMock : public MockWddmMemoryManager { + using MemoryManager::freeGpuAddress; + using MemoryManager::reserveGpuAddress; + MemoryReservationMock(NEO::ExecutionEnvironment &executionEnvironment) : MockWddmMemoryManager(executionEnvironment) {} +}; + +class WddmMemManagerFixture { + public: + void setUp() { + executionEnvironment = std::make_unique(); + executionEnvironment->prepareRootDeviceEnvironments(1); + executionEnvironment->rootDeviceEnvironments[0]->setHwInfo(defaultHwInfo.get()); + executionEnvironment->rootDeviceEnvironments[0]->initGmm(); + + DeviceFactory::prepareDeviceEnvironments(*executionEnvironment); + auto wddm = static_cast(executionEnvironment->rootDeviceEnvironments[0]->osInterface->getDriverModel()->as()); + wddm->callBaseMapGpuVa = false; + memManager = std::unique_ptr(new MemoryReservationMock(*executionEnvironment)); + } + void tearDown() { + } + std::unique_ptr memManager; + std::unique_ptr executionEnvironment; +}; + +typedef ::Test WddmMemoryReservationTests; + +TEST_F(WddmMemoryReservationTests, givenWddmMemoryManagerWhenGpuAddressIsReservedAndFreedThenAddressFromGfxPartitionIsUsed) { + RootDeviceIndicesContainer rootDevices; + rootDevices.push_back(0); + uint32_t rootDeviceIndexReserved = 1; + auto addressRange = memManager->reserveGpuAddress(nullptr, MemoryConstants::pageSize64k, rootDevices, &rootDeviceIndexReserved); + auto gmmHelper = memManager->getGmmHelper(0); + + EXPECT_LE(memManager->getGfxPartition(0)->getHeapBase(HeapIndex::HEAP_STANDARD64KB), gmmHelper->decanonize(addressRange.address)); + EXPECT_GT(memManager->getGfxPartition(0)->getHeapLimit(HeapIndex::HEAP_STANDARD64KB), gmmHelper->decanonize(addressRange.address)); + memManager->freeGpuAddress(addressRange, 0); +} + +TEST(WddmMemoryReservationFailTest, givenWddmMemoryManagerWhenGpuAddressReservationIsAttemptedThenNullPtrAndSizeReturned) { + std::unique_ptr memManager; + std::unique_ptr executionEnvironment; + + executionEnvironment = std::make_unique(); + executionEnvironment->prepareRootDeviceEnvironments(1); + executionEnvironment->rootDeviceEnvironments[0]->setHwInfo(defaultHwInfo.get()); + executionEnvironment->rootDeviceEnvironments[0]->initGmm(); + + DeviceFactory::prepareDeviceEnvironments(*executionEnvironment); + auto wddm = static_cast(executionEnvironment->rootDeviceEnvironments[0]->osInterface->getDriverModel()->as()); + wddm->callBaseMapGpuVa = false; + wddm->failReserveGpuVirtualAddress = true; + memManager = std::unique_ptr(new MemoryReservationMock(*executionEnvironment)); + + RootDeviceIndicesContainer rootDevices; + rootDevices.push_back(0); + uint32_t rootDeviceIndexReserved = 1; + auto addressRange = memManager->reserveGpuAddress(nullptr, MemoryConstants::pageSize64k, rootDevices, &rootDeviceIndexReserved); + EXPECT_EQ(addressRange.address, 0ull); + EXPECT_EQ(addressRange.size, 0u); +} +} // namespace NEO