From f928d695e7fa0f6d1095fe4290e73d1464a73db5 Mon Sep 17 00:00:00 2001 From: "Spruit, Neil R" Date: Tue, 6 Dec 2022 22:36:12 +0000 Subject: [PATCH] Support for Mapping Physical with Virtual Memory Related-To: LOCI-3422, LOCI-3421 Signed-off-by: Neil R Spruit --- .../core/source/context/context_imp.cpp | 121 +++++++- .../unit_tests/fixtures/memory_ipc_fixture.h | 8 + .../sources/context/test_context.cpp | 281 ++++++++++++++++-- .../unit_tests/sources/event/test_event.cpp | 4 + ..._manager_allocate_in_device_pool_tests.cpp | 114 +++++++ .../windows/wddm_memory_manager_tests.cpp | 94 ++++++ .../source/memory_manager/memory_manager.cpp | 28 ++ shared/source/memory_manager/memory_manager.h | 16 +- .../os_agnostic_memory_manager.cpp | 74 +++++ .../os_agnostic_memory_manager.h | 4 + .../os_interface/linux/drm_memory_manager.cpp | 117 ++++++++ .../os_interface/linux/drm_memory_manager.h | 4 + .../os_interface/windows/wddm_allocation.h | 4 +- .../windows/wddm_memory_manager.cpp | 116 +++++++- .../windows/wddm_memory_manager.h | 7 +- .../mocks/linux/mock_drm_memory_manager.h | 6 +- .../test/common/mocks/mock_memory_manager.cpp | 8 + .../test/common/mocks/mock_memory_manager.h | 10 + shared/test/common/mocks/mock_wddm.cpp | 3 + shared/test/common/mocks/mock_wddm.h | 1 + .../windows/mock_wddm_memory_manager.h | 7 +- .../os_interface/windows/wddm_fixture.h | 4 + .../unit_test/device/neo_device_tests.cpp | 4 + ..._manager_allocate_in_device_pool_tests.cpp | 132 ++++++++ .../linux/drm_memory_manager_tests.cpp | 171 +++++++++++ ..._device_address_space_drm_or_wddm_test.cpp | 41 +++ .../windows/wddm_memory_manager_tests.cpp | 3 +- 27 files changed, 1353 insertions(+), 29 deletions(-) diff --git a/level_zero/core/source/context/context_imp.cpp b/level_zero/core/source/context/context_imp.cpp index ab3972a8d2..0e62b24f4f 100644 --- a/level_zero/core/source/context/context_imp.cpp +++ b/level_zero/core/source/context/context_imp.cpp @@ -8,6 +8,7 @@ #include "level_zero/core/source/context/context_imp.h" #include "shared/source/command_container/implicit_scaling.h" +#include "shared/source/command_stream/command_stream_receiver.h" #include "shared/source/execution_environment/root_device_environment.h" #include "shared/source/helpers/basic_math.h" #include "shared/source/memory_manager/allocation_properties.h" @@ -844,7 +845,7 @@ ze_result_t ContextImp::reserveVirtualMem(const void *pStart, virtualMemoryReservation->flags.readWrite = false; virtualMemoryReservation->flags.readOnly = false; virtualMemoryReservation->flags.noAccess = true; - virtualMemoryReservation->mapped = false; + virtualMemoryReservation->mappedAllocation = nullptr; auto lock = this->driverHandle->getMemoryManager()->lockVirtualMemoryReservationMap(); this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().insert(std::pair(reinterpret_cast(virtualMemoryReservation->virtualAddressRange.address), virtualMemoryReservation)); *pptr = reinterpret_cast(virtualMemoryReservation->virtualAddressRange.address); @@ -884,11 +885,46 @@ ze_result_t ContextImp::queryVirtualMemPageSize(ze_device_handle_t hDevice, ze_result_t ContextImp::createPhysicalMem(ze_device_handle_t hDevice, ze_physical_mem_desc_t *desc, ze_physical_mem_handle_t *phPhysicalMemory) { - return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; + if ((getPageSizeRequired(desc->size) != desc->size)) { + return ZE_RESULT_ERROR_UNSUPPORTED_SIZE; + } + auto device = Device::fromHandle(hDevice); + auto neoDevice = device->getNEODevice(); + + NEO::AllocationProperties physicalDeviceMemoryProperties{neoDevice->getRootDeviceIndex(), + true, + desc->size, + NEO::AllocationType::BUFFER, + false, + false, + device->getNEODevice()->getDeviceBitfield()}; + physicalDeviceMemoryProperties.flags.isUSMDeviceAllocation = true; + physicalDeviceMemoryProperties.flags.shareable = 1; + + NEO::GraphicsAllocation *physicalDeviceMemoryAllocation = this->driverHandle->getMemoryManager()->allocatePhysicalGraphicsMemory(physicalDeviceMemoryProperties); + if (!physicalDeviceMemoryAllocation) { + return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY; + } + NEO::PhysicalMemoryAllocation *physicalMemoryAllocation = new NEO::PhysicalMemoryAllocation; + physicalMemoryAllocation->allocation = physicalDeviceMemoryAllocation; + physicalMemoryAllocation->device = neoDevice; + auto lock = this->driverHandle->getMemoryManager()->lockPhysicalMemoryAllocationMap(); + this->driverHandle->getMemoryManager()->getPhysicalMemoryAllocationMap().insert(std::pair(reinterpret_cast(physicalDeviceMemoryAllocation), physicalMemoryAllocation)); + *phPhysicalMemory = reinterpret_cast(physicalDeviceMemoryAllocation); + return ZE_RESULT_SUCCESS; } ze_result_t ContextImp::destroyPhysicalMem(ze_physical_mem_handle_t hPhysicalMemory) { - return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; + std::map::iterator it; + auto lock = this->driverHandle->getMemoryManager()->lockPhysicalMemoryAllocationMap(); + it = this->driverHandle->getMemoryManager()->getPhysicalMemoryAllocationMap().find(static_cast(hPhysicalMemory)); + if (it != this->driverHandle->getMemoryManager()->getPhysicalMemoryAllocationMap().end()) { + NEO::PhysicalMemoryAllocation *allocationNode = it->second; + this->driverHandle->getMemoryManager()->freeGraphicsMemoryImpl(allocationNode->allocation); + this->driverHandle->getMemoryManager()->getPhysicalMemoryAllocationMap().erase(it); + delete allocationNode; + } + return ZE_RESULT_SUCCESS; } ze_result_t ContextImp::mapVirtualMem(const void *ptr, @@ -896,12 +932,87 @@ ze_result_t ContextImp::mapVirtualMem(const void *ptr, ze_physical_mem_handle_t hPhysicalMemory, size_t offset, ze_memory_access_attribute_t access) { - return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; + std::map::iterator physicalIt; + NEO::PhysicalMemoryAllocation *allocationNode = nullptr; + + if ((getPageSizeRequired(size) != size)) { + return ZE_RESULT_ERROR_UNSUPPORTED_ALIGNMENT; + } + auto lockPhysical = this->driverHandle->getMemoryManager()->lockPhysicalMemoryAllocationMap(); + physicalIt = this->driverHandle->getMemoryManager()->getPhysicalMemoryAllocationMap().find(static_cast(hPhysicalMemory)); + if (physicalIt != this->driverHandle->getMemoryManager()->getPhysicalMemoryAllocationMap().end()) { + allocationNode = physicalIt->second; + } else { + return ZE_RESULT_ERROR_INVALID_ARGUMENT; + } + + std::map::iterator virtualIt; + NEO::VirtualMemoryReservation *virtualMemoryReservation = nullptr; + auto lockVirtual = this->driverHandle->getMemoryManager()->lockVirtualMemoryReservationMap(); + virtualIt = this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().find(const_cast(ptr)); + if (virtualIt != this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().end()) { + virtualMemoryReservation = virtualIt->second; + switch (access) { + case ZE_MEMORY_ACCESS_ATTRIBUTE_NONE: + virtualMemoryReservation->flags.readOnly = false; + virtualMemoryReservation->flags.noAccess = true; + virtualMemoryReservation->flags.readWrite = false; + break; + case ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE: + virtualMemoryReservation->flags.readOnly = false; + virtualMemoryReservation->flags.noAccess = false; + virtualMemoryReservation->flags.readWrite = true; + break; + case ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY: + virtualMemoryReservation->flags.readWrite = false; + virtualMemoryReservation->flags.noAccess = false; + virtualMemoryReservation->flags.readOnly = true; + break; + default: + return ZE_RESULT_ERROR_INVALID_ENUMERATION; + } + if (virtualMemoryReservation->mappedAllocation != nullptr) { + return ZE_RESULT_ERROR_INVALID_ARGUMENT; + } + } else { + return ZE_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (this->driverHandle->getMemoryManager()->mapPhysicalToVirtualMemory(allocationNode->allocation, reinterpret_cast(ptr), size) == true) { + NEO::SvmAllocationData allocData(allocationNode->allocation->getRootDeviceIndex()); + allocData.gpuAllocations.addAllocation(allocationNode->allocation); + allocData.cpuAllocation = nullptr; + allocData.device = allocationNode->device; + allocData.size = virtualMemoryReservation->virtualAddressRange.size; + allocData.pageSizeForAlignment = MemoryConstants::pageSize64k; + allocData.setAllocId(this->driverHandle->svmAllocsManager->allocationsCounter++); + virtualMemoryReservation->mappedAllocation = allocationNode; + this->driverHandle->getSvmAllocsManager()->insertSVMAlloc(allocData); + NEO::MemoryOperationsHandler *memoryOperationsIface = allocationNode->device->getRootDeviceEnvironment().memoryOperationsInterface.get(); + auto success = memoryOperationsIface->makeResident(allocationNode->device, ArrayRef(&allocationNode->allocation, 1)); + ze_result_t res = changeMemoryOperationStatusToL0ResultType(success); + return res; + } + return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY; } ze_result_t ContextImp::unMapVirtualMem(const void *ptr, size_t size) { - return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; + + std::map::iterator it; + auto lock = this->driverHandle->getMemoryManager()->lockVirtualMemoryReservationMap(); + it = this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().find(const_cast(ptr)); + if (it != this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().end()) { + NEO::VirtualMemoryReservation *virtualMemoryReservation = this->driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().at(const_cast(ptr)); + NEO::SvmAllocationData *allocData = this->driverHandle->getSvmAllocsManager()->getSVMAlloc(reinterpret_cast(virtualMemoryReservation->mappedAllocation->allocation->getGpuAddress())); + this->driverHandle->getSvmAllocsManager()->removeSVMAlloc(*allocData); + NEO::Device *device = virtualMemoryReservation->mappedAllocation->device; + NEO::CommandStreamReceiver *csr = device->getDefaultEngine().commandStreamReceiver; + NEO::OsContext *osContext = &csr->getOsContext(); + this->driverHandle->getMemoryManager()->unMapPhysicalToVirtualMemory(virtualMemoryReservation->mappedAllocation->allocation, reinterpret_cast(ptr), size, osContext, virtualMemoryReservation->rootDeviceIndex); + virtualMemoryReservation->mappedAllocation = nullptr; + } + return ZE_RESULT_SUCCESS; } ze_result_t ContextImp::setVirtualMemAccessAttribute(const void *ptr, diff --git a/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h b/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h index 84197abba4..14e0cb8283 100644 --- a/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h +++ b/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h @@ -418,6 +418,10 @@ class MemoryManagerIpcMock : public NEO::MemoryManager { NEO::GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const NEO::AllocationData &allocationData, bool useLocalMemory) override { return nullptr; }; NEO::GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const NEO::AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; NEO::GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const NEO::AllocationData &allocationData) override { return nullptr; }; + GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; + bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; NEO::GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const NEO::AllocationData &allocationData, std::unique_ptr gmm) override { return nullptr; }; NEO::GraphicsAllocation *allocateMemoryByKMD(const NEO::AllocationData &allocationData) override { return nullptr; }; @@ -628,6 +632,10 @@ class MemoryManagerIpcImplicitScalingMock : public NEO::MemoryManager { NEO::GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const NEO::AllocationData &allocationData, bool useLocalMemory) override { return nullptr; }; NEO::GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const NEO::AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; NEO::GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const NEO::AllocationData &allocationData) override { return nullptr; }; + GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; + bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; NEO::GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const NEO::AllocationData &allocationData, std::unique_ptr gmm) override { return nullptr; }; NEO::GraphicsAllocation *allocateMemoryByKMD(const NEO::AllocationData &allocationData) override { return nullptr; }; diff --git a/level_zero/core/test/unit_tests/sources/context/test_context.cpp b/level_zero/core/test/unit_tests/sources/context/test_context.cpp index 707ea84ca9..7ae9a33807 100644 --- a/level_zero/core/test/unit_tests/sources/context/test_context.cpp +++ b/level_zero/core/test/unit_tests/sources/context/test_context.cpp @@ -843,7 +843,7 @@ TEST_F(ContextTest, whenCallingVirtualMemInterfacesThenSuccessIsReturned) { EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, res); - EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0); + EXPECT_GT(static_cast(driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size()), 0); res = contextImp->freeVirtualMem(ptr, pagesize); EXPECT_EQ(ZE_RESULT_SUCCESS, res); @@ -883,7 +883,7 @@ TEST_F(ContextTest, whenCallingQueryVirtualMemPageSizeCorrectAlignmentIsReturned EXPECT_EQ(ZE_RESULT_SUCCESS, res); } -TEST_F(ContextTest, whenCallingPhysicalMemInterfacesThenUnsupportedIsReturned) { +TEST_F(ContextTest, whenCallingPhysicalMemInterfacesThenSuccessIsReturned) { ze_context_handle_t hContext; ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; @@ -892,19 +892,24 @@ TEST_F(ContextTest, whenCallingPhysicalMemInterfacesThenUnsupportedIsReturned) { ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); - ze_physical_mem_desc_t descMem = {}; + size_t size = 1024; + size_t pagesize = 0u; + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, 0, pagesize}; ze_physical_mem_handle_t mem = {}; res = contextImp->createPhysicalMem(device, &descMem, &mem); - EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = contextImp->destroyPhysicalMem(mem); - EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = contextImp->destroy(); EXPECT_EQ(ZE_RESULT_SUCCESS, res); } -TEST_F(ContextTest, whenCallingMappingVirtualInterfacesThenUnsupportedIsReturned) { +TEST_F(ContextTest, whenCallingPhysicalMemCreateWithInvalisSizeThenUnsupportedSizeReturned) { ze_context_handle_t hContext; ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; @@ -913,6 +918,64 @@ TEST_F(ContextTest, whenCallingMappingVirtualInterfacesThenUnsupportedIsReturned ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + size_t size = 1024; + size_t pagesize = 0u; + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, 0, 10}; + ze_physical_mem_handle_t mem = {}; + res = contextImp->createPhysicalMem(device, &descMem, &mem); + EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_SIZE, res); + + res = contextImp->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + +TEST_F(ContextTest, whenCallingDestroyPhysicalMemWithIncorrectPointerThenMemoryNotFreed) { + ze_context_handle_t hContext; + ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + + ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + + size_t size = 1024; + size_t pagesize = 0u; + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, 0, pagesize}; + ze_physical_mem_handle_t mem = {}; + res = contextImp->createPhysicalMem(device, &descMem, &mem); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_GT(static_cast(driverHandle->getMemoryManager()->getPhysicalMemoryAllocationMap().size()), 0); + + res = contextImp->destroyPhysicalMem(nullptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_GT(static_cast(driverHandle->getMemoryManager()->getPhysicalMemoryAllocationMap().size()), 0); + + res = contextImp->destroyPhysicalMem(mem); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_EQ(static_cast(driverHandle->getMemoryManager()->getPhysicalMemoryAllocationMap().size()), 0); + + res = contextImp->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + +TEST_F(ContextTest, whenCallingMappingVirtualInterfacesThenSuccessIsReturned) { + ze_context_handle_t hContext; + ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + + ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[0]->memoryOperationsInterface = + std::make_unique(); + + ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + void *pStart = 0x0; size_t size = 4096u; void *ptr = nullptr; @@ -921,17 +984,29 @@ TEST_F(ContextTest, whenCallingMappingVirtualInterfacesThenUnsupportedIsReturned EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, res); - EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0); + EXPECT_GT(static_cast(driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size()), 0); - ze_physical_mem_desc_t descMem = {}; + ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, 0, pagesize}; ze_physical_mem_handle_t mem = {}; res = contextImp->createPhysicalMem(device, &descMem, &mem); - EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); - ze_memory_access_attribute_t access = {}; + ze_memory_access_attribute_t access = {ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE}; size_t offset = 0; + + std::vector memoryAccessFlags = { + ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE, ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY, + ZE_MEMORY_ACCESS_ATTRIBUTE_NONE}; + + for (auto accessFlags : memoryAccessFlags) { + res = contextImp->mapVirtualMem(ptr, pagesize, mem, offset, accessFlags); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + res = contextImp->unMapVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + } + res = contextImp->mapVirtualMem(ptr, pagesize, mem, offset, access); - EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = contextImp->setVirtualMemAccessAttribute(ptr, pagesize, access); EXPECT_EQ(ZE_RESULT_SUCCESS, res); @@ -943,10 +1018,10 @@ TEST_F(ContextTest, whenCallingMappingVirtualInterfacesThenUnsupportedIsReturned EXPECT_EQ(pagesize, outSize); res = contextImp->unMapVirtualMem(ptr, pagesize); - EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = contextImp->destroyPhysicalMem(mem); - EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, res); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = contextImp->freeVirtualMem(ptr, pagesize); EXPECT_EQ(ZE_RESULT_SUCCESS, res); @@ -972,7 +1047,7 @@ TEST_F(ContextTest, whenCallingVirtualMemorySetAttributeWithValidEnumerationsThe EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, res); - EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0); + EXPECT_GT(static_cast(driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size()), 0); std::vector memoryAccessFlags = { ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE, ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY, @@ -1019,7 +1094,7 @@ TEST_F(ContextTest, whenCallingVirtualMemorySetAttributeWithInvalidValuesThenFai EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, res); - EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0); + EXPECT_GT(static_cast(driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size()), 0); ze_memory_access_attribute_t access = {ZE_MEMORY_ACCESS_ATTRIBUTE_FORCE_UINT32}; res = contextImp->setVirtualMemAccessAttribute(ptr, pagesize, access); @@ -1052,7 +1127,7 @@ TEST_F(ContextTest, whenCallingVirtualMemoryGetAttributeWithInvalidValuesThenFai EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, res); - EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0); + EXPECT_GT(static_cast(driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size()), 0); size_t outSize = 0; ze_memory_access_attribute_t access = {ZE_MEMORY_ACCESS_ATTRIBUTE_NONE}; @@ -1084,7 +1159,7 @@ TEST_F(ContextTest, whenCallingVirtualMemoryFreeWithInvalidValuesThenFailuresRet EXPECT_EQ(ZE_RESULT_SUCCESS, res); res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, res); - EXPECT_GT((int)driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size(), 0); + EXPECT_GT(static_cast(driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size()), 0); res = contextImp->freeVirtualMem(ptr, invalidSize); EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res); @@ -1154,6 +1229,25 @@ class ReserveMemoryManagerMock : public NEO::MemoryManager { NEO::GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const NEO::AllocationData &allocationData, bool useLocalMemory) override { return nullptr; }; NEO::GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const NEO::AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; NEO::GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const NEO::AllocationData &allocationData) override { return nullptr; }; + GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; + bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { + if (failMapVirtualMemory) { + return false; + } else { + return true; + } + }; + + NEO::GraphicsAllocation *allocatePhysicalGraphicsMemory(const AllocationProperties &properties) override { + + if (failAllocatePhysicalGraphicsMemory) { + return nullptr; + } + mockAllocation.reset(new NEO::MockGraphicsAllocation(const_cast(buffer), size)); + return mockAllocation.get(); + } NEO::GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const NEO::AllocationData &allocationData, std::unique_ptr gmm) override { return nullptr; }; NEO::GraphicsAllocation *allocateMemoryByKMD(const NEO::AllocationData &allocationData) override { return nullptr; }; @@ -1161,6 +1255,11 @@ class ReserveMemoryManagerMock : public NEO::MemoryManager { void unlockResourceImpl(NEO::GraphicsAllocation &graphicsAllocation) override{}; bool failReserveGpuAddress = true; + bool failMapVirtualMemory = true; + bool failAllocatePhysicalGraphicsMemory = true; + void *buffer = nullptr; + size_t size = 0; + std::unique_ptr mockAllocation; }; TEST_F(ContextTest, whenCallingVirtualMemReserveWithPStartWithSuccessfulAllocationThenSuccessReturned) { @@ -1225,6 +1324,154 @@ TEST_F(ContextTest, whenCallingVirtualMemoryReservationWhenOutOfMemoryThenOutOfM EXPECT_EQ(ZE_RESULT_SUCCESS, res); } +TEST_F(ContextTest, whenCallingPhysicalMemoryAllocateWhenOutOfMemoryThenOutofMemoryReturned) { + ze_context_handle_t hContext; + ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + + ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + + size_t size = 0u; + size_t pagesize = 0u; + + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + NEO::MemoryManager *failingReserveMemoryManager = new ReserveMemoryManagerMock(*neoDevice->executionEnvironment); + auto memoryManager = driverHandle->getMemoryManager(); + driverHandle->setMemoryManager(failingReserveMemoryManager); + ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, 0, pagesize}; + ze_physical_mem_handle_t mem = {}; + res = contextImp->createPhysicalMem(device, &descMem, &mem); + EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY, res); + driverHandle->setMemoryManager(memoryManager); + delete failingReserveMemoryManager; + + res = contextImp->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + +TEST_F(ContextTest, whenCallingMapVirtualMemoryWithFailedMapThenOutOfMemoryreturned) { + ze_context_handle_t hContext; + std::unique_ptr mockMemoryManager; + ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + + ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[0]->memoryOperationsInterface = + std::make_unique(); + + ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + + mockMemoryManager = std::make_unique(); + + auto memoryManager = driverHandle->getMemoryManager(); + driverHandle->setMemoryManager(mockMemoryManager.get()); + + void *pStart = 0x0; + size_t size = 4096u; + void *ptr = nullptr; + size_t pagesize = 0u; + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_GT(static_cast(driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size()), 0); + + ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, 0, pagesize}; + ze_physical_mem_handle_t mem = {}; + + res = contextImp->createPhysicalMem(device, &descMem, &mem); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ze_memory_access_attribute_t access = {ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE}; + size_t offset = 0; + mockMemoryManager->failMapPhysicalToVirtualMemory = true; + res = contextImp->mapVirtualMem(ptr, pagesize, mem, offset, access); + EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY, res); + + res = contextImp->destroyPhysicalMem(mem); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->freeVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + driverHandle->setMemoryManager(memoryManager); + + res = contextImp->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + +TEST_F(ContextTest, whenCallingMapVirtualMemoryWithInvalidValuesThenFailureReturned) { + ze_context_handle_t hContext; + ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + + ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[0]->memoryOperationsInterface = + std::make_unique(); + + ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + + void *pStart = 0x0; + size_t size = 4096u; + void *ptr = nullptr; + size_t pagesize = 0u; + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_GT(static_cast(driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size()), 0); + + ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, 0, pagesize}; + ze_physical_mem_handle_t mem = {}; + ze_physical_mem_handle_t invalidMem = {}; + + res = contextImp->createPhysicalMem(device, &descMem, &mem); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ze_memory_access_attribute_t access = {ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE}; + size_t offset = 0; + + res = contextImp->mapVirtualMem(ptr, pagesize, invalidMem, offset, access); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res); + + res = contextImp->mapVirtualMem(nullptr, pagesize, mem, offset, access); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res); + + res = contextImp->unMapVirtualMem(nullptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->mapVirtualMem(ptr, 0u, mem, offset, access); + EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_ALIGNMENT, res); + + access = ZE_MEMORY_ACCESS_ATTRIBUTE_FORCE_UINT32; + res = contextImp->mapVirtualMem(ptr, pagesize, mem, offset, access); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ENUMERATION, res); + + access = ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE; + res = contextImp->mapVirtualMem(ptr, pagesize, mem, offset, access); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->mapVirtualMem(ptr, pagesize, mem, offset, access); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res); + + res = contextImp->unMapVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->destroyPhysicalMem(mem); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->freeVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + using IsAtMostProductDG1 = IsAtMostProduct; HWTEST2_F(ContextTest, WhenCreatingImageThenSuccessIsReturned, IsAtMostProductDG1) { diff --git a/level_zero/core/test/unit_tests/sources/event/test_event.cpp b/level_zero/core/test/unit_tests/sources/event/test_event.cpp index 773286aeb3..4e4069eef6 100644 --- a/level_zero/core/test/unit_tests/sources/event/test_event.cpp +++ b/level_zero/core/test/unit_tests/sources/event/test_event.cpp @@ -80,6 +80,10 @@ class MemoryManagerEventPoolFailMock : public NEO::MemoryManager { NEO::GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const NEO::AllocationData &allocationData, bool useLocalMemory) override { return nullptr; }; NEO::GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const NEO::AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; NEO::GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const NEO::AllocationData &allocationData) override { return nullptr; }; + GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; + bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; NEO::GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const NEO::AllocationData &allocationData, std::unique_ptr gmm) override { return nullptr; }; NEO::GraphicsAllocation *allocateMemoryByKMD(const NEO::AllocationData &allocationData) override { return nullptr; }; diff --git a/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_allocate_in_device_pool_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_allocate_in_device_pool_tests.cpp index 6eb2b5f995..69851e4fea 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_allocate_in_device_pool_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_allocate_in_device_pool_tests.cpp @@ -365,6 +365,101 @@ TEST_F(WddmMemoryManagerSimpleTest, givenEnabledLocalMemoryWhenAllocateFailsThen memoryManager->freeGraphicsMemory(allocation); } +TEST_F(WddmMemoryManagerSimpleTest, givenEnabledLocalMemoryWhenAllocateFailsThenGraphicsAllocationInPhysicalLocalDeviceMemoryReturnsError) { + const bool localMemoryEnabled = true; + auto executionEnvironment = platform()->peekExecutionEnvironment(); + memoryManager = std::make_unique(false, localMemoryEnabled, *executionEnvironment); + + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Success; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize; + allocData.flags.allocateMemory = true; + + wddm->callBaseDestroyAllocations = false; + wddm->createAllocationStatus = STATUS_NO_MEMORY; + + auto allocation = memoryManager->allocatePhysicalLocalDeviceMemory(allocData, status); + EXPECT_EQ(nullptr, allocation); + EXPECT_EQ(MemoryManager::AllocationStatus::Error, status); + + memoryManager->freeGraphicsMemory(allocation); +} + +TEST_F(WddmMemoryManagerSimpleTest, givenAllocatePhysicalLocalDeviceMemoryThenLocalMemoryAllocationHasCorrectStorageInfoAndNoGpuAddress) { + auto executionEnvironment = platform()->peekExecutionEnvironment(); + memoryManager = std::make_unique(false, true, *executionEnvironment); + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize; + allocData.flags.allocateMemory = true; + allocData.storageInfo.memoryBanks = 0x1; + allocData.storageInfo.pageTablesVisibility = 0x2; + allocData.storageInfo.cloningOfPageTables = false; + allocData.flags.flushL3 = true; + + auto allocation = memoryManager->allocatePhysicalLocalDeviceMemory(allocData, status); + EXPECT_NE(nullptr, allocation); + + EXPECT_EQ(allocData.storageInfo.memoryBanks, allocation->storageInfo.memoryBanks); + EXPECT_EQ(allocData.storageInfo.pageTablesVisibility, allocation->storageInfo.pageTablesVisibility); + EXPECT_FALSE(allocation->storageInfo.cloningOfPageTables); + EXPECT_EQ(0u, allocation->getGpuAddress()); + + memoryManager->freeGraphicsMemory(allocation); +} + +TEST_F(WddmMemoryManagerSimpleTest, givenAllocatePhysicalLocalDeviceMemoryWithMultiStorageThenLocalMemoryAllocationHasCorrectStorageInfoAndNoGpuAddress) { + auto executionEnvironment = platform()->peekExecutionEnvironment(); + memoryManager = std::make_unique(false, true, *executionEnvironment); + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize64k * 4; + allocData.flags.allocateMemory = true; + allocData.storageInfo.memoryBanks = 0b11; + allocData.storageInfo.pageTablesVisibility = 0x2; + allocData.storageInfo.cloningOfPageTables = false; + allocData.flags.flushL3 = true; + allocData.storageInfo.multiStorage = true; + + auto allocation = memoryManager->allocatePhysicalLocalDeviceMemory(allocData, status); + EXPECT_NE(nullptr, allocation); + + EXPECT_EQ(allocData.storageInfo.memoryBanks, allocation->storageInfo.memoryBanks); + EXPECT_EQ(allocData.storageInfo.pageTablesVisibility, allocation->storageInfo.pageTablesVisibility); + EXPECT_FALSE(allocation->storageInfo.cloningOfPageTables); + EXPECT_EQ(0u, allocation->getGpuAddress()); + + memoryManager->freeGraphicsMemory(allocation); +} + +TEST_F(WddmMemoryManagerSimpleTest, givenAllocatePhysicalLocalDeviceMemoryWithMultiBanksThenLocalMemoryAllocationHasCorrectStorageInfoAndNoGpuAddress) { + auto executionEnvironment = platform()->peekExecutionEnvironment(); + memoryManager = std::make_unique(false, true, *executionEnvironment); + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize64k * 4; + allocData.flags.allocateMemory = true; + allocData.storageInfo.memoryBanks = 0b11; + allocData.storageInfo.pageTablesVisibility = 0x2; + allocData.storageInfo.cloningOfPageTables = false; + allocData.flags.flushL3 = true; + allocData.storageInfo.multiStorage = false; + + auto allocation = memoryManager->allocatePhysicalLocalDeviceMemory(allocData, status); + EXPECT_NE(nullptr, allocation); + + EXPECT_EQ(allocData.storageInfo.memoryBanks, allocation->storageInfo.memoryBanks); + EXPECT_EQ(allocData.storageInfo.pageTablesVisibility, allocation->storageInfo.pageTablesVisibility); + EXPECT_FALSE(allocation->storageInfo.cloningOfPageTables); + EXPECT_EQ(0u, allocation->getGpuAddress()); + + memoryManager->freeGraphicsMemory(allocation); +} + TEST_F(WddmMemoryManagerTest, givenLocalMemoryAllocationWhenCpuPointerNotMeetRestrictionsThenDontReserveMemRangeForMap) { const bool localMemoryEnabled = true; auto executionEnvironment = platform()->peekExecutionEnvironment(); @@ -479,6 +574,25 @@ TEST_F(WddmMemoryManagerSimpleTest, givenSetAllocationPriorityFailureWhenMemoryI EXPECT_EQ(MemoryManager::AllocationStatus::Error, status); } +TEST_F(WddmMemoryManagerSimpleTest, givenSetAllocationPriorityFailureWhenMemoryIsAllocatedInLocalPhysicalMemoryThenNullptrIsReturned) { + const bool localMemoryEnabled = true; + auto executionEnvironment = platform()->peekExecutionEnvironment(); + memoryManager = std::make_unique(false, localMemoryEnabled, *executionEnvironment); + + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize; + allocData.flags.allocateMemory = true; + + wddm->callBaseSetAllocationPriority = false; + wddm->setAllocationPriorityResult.success = false; + + auto allocation = static_cast(memoryManager->allocatePhysicalLocalDeviceMemory(allocData, status)); + EXPECT_EQ(nullptr, allocation); + EXPECT_EQ(MemoryManager::AllocationStatus::Error, status); +} + class WddmMemoryManagerSimpleTestWithLocalMemory : public MockWddmMemoryManagerFixture, public ::testing::Test { public: void SetUp() override { diff --git a/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp index de090a394e..26b8e92160 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp @@ -206,6 +206,9 @@ class MockCreateWddmAllocationMemoryManager : public MockWddmMemoryManager { bool createWddmAllocation(WddmAllocation *allocation, void *requiredGpuPtr) override { return false; } + bool createPhysicalAllocation(WddmAllocation *allocation) override { + return false; + } }; TEST_F(WddmMemoryManagerSimpleTest, givenMemoryManagerWhenAllocateGraphicsMemoryFailedThenNullptrFromAllocateMemoryByKMDIsReturned) { @@ -216,6 +219,24 @@ TEST_F(WddmMemoryManagerSimpleTest, givenMemoryManagerWhenAllocateGraphicsMemory EXPECT_EQ(nullptr, allocation); } +TEST_F(WddmMemoryManagerSimpleTest, givenMemoryManagerWhenAllocateGraphicsMemoryFailedThenNullptrFromAllocatePhysicalDeviceMemoryIsReturned) { + memoryManager.reset(new MockCreateWddmAllocationMemoryManager(*executionEnvironment)); + AllocationData allocationData; + allocationData.size = MemoryConstants::pageSize; + MemoryManager::AllocationStatus status; + auto allocation = memoryManager->allocatePhysicalDeviceMemory(allocationData, status); + EXPECT_EQ(nullptr, allocation); +} + +TEST_F(WddmMemoryManagerSimpleTest, givenMemoryManagerWhenAllocateGraphicsMemoryFailedThenNullptrFromAllocatePhysicalLocalDeviceMemoryIsReturned) { + memoryManager.reset(new MockCreateWddmAllocationMemoryManager(*executionEnvironment)); + AllocationData allocationData; + allocationData.size = MemoryConstants::pageSize; + MemoryManager::AllocationStatus status; + auto allocation = memoryManager->allocatePhysicalLocalDeviceMemory(allocationData, status); + EXPECT_EQ(nullptr, allocation); +} + TEST_F(WddmMemoryManagerSimpleTest, givenMemoryManagerWith64KBPagesEnabledWhenAllocateGraphicsMemory64kbIsCalledThenMemoryPoolIsSystem64KBPages) { memoryManager.reset(new MockWddmMemoryManager(false, false, *executionEnvironment)); AllocationData allocationData; @@ -470,6 +491,45 @@ TEST_F(WddmMemoryManagerSimpleTest, GivenShareableEnabledAndHugeSizeWhenAskedToC memoryManager->freeGraphicsMemory(allocation); } +TEST_F(WddmMemoryManagerSimpleTest, GivenShareableEnabledAndSmallSizeWhenAskedToCreatePhysicalGraphicsAllocationThenValidAllocationIsReturned) { + memoryManager.reset(new MockWddmMemoryManager(false, false, *executionEnvironment)); + memoryManager->hugeGfxMemoryChunkSize = MemoryConstants::pageSize64k; + AllocationData allocationData; + allocationData.size = 4096u; + allocationData.flags.shareable = true; + MemoryManager::AllocationStatus status; + auto allocation = memoryManager->allocatePhysicalDeviceMemory(allocationData, status); + EXPECT_NE(nullptr, allocation); + memoryManager->freeGraphicsMemory(allocation); +} + +TEST_F(WddmMemoryManagerSimpleTest, GivenShareableEnabledAndHugeSizeWhenAskedToCreatePhysicalLocalGraphicsAllocationThenValidAllocationIsReturned) { + memoryManager.reset(new MockWddmMemoryManager(false, false, *executionEnvironment)); + memoryManager->hugeGfxMemoryChunkSize = MemoryConstants::pageSize64k; + AllocationData allocationData; + allocationData.size = 2ULL * MemoryConstants::pageSize64k; + allocationData.flags.shareable = true; + MemoryManager::AllocationStatus status; + auto allocation = memoryManager->allocatePhysicalLocalDeviceMemory(allocationData, status); + EXPECT_NE(nullptr, allocation); + memoryManager->freeGraphicsMemory(allocation); +} + +TEST_F(WddmMemoryManagerSimpleTest, GivenPhysicalMemoryAndVirtualMemoryThenMapSucceeds) { + memoryManager.reset(new MockWddmMemoryManager(false, false, *executionEnvironment)); + memoryManager->hugeGfxMemoryChunkSize = MemoryConstants::pageSize64k; + AllocationData allocationData; + allocationData.size = 2ULL * MemoryConstants::pageSize64k; + uint64_t gpuRange = 0x1234; + MemoryManager::AllocationStatus status; + auto allocation = memoryManager->allocatePhysicalLocalDeviceMemory(allocationData, status); + EXPECT_NE(nullptr, allocation); + auto res = memoryManager->mapPhysicalToVirtualMemory(allocation, gpuRange, allocationData.size); + EXPECT_TRUE(res); + memoryManager->unMapPhysicalToVirtualMemory(allocation, gpuRange, allocationData.size, osContext, 0u); + memoryManager->freeGraphicsMemory(allocation); +} + TEST_F(WddmMemoryManagerSimpleTest, givenZeroFenceValueOnSingleEngineRegisteredWhenHandleFenceCompletionIsCalledThenDoNotWaitOnCpu) { ASSERT_EQ(1u, memoryManager->getRegisteredEnginesCount()); @@ -2423,6 +2483,40 @@ TEST_F(WddmMemoryManagerSimpleTest, givenMultiHandleAllocationAndPreferredGpuVaI EXPECT_LT(lastRequiredAddress, memoryManager->getGfxPartition(0)->getHeapLimit(HeapIndex::HEAP_SVM)); } +TEST_F(WddmMemoryManagerSimpleTest, givenMultiHandleAllocationWhenCreatePhysicalAllocationIsCalledThenAllocationSuccess) { + if (memoryManager->isLimitedRange(0)) { + GTEST_SKIP(); + } + + uint32_t numGmms = 10; + MockWddmAllocation allocation(rootDeviceEnvironment->getGmmHelper(), numGmms); + allocation.setAllocationType(AllocationType::BUFFER); + allocation.storageInfo.multiStorage = true; + + wddm->callBaseMapGpuVa = true; + + memoryManager->createPhysicalAllocation(&allocation); + EXPECT_EQ(0ull, allocation.getGpuAddress()); + EXPECT_EQ(0u, wddm->mapGpuVirtualAddressResult.called); +} + +TEST_F(WddmMemoryManagerSimpleTest, givenMultiHandleAllocationWhenCreatePhysicalAllocationIsCalledThenFailureReturned) { + if (memoryManager->isLimitedRange(0)) { + GTEST_SKIP(); + } + + uint32_t numGmms = 10; + MockWddmAllocation allocation(rootDeviceEnvironment->getGmmHelper(), numGmms); + allocation.setAllocationType(AllocationType::BUFFER); + allocation.storageInfo.multiStorage = true; + + wddm->callBaseMapGpuVa = true; + + wddm->failCreateAllocation = true; + auto ret = memoryManager->createPhysicalAllocation(&allocation); + EXPECT_FALSE(ret); +} + TEST_F(WddmMemoryManagerSimpleTest, givenSvmCpuAllocationWhenSizeAndAlignmentProvidedThenAllocateMemoryReserveGpuVa) { if (memoryManager->isLimitedGPU(0)) { GTEST_SKIP(); diff --git a/shared/source/memory_manager/memory_manager.cpp b/shared/source/memory_manager/memory_manager.cpp index 4789ed1e36..57fea243f7 100644 --- a/shared/source/memory_manager/memory_manager.cpp +++ b/shared/source/memory_manager/memory_manager.cpp @@ -536,6 +536,34 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, const Allo return true; } +GraphicsAllocation *MemoryManager::allocatePhysicalGraphicsMemory(const AllocationProperties &properties) { + AllocationData allocationData; + GraphicsAllocation *allocation = nullptr; + getAllocationData(allocationData, properties, nullptr, createStorageInfoFromProperties(properties)); + + AllocationStatus status = AllocationStatus::Error; + if (this->localMemorySupported[allocationData.rootDeviceIndex]) { + allocation = allocatePhysicalLocalDeviceMemory(allocationData, status); + if (allocation) { + getLocalMemoryUsageBankSelector(properties.allocationType, properties.rootDeviceIndex)->reserveOnBanks(allocationData.storageInfo.getMemoryBanks(), allocation->getUnderlyingBufferSize()); + status = this->registerLocalMemAlloc(allocation, properties.rootDeviceIndex); + } + } else { + allocation = allocatePhysicalDeviceMemory(allocationData, status); + } + if (allocation && status != AllocationStatus::Success) { + freeGraphicsMemory(allocation); + allocation = nullptr; + } + if (!allocation) { + return nullptr; + } + + fileLoggerInstance().logAllocation(allocation); + registerAllocationInOs(allocation); + return allocation; +} + GraphicsAllocation *MemoryManager::allocateGraphicsMemoryInPreferredPool(const AllocationProperties &properties, const void *hostPtr) { AllocationData allocationData; getAllocationData(allocationData, properties, hostPtr, createStorageInfoFromProperties(properties)); diff --git a/shared/source/memory_manager/memory_manager.h b/shared/source/memory_manager/memory_manager.h index 30dba5f3e8..0a172d09b9 100644 --- a/shared/source/memory_manager/memory_manager.h +++ b/shared/source/memory_manager/memory_manager.h @@ -58,10 +58,15 @@ struct AddressRange { struct VirtualMemoryReservation { AddressRange virtualAddressRange; MemoryFlags flags; - bool mapped; + struct PhysicalMemoryAllocation *mappedAllocation; uint32_t rootDeviceIndex; }; +struct PhysicalMemoryAllocation { + GraphicsAllocation *allocation; + Device *device; +}; + constexpr size_t paddingBufferSize = 2 * MemoryConstants::megaByte; namespace MemoryTransferHelper { @@ -99,6 +104,7 @@ class MemoryManager { GraphicsAllocation *allocateInternalGraphicsMemoryWithHostCopy(uint32_t rootDeviceIndex, DeviceBitfield bitField, const void *ptr, size_t size); MOCKABLE_VIRTUAL GraphicsAllocation *allocateGraphicsMemoryInPreferredPool(const AllocationProperties &properties, const void *hostPtr); + MOCKABLE_VIRTUAL GraphicsAllocation *allocatePhysicalGraphicsMemory(const AllocationProperties &properties); virtual bool verifyHandle(osHandle handle, uint32_t rootDeviceIndex, bool) { return true; } virtual bool isNTHandle(osHandle handle, uint32_t rootDeviceIndex) { return false; } @@ -257,6 +263,10 @@ class MemoryManager { [[nodiscard]] std::unique_lock lockKernelAllocationMap() { return std::unique_lock(this->kernelAllocationMutex); }; std::map &getVirtualMemoryReservationMap() { return this->virtualMemoryReservationMap; }; [[nodiscard]] std::unique_lock lockVirtualMemoryReservationMap() { return std::unique_lock(this->virtualMemoryReservationMapMutex); }; + std::map &getPhysicalMemoryAllocationMap() { return this->physicalMemoryAllocationMap; }; + [[nodiscard]] std::unique_lock lockPhysicalMemoryAllocationMap() { return std::unique_lock(this->physicalMemoryAllocationMapMutex); }; + virtual bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) = 0; + virtual void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) = 0; protected: bool getAllocationData(AllocationData &allocationData, const AllocationProperties &properties, const void *hostPtr, const StorageInfo &storageInfo); @@ -282,6 +292,8 @@ class MemoryManager { MOCKABLE_VIRTUAL GraphicsAllocation *allocateGraphicsMemoryForImage(const AllocationData &allocationData); virtual GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr gmm) = 0; virtual GraphicsAllocation *allocateMemoryByKMD(const AllocationData &allocationData) = 0; + virtual GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) = 0; + virtual GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) = 0; virtual void *lockResourceImpl(GraphicsAllocation &graphicsAllocation) = 0; virtual void unlockResourceImpl(GraphicsAllocation &graphicsAllocation) = 0; virtual void freeAssociatedResourceImpl(GraphicsAllocation &graphicsAllocation) { return unlockResourceImpl(graphicsAllocation); }; @@ -320,6 +332,8 @@ class MemoryManager { std::mutex kernelAllocationMutex; std::map virtualMemoryReservationMap; std::mutex virtualMemoryReservationMapMutex; + std::map physicalMemoryAllocationMap; + std::mutex physicalMemoryAllocationMapMutex; }; std::unique_ptr createDeferredDeleter(); diff --git a/shared/source/memory_manager/os_agnostic_memory_manager.cpp b/shared/source/memory_manager/os_agnostic_memory_manager.cpp index 32b1f51ffc..fee0ab34fb 100644 --- a/shared/source/memory_manager/os_agnostic_memory_manager.cpp +++ b/shared/source/memory_manager/os_agnostic_memory_manager.cpp @@ -372,6 +372,80 @@ void OsAgnosticMemoryManager::cleanOsHandles(OsHandleStorage &handleStorage, uin } } +void OsAgnosticMemoryManager::unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) { + physicalAllocation->setCpuPtrAndGpuAddress(nullptr, 0u); + physicalAllocation->setReservedAddressRange(nullptr, 0u); +} + +bool OsAgnosticMemoryManager::mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { + physicalAllocation->setCpuPtrAndGpuAddress(nullptr, gpuRange); + physicalAllocation->setReservedAddressRange(reinterpret_cast(gpuRange), bufferSize); + return true; +} + +GraphicsAllocation *OsAgnosticMemoryManager::allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) { + status = AllocationStatus::Error; + MemoryAllocation *allocation = nullptr; + auto numHandles = allocationData.storageInfo.getNumBanks(); + + std::unique_ptr gmm; + size_t sizeAligned64k = 0; + sizeAligned64k = alignUp(allocationData.size, MemoryConstants::pageSize64k); + + auto hwInfo = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHardwareInfo(); + gmm = std::make_unique(executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getGmmHelper(), + nullptr, + sizeAligned64k, + MemoryConstants::pageSize64k, + CacheSettingsHelper::getGmmUsageType(allocationData.type, allocationData.flags.uncacheable, *hwInfo), + allocationData.flags.preferCompressed, + allocationData.storageInfo, + true); + + auto systemMemory = allocateSystemMemory(sizeAligned64k, MemoryConstants::pageSize64k); + if (systemMemory) { + auto sizeOfHeapChunk = sizeAligned64k; + allocation = new MemoryAllocation(allocationData.rootDeviceIndex, numHandles, allocationData.type, systemMemory, systemMemory, + 0u, sizeAligned64k, counter, + MemoryPool::LocalMemory, false, allocationData.flags.flushL3, maxOsContextCount); + counter++; + allocation->setDefaultGmm(gmm.release()); + allocation->sizeToFree = sizeOfHeapChunk; + } + + if (allocation) { + allocation->overrideMemoryPool(MemoryPool::LocalMemory); + allocation->storageInfo = allocationData.storageInfo; + status = AllocationStatus::Success; + } + + return allocation; +} + +GraphicsAllocation *OsAgnosticMemoryManager::allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) { + status = AllocationStatus::Error; + auto hwInfo = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHardwareInfo(); + + auto gmm = std::make_unique(executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getGmmHelper(), allocationData.hostPtr, + allocationData.size, 0u, CacheSettingsHelper::getGmmUsageType(allocationData.type, allocationData.flags.uncacheable, *hwInfo), + allocationData.flags.preferCompressed, allocationData.storageInfo, true); + + GraphicsAllocation *alloc = nullptr; + + auto ptr = allocateSystemMemory(alignUp(allocationData.size, MemoryConstants::pageSize), MemoryConstants::pageSize); + if (ptr != nullptr) { + alloc = new MemoryAllocation(allocationData.rootDeviceIndex, allocationData.type, ptr, ptr, 0u, allocationData.size, + counter, MemoryPool::SystemCpuInaccessible, allocationData.flags.uncacheable, allocationData.flags.flushL3, maxOsContextCount); + counter++; + } + + if (alloc) { + alloc->setDefaultGmm(gmm.release()); + status = AllocationStatus::Success; + } + return alloc; +} + GraphicsAllocation *OsAgnosticMemoryManager::allocateMemoryByKMD(const AllocationData &allocationData) { auto hwInfo = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHardwareInfo(); diff --git a/shared/source/memory_manager/os_agnostic_memory_manager.h b/shared/source/memory_manager/os_agnostic_memory_manager.h index 0f116b091c..09db98df77 100644 --- a/shared/source/memory_manager/os_agnostic_memory_manager.h +++ b/shared/source/memory_manager/os_agnostic_memory_manager.h @@ -55,6 +55,10 @@ class OsAgnosticMemoryManager : public MemoryManager { GraphicsAllocation *allocateUSMHostGraphicsMemory(const AllocationData &allocationData) override; GraphicsAllocation *allocateGraphicsMemory64kb(const AllocationData &allocationData) override; GraphicsAllocation *allocateMemoryByKMD(const AllocationData &allocationData) override; + GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; + GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; + bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; + void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override; GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr gmm) override; GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const AllocationData &allocationData) override; diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index 7732c69813..4f3d298684 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -540,6 +540,63 @@ GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemory64kb(const Allocatio return nullptr; } +void DrmMemoryManager::unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) { + DrmAllocation *drmAllocation = reinterpret_cast(physicalAllocation); + auto bufferObjects = drmAllocation->getBOs(); + for (auto bufferObject : bufferObjects) { + if (bufferObject) { + auto address = bufferObject->peekAddress(); + uint64_t offset = address - gpuRange; + bufferObject->setAddress(offset); + } + } + physicalAllocation->setCpuPtrAndGpuAddress(nullptr, 0u); + physicalAllocation->setReservedAddressRange(nullptr, 0u); +} + +bool DrmMemoryManager::mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { + DrmAllocation *drmAllocation = reinterpret_cast(physicalAllocation); + auto bufferObjects = drmAllocation->getBOs(); + for (auto bufferObject : bufferObjects) { + if (bufferObject) { + auto offset = bufferObject->peekAddress(); + bufferObject->setAddress(gpuRange + offset); + } + } + physicalAllocation->setCpuPtrAndGpuAddress(nullptr, gpuRange); + physicalAllocation->setReservedAddressRange(reinterpret_cast(gpuRange), bufferSize); + return true; +} + +GraphicsAllocation *DrmMemoryManager::allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) { + auto hwInfo = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHardwareInfo(); + + StorageInfo systemMemoryStorageInfo = {}; + auto gmm = std::make_unique(executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getGmmHelper(), nullptr, + allocationData.size, 0u, CacheSettingsHelper::getGmmUsageType(allocationData.type, allocationData.flags.uncacheable, *hwInfo), false, systemMemoryStorageInfo, true); + size_t bufferSize = allocationData.size; + + GemCreate create{}; + create.size = bufferSize; + + auto &drm = getDrm(allocationData.rootDeviceIndex); + auto ioctlHelper = drm.getIoctlHelper(); + + [[maybe_unused]] auto ret = ioctlHelper->ioctl(DrmIoctl::GemCreate, &create); + DEBUG_BREAK_IF(ret != 0); + + auto patIndex = drm.getPatIndex(gmm.get(), allocationData.type, CacheRegion::Default, CachePolicy::WriteBack, false); + + std::unique_ptr bo(new BufferObject(&drm, patIndex, create.handle, bufferSize, maxOsContextCount)); + + auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, allocationData.type, bo.get(), nullptr, 0u, bufferSize, MemoryPool::SystemCpuInaccessible); + allocation->setDefaultGmm(gmm.release()); + + bo.release(); + status = AllocationStatus::Success; + return allocation; +} + GraphicsAllocation *DrmMemoryManager::allocateMemoryByKMD(const AllocationData &allocationData) { auto hwInfo = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHardwareInfo(); @@ -1461,6 +1518,66 @@ void DrmMemoryManager::cleanupBeforeReturn(const AllocationData &allocationData, gfxPartition->freeGpuAddressRange(gmmHelper->decanonize(gpuAddress), sizeAllocated); } +GraphicsAllocation *DrmMemoryManager::allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) { + auto hwInfo = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHardwareInfo(); + + std::unique_ptr gmm; + size_t sizeAligned = 0; + auto numHandles = allocationData.storageInfo.getNumBanks(); + bool createSingleHandle = 1 == numHandles; + auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); + + sizeAligned = alignUp(allocationData.size, MemoryConstants::pageSize64k); + if (createSingleHandle) { + gmm = std::make_unique(gmmHelper, + nullptr, + sizeAligned, + 0u, + CacheSettingsHelper::getGmmUsageType(allocationData.type, !!allocationData.flags.uncacheable, *hwInfo), + allocationData.flags.preferCompressed, + allocationData.storageInfo, + true); + } + + auto allocation = std::make_unique(allocationData.rootDeviceIndex, numHandles, allocationData.type, nullptr, nullptr, 0u, sizeAligned, MemoryPool::LocalMemory); + DrmAllocation *drmAllocation = static_cast(allocation.get()); + + if (createSingleHandle) { + allocation->setDefaultGmm(gmm.release()); + } else if (allocationData.storageInfo.multiStorage) { + createColouredGmms(gmmHelper, + *allocation, + allocationData.storageInfo, + allocationData.flags.preferCompressed); + } else { + fillGmmsInAllocation(gmmHelper, allocation.get(), allocationData.storageInfo); + } + allocation->storageInfo = allocationData.storageInfo; + allocation->setFlushL3Required(allocationData.flags.flushL3); + allocation->setUncacheable(allocationData.flags.uncacheable); + + if (!createDrmAllocation(&getDrm(allocationData.rootDeviceIndex), allocation.get(), 0u, maxOsContextCount)) { + for (auto handleId = 0u; handleId < allocationData.storageInfo.getNumBanks(); handleId++) { + delete allocation->getGmm(handleId); + } + status = AllocationStatus::Error; + return nullptr; + } + if (!allocation->setCacheRegion(&getDrm(allocationData.rootDeviceIndex), static_cast(allocationData.cacheRegion))) { + for (auto bo : drmAllocation->getBOs()) { + delete bo; + } + for (auto handleId = 0u; handleId < allocationData.storageInfo.getNumBanks(); handleId++) { + delete allocation->getGmm(handleId); + } + status = AllocationStatus::Error; + return nullptr; + } + + status = AllocationStatus::Success; + return allocation.release(); +} + GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) { status = AllocationStatus::RetryInNonDevicePool; if (!this->localMemorySupported[allocationData.rootDeviceIndex] || diff --git a/shared/source/os_interface/linux/drm_memory_manager.h b/shared/source/os_interface/linux/drm_memory_manager.h index eb27e5497c..e66099125a 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.h +++ b/shared/source/os_interface/linux/drm_memory_manager.h @@ -111,6 +111,10 @@ class DrmMemoryManager : public MemoryManager { GraphicsAllocation *allocateGraphicsMemoryWithHostPtr(const AllocationData &allocationData) override; GraphicsAllocation *allocateGraphicsMemory64kb(const AllocationData &allocationData) override; GraphicsAllocation *allocateMemoryByKMD(const AllocationData &allocationData) override; + GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; + GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; + bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; + void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override; GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr gmm) override; GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const AllocationData &allocationData) override; GraphicsAllocation *createSharedUnifiedMemoryAllocation(const AllocationData &allocationData); diff --git a/shared/source/os_interface/windows/wddm_allocation.h b/shared/source/os_interface/windows/wddm_allocation.h index 3f569cb271..3f6c82e442 100644 --- a/shared/source/os_interface/windows/wddm_allocation.h +++ b/shared/source/os_interface/windows/wddm_allocation.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2022 Intel Corporation + * Copyright (C) 2018-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -108,6 +108,8 @@ class WddmAllocation : public GraphicsAllocation { uint64_t reservedSizeForGpuVirtualAddress = 0u; uint32_t shareable = 0u; bool allocInFrontWindowPool = false; + bool physicalMemoryReservation = false; + bool mappedPhysicalMemoryReservation = false; protected: uint64_t ntSecureHandle = 0u; diff --git a/shared/source/os_interface/windows/wddm_memory_manager.cpp b/shared/source/os_interface/windows/wddm_memory_manager.cpp index 9e9dcd26ea..150a65a2bc 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.cpp +++ b/shared/source/os_interface/windows/wddm_memory_manager.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2022 Intel Corporation + * Copyright (C) 2018-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -71,6 +71,41 @@ WddmMemoryManager::WddmMemoryManager(ExecutionEnvironment &executionEnvironment) initialized = true; } +void WddmMemoryManager::unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) { + const auto wddm = static_cast(osContext)->getWddm(); + wddm->freeGpuVirtualAddress(gpuRange, bufferSize); + auto gfxPartition = getGfxPartition(rootDeviceIndex); + wddm->reserveGpuVirtualAddress(gpuRange, gfxPartition->getHeapMinimalAddress(HeapIndex::HEAP_STANDARD64KB), gfxPartition->getHeapLimit(HeapIndex::HEAP_STANDARD64KB), bufferSize); + physicalAllocation->setCpuPtrAndGpuAddress(nullptr, 0u); + physicalAllocation->setReservedAddressRange(nullptr, 0u); +} + +bool WddmMemoryManager::mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { + WddmAllocation *wddmAllocation = reinterpret_cast(physicalAllocation); + wddmAllocation->mappedPhysicalMemoryReservation = mapGpuVirtualAddress(wddmAllocation, reinterpret_cast(gpuRange)); + return wddmAllocation->mappedPhysicalMemoryReservation; +} + +GraphicsAllocation *WddmMemoryManager::allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) { + auto hwInfo = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHardwareInfo(); + + StorageInfo systemMemoryStorageInfo = {}; + auto gmm = std::make_unique(executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getGmmHelper(), nullptr, allocationData.size, 0u, + CacheSettingsHelper::getGmmUsageType(allocationData.type, !!allocationData.flags.uncacheable, *hwInfo), false, systemMemoryStorageInfo, true); + auto allocation = std::make_unique(allocationData.rootDeviceIndex, + 1u, // numGmms + allocationData.type, nullptr, 0, allocationData.size, nullptr, + MemoryPool::SystemCpuInaccessible, allocationData.flags.shareable, maxOsContextCount); + allocation->setDefaultGmm(gmm.get()); + if (!createPhysicalAllocation(allocation.get())) { + return nullptr; + } + + gmm.release(); + status = AllocationStatus::Success; + return allocation.release(); +} + GraphicsAllocation *WddmMemoryManager::allocateMemoryByKMD(const AllocationData &allocationData) { if (allocationData.size > getHugeGfxMemoryChunkSize(GfxMemoryAllocationMethod::AllocateByKmd)) { return allocateHugeGraphicsMemory(allocationData, false); @@ -642,6 +677,9 @@ bool WddmMemoryManager::validateAllocation(WddmAllocation *alloc) { if (alloc == nullptr) return false; auto size = alloc->getUnderlyingBufferSize(); + if (alloc->physicalMemoryReservation && !alloc->mappedPhysicalMemoryReservation) { + return true; + } if (alloc->getGpuAddress() == 0u || size == 0 || (alloc->getDefaultHandle() == 0 && alloc->fragmentsStorage.fragmentCount == 0)) return false; return true; @@ -758,6 +796,15 @@ AlignedMallocRestrictions *WddmMemoryManager::getAlignedMallocRestrictions() { return &mallocRestrictions; } +bool WddmMemoryManager::createPhysicalAllocation(WddmAllocation *allocation) { + auto status = createGpuAllocationsWithRetry(allocation); + if (!status) { + return false; + } + allocation->physicalMemoryReservation = true; + return true; +} + bool WddmMemoryManager::createWddmAllocation(WddmAllocation *allocation, void *requiredGpuPtr) { auto status = createGpuAllocationsWithRetry(allocation); if (!status) { @@ -1057,6 +1104,73 @@ uint32_t getPriorityForAllocation(AllocationType allocationType) { return DXGI_RESOURCE_PRIORITY_NORMAL; } +GraphicsAllocation *WddmMemoryManager::allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) { + auto &rootDeviceEnvironment = *executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]; + auto gmmHelper = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getGmmHelper(); + + std::unique_ptr gmm; + size_t sizeAligned = 0; + size_t alignment = 0; + auto numBanks = allocationData.storageInfo.getNumBanks(); + bool singleBankAllocation = numBanks == 1; + alignment = alignmentSelector.selectAlignment(allocationData.size).alignment; + sizeAligned = alignUp(allocationData.size, alignment); + + if (singleBankAllocation) { + gmm = std::make_unique(gmmHelper, + nullptr, + sizeAligned, + alignment, + CacheSettingsHelper::getGmmUsageType(allocationData.type, !!allocationData.flags.uncacheable, *rootDeviceEnvironment.getHardwareInfo()), + allocationData.flags.preferCompressed, + allocationData.storageInfo, + true); + } + + const auto chunkSize = alignDown(getHugeGfxMemoryChunkSize(GfxMemoryAllocationMethod::AllocateByKmd), alignment); + const size_t numGmms = (static_cast(sizeAligned) + chunkSize - 1) / chunkSize; + + auto wddmAllocation = std::make_unique(allocationData.rootDeviceIndex, singleBankAllocation ? numGmms : numBanks, + allocationData.type, nullptr, 0, sizeAligned, nullptr, MemoryPool::LocalMemory, allocationData.flags.shareable, maxOsContextCount); + if (singleBankAllocation) { + if (numGmms > 1) { + splitGmmsInAllocation(gmmHelper, wddmAllocation.get(), alignment, chunkSize, const_cast(allocationData.storageInfo)); + } else { + wddmAllocation->setDefaultGmm(gmm.release()); + } + } else if (allocationData.storageInfo.multiStorage) { + createColouredGmms(gmmHelper, *wddmAllocation, allocationData.storageInfo, allocationData.flags.preferCompressed); + } else { + fillGmmsInAllocation(gmmHelper, wddmAllocation.get(), allocationData.storageInfo); + } + wddmAllocation->storageInfo = allocationData.storageInfo; + wddmAllocation->setFlushL3Required(allocationData.flags.flushL3); + wddmAllocation->needsMakeResidentBeforeLock = true; + + auto &wddm = getWddm(allocationData.rootDeviceIndex); + + if (!createPhysicalAllocation(wddmAllocation.get())) { + for (auto handleId = 0u; handleId < allocationData.storageInfo.getNumBanks(); handleId++) { + delete wddmAllocation->getGmm(handleId); + } + status = AllocationStatus::Error; + return nullptr; + } + + auto handles = wddmAllocation->getHandles(); + + if (!wddm.setAllocationPriority(handles.data(), static_cast(handles.size()), getPriorityForAllocation(allocationData.type))) { + for (auto handleId = 0u; handleId < allocationData.storageInfo.getNumBanks(); handleId++) { + delete wddmAllocation->getGmm(handleId); + } + status = AllocationStatus::Error; + return nullptr; + } + + status = AllocationStatus::Success; + return wddmAllocation.release(); +} + GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) { status = AllocationStatus::RetryInNonDevicePool; diff --git a/shared/source/os_interface/windows/wddm_memory_manager.h b/shared/source/os_interface/windows/wddm_memory_manager.h index 8ec03f0852..12d011fc58 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.h +++ b/shared/source/os_interface/windows/wddm_memory_manager.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2022 Intel Corporation + * Copyright (C) 2018-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -88,6 +88,10 @@ class WddmMemoryManager : public MemoryManager { GraphicsAllocation *allocateGraphicsMemoryWithHostPtr(const AllocationData &allocationData) override; GraphicsAllocation *allocateGraphicsMemory64kb(const AllocationData &allocationData) override; GraphicsAllocation *allocateMemoryByKMD(const AllocationData &allocationData) override; + GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; + GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; + void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override; + bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr gmm) override; GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const AllocationData &allocationData) override { return nullptr; } @@ -108,6 +112,7 @@ class WddmMemoryManager : public MemoryManager { static bool validateAllocation(WddmAllocation *alloc); MOCKABLE_VIRTUAL bool createWddmAllocation(WddmAllocation *allocation, void *requiredGpuPtr); MOCKABLE_VIRTUAL bool mapGpuVirtualAddress(WddmAllocation *graphicsAllocation, const void *requiredGpuPtr); + MOCKABLE_VIRTUAL bool createPhysicalAllocation(WddmAllocation *allocation); bool mapGpuVaForOneHandleAllocation(WddmAllocation *graphicsAllocation, const void *requiredGpuPtr); bool mapMultiHandleAllocationWithRetry(WddmAllocation *allocation, const void *requiredGpuPtr); bool createGpuAllocationsWithRetry(WddmAllocation *graphicsAllocation); diff --git a/shared/test/common/mocks/linux/mock_drm_memory_manager.h b/shared/test/common/mocks/linux/mock_drm_memory_manager.h index ff52b40ece..401bbe0de1 100644 --- a/shared/test/common/mocks/linux/mock_drm_memory_manager.h +++ b/shared/test/common/mocks/linux/mock_drm_memory_manager.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2022 Intel Corporation + * Copyright (C) 2018-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -71,6 +71,8 @@ class TestedDrmMemoryManager : public MemoryManagerCreate { using DrmMemoryManager::allocateGraphicsMemoryWithAlignment; using DrmMemoryManager::allocateGraphicsMemoryWithHostPtr; using DrmMemoryManager::allocateMemoryByKMD; + using DrmMemoryManager::allocatePhysicalDeviceMemory; + using DrmMemoryManager::allocatePhysicalLocalDeviceMemory; using DrmMemoryManager::allocationTypeForCompletionFence; using DrmMemoryManager::allocUserptr; using DrmMemoryManager::createAllocWithAlignment; @@ -86,6 +88,7 @@ class TestedDrmMemoryManager : public MemoryManagerCreate { using DrmMemoryManager::handleFenceCompletion; using DrmMemoryManager::lockBufferObject; using DrmMemoryManager::lockResourceImpl; + using DrmMemoryManager::mapPhysicalToVirtualMemory; using DrmMemoryManager::memoryForPinBBs; using DrmMemoryManager::mmapFunction; using DrmMemoryManager::munmapFunction; @@ -99,6 +102,7 @@ class TestedDrmMemoryManager : public MemoryManagerCreate { using DrmMemoryManager::sharingBufferObjects; using DrmMemoryManager::supportsMultiStorageResources; using DrmMemoryManager::unlockBufferObject; + using DrmMemoryManager::unMapPhysicalToVirtualMemory; using DrmMemoryManager::waitOnCompletionFence; using MemoryManager::allocateGraphicsMemoryInDevicePool; using MemoryManager::heapAssigner; diff --git a/shared/test/common/mocks/mock_memory_manager.cpp b/shared/test/common/mocks/mock_memory_manager.cpp index b1c0799b8f..e9f82c9d5a 100644 --- a/shared/test/common/mocks/mock_memory_manager.cpp +++ b/shared/test/common/mocks/mock_memory_manager.cpp @@ -114,6 +114,14 @@ GraphicsAllocation *MockMemoryManager::allocateMemoryByKMD(const AllocationData return OsAgnosticMemoryManager::allocateMemoryByKMD(allocationData); } +GraphicsAllocation *MockMemoryManager::allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) { + return OsAgnosticMemoryManager::allocatePhysicalDeviceMemory(allocationData, status); +} + +GraphicsAllocation *MockMemoryManager::allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) { + return OsAgnosticMemoryManager::allocatePhysicalLocalDeviceMemory(allocationData, status); +} + GraphicsAllocation *MockMemoryManager::allocateGraphicsMemory64kb(const AllocationData &allocationData) { allocation64kbPageCreated = true; preferCompressedFlagPassed = forceCompressed ? true : allocationData.flags.preferCompressed; diff --git a/shared/test/common/mocks/mock_memory_manager.h b/shared/test/common/mocks/mock_memory_manager.h index 6e8e74de70..23674ea699 100644 --- a/shared/test/common/mocks/mock_memory_manager.h +++ b/shared/test/common/mocks/mock_memory_manager.h @@ -70,6 +70,8 @@ class MockMemoryManager : public MemoryManagerCreate { void overrideAsyncDeleterFlag(bool newValue); GraphicsAllocation *allocateGraphicsMemoryForImage(const AllocationData &allocationData) override; GraphicsAllocation *allocateMemoryByKMD(const AllocationData &allocationData) override; + GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; + GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; int redundancyRatio = 1; GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override; @@ -198,6 +200,13 @@ class MockMemoryManager : public MemoryManagerCreate { } } + bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { + if (failMapPhysicalToVirtualMemory) { + return false; + } + return OsAgnosticMemoryManager::mapPhysicalToVirtualMemory(physicalAllocation, gpuRange, bufferSize); + }; + uint32_t copyMemoryToAllocationBanksCalled = 0u; uint32_t populateOsHandlesCalled = 0u; uint32_t allocateGraphicsMemoryForNonSvmHostPtrCalled = 0u; @@ -244,6 +253,7 @@ class MockMemoryManager : public MemoryManagerCreate { bool returnFakeAllocation = false; bool callBasePopulateOsHandles = true; bool callBaseAllocateGraphicsMemoryForNonSvmHostPtr = true; + bool failMapPhysicalToVirtualMemory = false; std::unique_ptr mockExecutionEnvironment; DeviceBitfield recentlyPassedDeviceBitfield{}; std::unique_ptr waitAllocations = nullptr; diff --git a/shared/test/common/mocks/mock_wddm.cpp b/shared/test/common/mocks/mock_wddm.cpp index 0d428b11e5..19645cf430 100644 --- a/shared/test/common/mocks/mock_wddm.cpp +++ b/shared/test/common/mocks/mock_wddm.cpp @@ -109,6 +109,9 @@ NTSTATUS WddmMock::createAllocation(WddmAllocation *wddmAllocation) { } NTSTATUS WddmMock::createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKMT_HANDLE &outHandle, D3DKMT_HANDLE &outResourceHandle, uint64_t *outSharedHandle) { createAllocationResult.called++; + if (failCreateAllocation) { + return STATUS_NO_MEMORY; + } if (callBaseDestroyAllocations) { createAllocationStatus = Wddm::createAllocation(alignedCpuPtr, gmm, outHandle, outResourceHandle, outSharedHandle); createAllocationResult.success = createAllocationStatus == STATUS_SUCCESS; diff --git a/shared/test/common/mocks/mock_wddm.h b/shared/test/common/mocks/mock_wddm.h index b63afd5a1a..fe05ab517f 100644 --- a/shared/test/common/mocks/mock_wddm.h +++ b/shared/test/common/mocks/mock_wddm.h @@ -186,5 +186,6 @@ class WddmMock : public Wddm { bool callBaseSetAllocationPriority = true; bool callBaseWaitFromCpu = true; bool failReserveGpuVirtualAddress = false; + bool failCreateAllocation = false; }; } // namespace NEO diff --git a/shared/test/common/os_interface/windows/mock_wddm_memory_manager.h b/shared/test/common/os_interface/windows/mock_wddm_memory_manager.h index 3e92fce6f0..ab294ecf5c 100644 --- a/shared/test/common/os_interface/windows/mock_wddm_memory_manager.h +++ b/shared/test/common/os_interface/windows/mock_wddm_memory_manager.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2022 Intel Corporation + * Copyright (C) 2018-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -23,12 +23,17 @@ class MockWddmMemoryManager : public MemoryManagerCreate { using BaseClass::allocateGraphicsMemoryWithGpuVa; using BaseClass::allocateGraphicsMemoryWithProperties; using BaseClass::allocateMemoryByKMD; + using BaseClass::allocatePhysicalDeviceMemory; + using BaseClass::allocatePhysicalLocalDeviceMemory; using BaseClass::createGraphicsAllocation; + using BaseClass::createPhysicalAllocation; using BaseClass::createWddmAllocation; using BaseClass::getWddm; using BaseClass::gfxPartitions; using BaseClass::localMemorySupported; + using BaseClass::mapPhysicalToVirtualMemory; using BaseClass::supportsMultiStorageResources; + using BaseClass::unMapPhysicalToVirtualMemory; using MemoryManagerCreate::MemoryManagerCreate; using BaseClass::getHugeGfxMemoryChunkSize; diff --git a/shared/test/common/os_interface/windows/wddm_fixture.h b/shared/test/common/os_interface/windows/wddm_fixture.h index d71f53d081..97626e64e1 100644 --- a/shared/test/common/os_interface/windows/wddm_fixture.h +++ b/shared/test/common/os_interface/windows/wddm_fixture.h @@ -28,6 +28,7 @@ #include "shared/test/common/mocks/windows/mock_gdi_interface.h" #include "shared/test/common/mocks/windows/mock_gmm_memory_base.h" #include "shared/test/common/os_interface/windows/gdi_dll_fixture.h" +#include "shared/test/common/os_interface/windows/mock_wddm_memory_manager.h" #include "shared/test/common/test_macros/hw_test.h" namespace NEO { @@ -46,6 +47,8 @@ struct WddmFixture : public Test { osInterface = rootDeviceEnvironment->osInterface.get(); auto preemptionMode = PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo); wddm->init(); + executionEnvironment->initializeMemoryManager(); + memoryManager = std::make_unique(*executionEnvironment); auto hwInfo = rootDeviceEnvironment->getHardwareInfo(); auto &gfxCoreHelper = rootDeviceEnvironment->getHelper(); auto engine = gfxCoreHelper.getGpgpuEngineInstances(*hwInfo)[0]; @@ -58,6 +61,7 @@ struct WddmFixture : public Test { OSInterface *osInterface; RootDeviceEnvironment *rootDeviceEnvironment = nullptr; std::unique_ptr osContext; + std::unique_ptr memoryManager; MockGdi *gdi = nullptr; MockWddmResidentAllocationsContainer *mockTemporaryResources; diff --git a/shared/test/unit_test/device/neo_device_tests.cpp b/shared/test/unit_test/device/neo_device_tests.cpp index 99a0dfe26b..dc75f399e8 100644 --- a/shared/test/unit_test/device/neo_device_tests.cpp +++ b/shared/test/unit_test/device/neo_device_tests.cpp @@ -415,6 +415,10 @@ TEST_F(DeviceGetCapsTest, givenFlagEnabled64kbPagesWhenCallConstructorMemoryMana GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData, bool useLocalMemory) override { return nullptr; }; GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const AllocationData &allocationData) override { return nullptr; }; + GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; + bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr gmm) override { return nullptr; }; GraphicsAllocation *allocateMemoryByKMD(const AllocationData &allocationData) override { return nullptr; }; diff --git a/shared/test/unit_test/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp b/shared/test/unit_test/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp index 95c79a1148..2703324b88 100644 --- a/shared/test/unit_test/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp +++ b/shared/test/unit_test/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp @@ -331,6 +331,138 @@ TEST(BaseMemoryManagerTest, givenDebugVariableSetWhenCompressedBufferIsCreatedTh memoryManager.freeGraphicsMemory(allocationBufferCompressed); } +TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsMemoryThenPhysicalAllocationReturned) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + executionEnvironment.initGmm(); + MemoryManagerCreate memoryManager(false, true, executionEnvironment); + + AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::BUFFER, mockDeviceBitfield); + + auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); + EXPECT_NE(nullptr, allocationBuffer); + memoryManager.freeGraphicsMemory(allocationBuffer); +} + +class MockMemoryManagerLocalMemory : public OsAgnosticMemoryManager { + public: + using OsAgnosticMemoryManager::localMemorySupported; + using OsAgnosticMemoryManager::mapPhysicalToVirtualMemory; + using OsAgnosticMemoryManager::unMapPhysicalToVirtualMemory; + MockMemoryManagerLocalMemory(ExecutionEnvironment &executionEnvironment) : OsAgnosticMemoryManager(executionEnvironment) {} +}; + +TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsMemoryWithoutLocalMemoryThenPhysicalAllocationReturned) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + executionEnvironment.initGmm(); + MemoryManagerCreate memoryManager(false, true, executionEnvironment); + memoryManager.localMemorySupported[0] = 0; + + AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::BUFFER, mockDeviceBitfield); + + auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); + EXPECT_NE(nullptr, allocationBuffer); + memoryManager.freeGraphicsMemory(allocationBuffer); +} + +class MockMemoryManagerNoLocalMemoryFail : public OsAgnosticMemoryManager { + public: + using OsAgnosticMemoryManager::localMemorySupported; + MockMemoryManagerNoLocalMemoryFail(ExecutionEnvironment &executionEnvironment) : OsAgnosticMemoryManager(executionEnvironment) {} + void *allocateSystemMemory(size_t size, size_t alignment) override { + return nullptr; + } +}; + +TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsMemoryWithoutLocalMemoryThenNullptrReturned) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + executionEnvironment.initGmm(); + MemoryManagerCreate memoryManager(false, true, executionEnvironment); + memoryManager.localMemorySupported[0] = 0; + + AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::BUFFER, mockDeviceBitfield); + + auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); + EXPECT_EQ(nullptr, allocationBuffer); +} + +TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsMemoryWithLocalMemoryThenNullptrReturned) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + executionEnvironment.initGmm(); + MemoryManagerCreate memoryManager(false, true, executionEnvironment); + memoryManager.localMemorySupported[0] = 1; + + AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::BUFFER, mockDeviceBitfield); + + auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); + EXPECT_EQ(nullptr, allocationBuffer); +} + +class MockAgnosticMemoryManager : public OsAgnosticMemoryManager { + public: + using OsAgnosticMemoryManager::mapPhysicalToVirtualMemory; + using OsAgnosticMemoryManager::unMapPhysicalToVirtualMemory; + MockAgnosticMemoryManager(ExecutionEnvironment &executionEnvironment) : OsAgnosticMemoryManager(executionEnvironment) {} +}; + +TEST(BaseMemoryManagerTest, givenCalltoMapAndUnMapThenVirtialAddressSetUnSetOnPhysicalMemoryReturned) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + executionEnvironment.initGmm(); + MemoryManagerCreate memoryManager(false, true, executionEnvironment); + + AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::BUFFER, mockDeviceBitfield); + + auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); + EXPECT_NE(nullptr, allocationBuffer); + uint64_t gpuAddress = 0x1234; + size_t size = 4096; + memoryManager.mapPhysicalToVirtualMemory(allocationBuffer, gpuAddress, size); + EXPECT_EQ(gpuAddress, allocationBuffer->getGpuAddress()); + memoryManager.unMapPhysicalToVirtualMemory(allocationBuffer, gpuAddress, size, nullptr, 0u); + EXPECT_NE(gpuAddress, allocationBuffer->getGpuAddress()); + memoryManager.freeGraphicsMemory(allocationBuffer); +} + +class FailingMemoryManager : public OsAgnosticMemoryManager { + public: + using OsAgnosticMemoryManager::localMemorySupported; + FailingMemoryManager(ExecutionEnvironment &executionEnvironment) : OsAgnosticMemoryManager(executionEnvironment) {} + + GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { + if (failAllocate) { + return nullptr; + } + return OsAgnosticMemoryManager::allocatePhysicalLocalDeviceMemory(allocationData, status); + }; + AllocationStatus registerLocalMemAlloc(GraphicsAllocation *allocation, uint32_t rootDeviceIndex) override { return AllocationStatus::Error; }; + + bool failAllocate = false; +}; + +TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsMemoryWithFailedRegisterLocalMemAllocThenNullptrReturned) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + executionEnvironment.initGmm(); + MemoryManagerCreate memoryManager(false, true, executionEnvironment); + memoryManager.localMemorySupported[0] = 1; + + AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::BUFFER, mockDeviceBitfield); + + auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); + EXPECT_EQ(nullptr, allocationBuffer); +} + +TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsMemoryWithFailedLocalMemAllocThenNullptrReturned) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + executionEnvironment.initGmm(); + MemoryManagerCreate memoryManager(false, true, executionEnvironment); + memoryManager.localMemorySupported[0] = 1; + memoryManager.failAllocate = true; + + AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::BUFFER, mockDeviceBitfield); + + auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); + EXPECT_EQ(nullptr, allocationBuffer); +} + TEST(BaseMemoryManagerTest, givenMemoryManagerWithForced32BitsWhenSystemMemoryIsNotSetAnd32BitAllowedThenAllocateInDevicePoolReturnsRetryInNonDevicePool) { MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); MockMemoryManagerBaseImplementationOfDevicePool memoryManager(false, true, executionEnvironment); diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index 77576bc953..cf89e176f1 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -236,6 +236,21 @@ TEST_F(DrmMemoryManagerTest, GivenGraphicsAllocationWhenAddAndRemoveAllocationTo EXPECT_EQ(fragment, nullptr); } +TEST_F(DrmMemoryManagerTest, GivenAllocatePhysicalDeviceMemoryThenSuccessReturnedAndNoVirtualMemoryAssigned) { + mock->ioctl_expected.gemWait = 1; + mock->ioctl_expected.gemCreate = 1; + mock->ioctl_expected.gemClose = 1; + + allocationData.size = MemoryConstants::pageSize; + allocationData.flags.shareable = true; + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + auto allocation = memoryManager->allocatePhysicalDeviceMemory(allocationData, status); + EXPECT_EQ(status, MemoryManager::AllocationStatus::Success); + EXPECT_NE(nullptr, allocation); + EXPECT_EQ(0u, allocation->getGpuAddress()); + memoryManager->freeGraphicsMemory(allocation); +} + TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDrmMemoryManagerWhenGpuAddressReservationIsAttemptedAtIndex1ThenAddressFromGfxPartitionIsUsed) { auto memoryManager = std::make_unique(false, true, false, *executionEnvironment); RootDeviceIndicesContainer rootDevices; @@ -4275,6 +4290,24 @@ TEST(DrmMemoryManagerSimpleTest, givenDrmMemoryManagerWhenAllocateInDevicePoolIs EXPECT_EQ(MemoryManager::AllocationStatus::RetryInNonDevicePool, status); } +TEST(DrmMemoryManagerSimpleTest, givenDrmMemoryManagerWhenAllocateInLocalDeviceMemoryIsCalledThenNullptrAndStatusRetryIsReturned) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + executionEnvironment.rootDeviceEnvironments[0]->osInterface = std::make_unique(); + auto drm = Drm::create(nullptr, *executionEnvironment.rootDeviceEnvironments[0]); + executionEnvironment.rootDeviceEnvironments[0]->osInterface->setDriverModel(std::unique_ptr(drm)); + executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(*drm, 0u); + TestedDrmMemoryManager memoryManager(executionEnvironment); + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Success; + AllocationData allocData; + allocData.size = MemoryConstants::pageSize; + allocData.flags.useSystemMemory = true; + allocData.flags.allocateMemory = true; + + auto allocation = memoryManager.allocatePhysicalLocalDeviceMemory(allocData, status); + EXPECT_EQ(nullptr, allocation); + EXPECT_EQ(MemoryManager::AllocationStatus::Error, status); +} + TEST(DrmMemoryManagerSimpleTest, givenDrmMemoryManagerWhenLockResourceIsCalledOnNullBufferObjectThenReturnNullPtr) { MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); executionEnvironment.rootDeviceEnvironments[0]->osInterface = std::make_unique(); @@ -4695,6 +4728,47 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenUseKmdMi memoryManager->freeGraphicsMemory(allocation); } +TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenUseKmdMigrationForBuffersWhenGraphicsAllocationInPhysicalLocalMemoryIsAllocatedForBufferWithSeveralMemoryBanksThenCreateGemObjectWithMultipleRegions) { + DebugManager.flags.UseKmdMigrationForBuffers.set(1); + + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Success; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize; + allocData.type = AllocationType::BUFFER; + allocData.rootDeviceIndex = rootDeviceIndex; + allocData.storageInfo.memoryBanks = 0b11; + + auto allocation = memoryManager->allocatePhysicalLocalDeviceMemory(allocData, status); + EXPECT_NE(nullptr, allocation); + EXPECT_EQ(MemoryManager::AllocationStatus::Success, status); + EXPECT_EQ(0u, allocation->getGpuAddress()); + + EXPECT_EQ(allocData.storageInfo.memoryBanks, static_cast(mock->getMemoryInfo())->banks); + + memoryManager->freeGraphicsMemory(allocation); +} + +TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenUAllocationInPhysicalLocalMemoryIsAllocatedWithNoCacheRegionThenFailureReturned) { + DebugManager.flags.UseKmdMigrationForBuffers.set(1); + + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Success; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize; + allocData.type = AllocationType::BUFFER; + allocData.rootDeviceIndex = rootDeviceIndex; + allocData.storageInfo.memoryBanks = 0b11; + allocData.cacheRegion = 0xFFFF; + + auto &drm = static_cast(memoryManager->getDrm(0)); + drm.cacheInfo.reset(nullptr); + + auto allocation = memoryManager->allocatePhysicalLocalDeviceMemory(allocData, status); + EXPECT_EQ(nullptr, allocation); + EXPECT_EQ(MemoryManager::AllocationStatus::Error, status); +} + TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenMemoryAllocationWithNoSetPairAndOneHandleAndCommandBufferTypeThenNoPairHandleIsPassed) { VariableBackup backupSetPairCallParent{&mock->getSetPairAvailableCall.callParent, false}; VariableBackup backupSetPairReturnValue{&mock->getSetPairAvailableCall.returnValue, true}; @@ -4716,6 +4790,24 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenMemoryAl memoryManager->freeGraphicsMemory(allocation); } +TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenPhysicalLocalMemoryAllocationThenSuccessAndNoVirtualMemoryAssigned) { + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Success; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize; + allocData.type = AllocationType::COMMAND_BUFFER; + allocData.rootDeviceIndex = rootDeviceIndex; + allocData.storageInfo.memoryBanks = 0b01; + + auto allocation = memoryManager->allocatePhysicalLocalDeviceMemory(allocData, status); + EXPECT_NE(nullptr, allocation); + EXPECT_EQ(MemoryManager::AllocationStatus::Success, status); + + EXPECT_EQ(0u, allocation->getGpuAddress()); + + memoryManager->freeGraphicsMemory(allocation); +} + TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenMemoryAllocationWithSetPairAndOneHandleThenThenNoPairHandleIsPassed) { VariableBackup backupSetPairCallParent{&mock->getSetPairAvailableCall.callParent, false}; VariableBackup backupSetPairReturnValue{&mock->getSetPairAvailableCall.returnValue, true}; @@ -4886,6 +4978,85 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenAllocati } } +TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenAllocationWithLargeBufferWhenAllocatingInLocalPhysicalMemoryOnAllMemoryBanksThenCreateFourBufferObjectsWithDifferentGpuVirtualAddressesAndPartialSizes) { + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Success; + AllocationData allocData; + uint64_t gpuAddress = 0x1234; + allocData.allFlags = 0; + allocData.size = 18 * MemoryConstants::pageSize64k; + allocData.flags.allocateMemory = true; + allocData.type = AllocationType::BUFFER; + allocData.storageInfo.memoryBanks = maxNBitValue(MemoryBanks::getBankForLocalMemory(3)); + allocData.storageInfo.multiStorage = true; + allocData.rootDeviceIndex = rootDeviceIndex; + + auto allocation = memoryManager->allocatePhysicalLocalDeviceMemory(allocData, status); + ASSERT_NE(nullptr, allocation); + EXPECT_EQ(MemoryManager::AllocationStatus::Success, status); + EXPECT_EQ(MemoryPool::LocalMemory, allocation->getMemoryPool()); + EXPECT_EQ(0u, allocation->getGpuAddress()); + EXPECT_EQ(EngineLimits::maxHandleCount, allocation->getNumGmms()); + memoryManager->mapPhysicalToVirtualMemory(allocation, gpuAddress, allocData.size); + EXPECT_EQ(gpuAddress, allocation->getGpuAddress()); + + auto drmAllocation = static_cast(allocation); + auto &bos = drmAllocation->getBOs(); + auto boAddress = drmAllocation->getGpuAddress(); + for (auto handleId = 0u; handleId < EngineLimits::maxHandleCount; handleId++) { + auto bo = bos[handleId]; + ASSERT_NE(nullptr, bo); + auto boSize = allocation->getGmm(handleId)->gmmResourceInfo->getSizeAllocation(); + EXPECT_EQ(boAddress, bo->peekAddress()); + EXPECT_EQ(boSize, bo->peekSize()); + EXPECT_EQ(boSize, handleId == 0 || handleId == 1 ? 5 * MemoryConstants::pageSize64k : 4 * MemoryConstants::pageSize64k); + boAddress += boSize; + } + auto osContext = device->getDefaultEngine().osContext; + memoryManager->unMapPhysicalToVirtualMemory(allocation, gpuAddress, allocData.size, osContext, 0u); + EXPECT_EQ(0u, allocation->getGpuAddress()); + memoryManager->freeGraphicsMemory(allocation); +} + +TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenAllocationWithKernelIsaWhenAllocationInLocalPhysicalMemoryAndDeviceBitfieldWithHolesThenCorrectAllocationCreated) { + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Success; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize; + allocData.flags.allocateMemory = true; + allocData.type = AllocationType::KERNEL_ISA; + allocData.storageInfo.memoryBanks = 0b1011; + allocData.storageInfo.multiStorage = false; + allocData.rootDeviceIndex = rootDeviceIndex; + uint64_t gpuAddress = 0x1234; + + auto kernelIsaAllocation = static_cast(memoryManager->allocatePhysicalLocalDeviceMemory(allocData, status)); + + EXPECT_NE(nullptr, kernelIsaAllocation); + + memoryManager->mapPhysicalToVirtualMemory(kernelIsaAllocation, gpuAddress, allocData.size); + + auto gpuAddressReserved = kernelIsaAllocation->getGpuAddress(); + auto &bos = kernelIsaAllocation->getBOs(); + + EXPECT_NE(nullptr, bos[0]); + EXPECT_EQ(gpuAddressReserved, bos[0]->peekAddress()); + EXPECT_NE(nullptr, bos[1]); + EXPECT_EQ(gpuAddressReserved, bos[1]->peekAddress()); + + EXPECT_EQ(nullptr, bos[2]); + + EXPECT_NE(nullptr, bos[3]); + EXPECT_EQ(gpuAddressReserved, bos[3]->peekAddress()); + + auto &storageInfo = kernelIsaAllocation->storageInfo; + EXPECT_EQ(0b1011u, storageInfo.memoryBanks.to_ulong()); + + auto osContext = device->getDefaultEngine().osContext; + memoryManager->unMapPhysicalToVirtualMemory(kernelIsaAllocation, gpuAddress, allocData.size, osContext, 0u); + + memoryManager->freeGraphicsMemory(kernelIsaAllocation); +} + TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenAllocationWithLargeBufferWhenAllocatingInDevicePoolOnAllMemoryBanksThenCreateFourBufferObjectsWithDifferentGpuVirtualAddressesAndPartialSizes) { MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Success; AllocationData allocData; diff --git a/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp b/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp index 8fe97c6c65..f8e8e8f001 100644 --- a/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp +++ b/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp @@ -77,6 +77,12 @@ struct MockGmmMemoryWddmLinux : NEO::GmmMemory { struct MockWddmLinuxMemoryManager : NEO::WddmMemoryManager { using WddmMemoryManager::allocate32BitGraphicsMemoryImpl; + using WddmMemoryManager::allocatePhysicalDeviceMemory; + using WddmMemoryManager::allocatePhysicalLocalDeviceMemory; + using WddmMemoryManager::createPhysicalAllocation; + using WddmMemoryManager::localMemorySupported; + using WddmMemoryManager::mapPhysicalToVirtualMemory; + using WddmMemoryManager::unMapPhysicalToVirtualMemory; using WddmMemoryManager::WddmMemoryManager; }; @@ -638,6 +644,41 @@ TEST_F(WddmLinuxTest, givenRequestFor32bitAllocationWithoutPreexistingHostPtrWhe memoryManager.freeGraphicsMemoryImpl(alloc); } +TEST_F(WddmLinuxTest, givenAllocatePhysicalDeviceMemoryThenAllocationReturned) { + osEnvironment->gdi->reserveGpuVirtualAddress = reserveDeviceAddressSpaceMock; + osEnvironment->gdi->createAllocation2 = createAllocation2Mock; + osEnvironment->gdi->mapGpuVirtualAddress = mapGpuVirtualAddressMock; + osEnvironment->gdi->lock2 = lock2Mock; + osEnvironment->gdi->destroyAllocation2 = destroyAllocations2Mock; + + MockWddmLinuxMemoryManager memoryManager{mockExecEnv}; + + NEO::AllocationData allocData = {}; + NEO::MemoryManager::AllocationStatus status = NEO::MemoryManager::AllocationStatus::Error; + allocData.size = 3U; + + auto alloc = memoryManager.allocatePhysicalDeviceMemory(allocData, status); + ASSERT_NE(nullptr, alloc); + memoryManager.freeGraphicsMemoryImpl(alloc); +} + +TEST_F(WddmLinuxTest, givenAllocatePhysicalLocalDeviceMemoryThenErrorReturned) { + osEnvironment->gdi->reserveGpuVirtualAddress = reserveDeviceAddressSpaceMock; + osEnvironment->gdi->createAllocation2 = createAllocation2Mock; + osEnvironment->gdi->mapGpuVirtualAddress = mapGpuVirtualAddressMock; + osEnvironment->gdi->lock2 = lock2Mock; + osEnvironment->gdi->destroyAllocation2 = destroyAllocations2Mock; + + MockWddmLinuxMemoryManager memoryManager{mockExecEnv}; + + NEO::AllocationData allocData = {}; + NEO::MemoryManager::AllocationStatus status = NEO::MemoryManager::AllocationStatus::Error; + allocData.size = 3U; + + auto alloc = memoryManager.allocatePhysicalLocalDeviceMemory(allocData, status); + EXPECT_EQ(nullptr, alloc); +} + TEST_F(WddmLinuxTest, whenCheckedIfResourcesCleanupCanBeSkippedAndDeviceIsAliveThenReturnsFalse) { osEnvironment->gdi->getDeviceState = getDeviceStateMock; gdiMockConfig.getDeviceStateClb.returnValue = STATUS_SUCCESS; diff --git a/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp index 1e238e47b9..dcd6ff8ede 100644 --- a/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 Intel Corporation + * Copyright (C) 2022-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -7,6 +7,7 @@ #include "shared/test/common/helpers/execution_environment_helper.h" #include "shared/test/common/mocks/mock_device.h" +#include "shared/test/common/mocks/mock_gmm.h" #include "shared/test/common/mocks/mock_memory_manager.h" #include "shared/test/common/os_interface/windows/mock_wddm_memory_manager.h" #include "shared/test/common/os_interface/windows/wddm_fixture.h"