diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.inl b/level_zero/core/source/cmdlist/cmdlist_hw.inl index 6f1392d15b..37ea875b23 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.inl +++ b/level_zero/core/source/cmdlist/cmdlist_hw.inl @@ -2406,7 +2406,7 @@ inline AlignedAllocationData CommandListCoreFamily::getAlignedAll for (const auto &mappedAllocationData : allocData->virtualReservationData->mappedAllocations) { // Add additional allocations to the residency container if the virtual reservation spans multiple allocations. if (buffer != mappedAllocationData.second->ptr) { - commandContainer.addToResidencyContainer(mappedAllocationData.second->mappedAllocation->allocation); + commandContainer.addToResidencyContainer(mappedAllocationData.second->mappedAllocation.allocation); } } } diff --git a/level_zero/core/source/context/context_imp.cpp b/level_zero/core/source/context/context_imp.cpp index 59d6c1fdea..85d0411e33 100644 --- a/level_zero/core/source/context/context_imp.cpp +++ b/level_zero/core/source/context/context_imp.cpp @@ -1175,32 +1175,53 @@ ze_result_t ContextImp::createPhysicalMem(ze_device_handle_t hDevice, ze_physical_mem_handle_t *phPhysicalMemory) { auto device = Device::fromHandle(hDevice); - auto neoDevice = device->getNEODevice(); - if ((getPageAlignedSizeRequired(desc->size, nullptr, nullptr) != desc->size)) { - return ZE_RESULT_ERROR_UNSUPPORTED_SIZE; + bool isPhysicalDeviceMem = true; + auto allocType = NEO::AllocationType::unknown; + NEO::Device *neoDevice = nullptr; + uint32_t rootDeviceIndex = 0; + NEO::DeviceBitfield deviceBitfield{}; + + if ((desc->flags & ZE_PHYSICAL_MEM_FLAG_ALLOCATE_ON_HOST) != 0) { + if (!isAligned(desc->size)) { + return ZE_RESULT_ERROR_UNSUPPORTED_SIZE; + } + isPhysicalDeviceMem = false; + allocType = NEO::AllocationType::bufferHostMemory; + rootDeviceIndex = this->rootDeviceIndices.at(0); + deviceBitfield = this->deviceBitfields.at(rootDeviceIndex); + } else { + if ((getPageAlignedSizeRequired(desc->size, nullptr, nullptr) != desc->size)) { + return ZE_RESULT_ERROR_UNSUPPORTED_SIZE; + } + neoDevice = device->getNEODevice(); + allocType = NEO::AllocationType::buffer; + rootDeviceIndex = neoDevice->getRootDeviceIndex(); + deviceBitfield = neoDevice->getDeviceBitfield(); } - NEO::AllocationProperties physicalDeviceMemoryProperties{neoDevice->getRootDeviceIndex(), - true, - desc->size, - NEO::AllocationType::buffer, - false, - false, - device->getNEODevice()->getDeviceBitfield()}; - physicalDeviceMemoryProperties.flags.isUSMDeviceAllocation = true; - physicalDeviceMemoryProperties.flags.shareable = 1; + NEO::AllocationProperties physicalMemoryProperties{rootDeviceIndex, + true, + desc->size, + allocType, + false, + false, + deviceBitfield}; + physicalMemoryProperties.flags.forceSystemMemory = !isPhysicalDeviceMem; + physicalMemoryProperties.flags.isUSMHostAllocation = !isPhysicalDeviceMem; + physicalMemoryProperties.flags.isUSMDeviceAllocation = isPhysicalDeviceMem; + physicalMemoryProperties.flags.shareable = 1; - NEO::GraphicsAllocation *physicalDeviceMemoryAllocation = this->driverHandle->getMemoryManager()->allocatePhysicalGraphicsMemory(physicalDeviceMemoryProperties); - if (!physicalDeviceMemoryAllocation) { + NEO::GraphicsAllocation *allocation = this->driverHandle->getMemoryManager()->allocatePhysicalGraphicsMemory(physicalMemoryProperties); + if (!allocation) { return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY; } NEO::PhysicalMemoryAllocation *physicalMemoryAllocation = new NEO::PhysicalMemoryAllocation; - physicalMemoryAllocation->allocation = physicalDeviceMemoryAllocation; + physicalMemoryAllocation->allocation = allocation; physicalMemoryAllocation->device = neoDevice; auto lock = this->driverHandle->getMemoryManager()->lockPhysicalMemoryAllocationMap(); - this->driverHandle->getMemoryManager()->getPhysicalMemoryAllocationMap().insert(std::pair(reinterpret_cast(physicalDeviceMemoryAllocation), physicalMemoryAllocation)); - *phPhysicalMemory = reinterpret_cast(physicalDeviceMemoryAllocation); + this->driverHandle->getMemoryManager()->getPhysicalMemoryAllocationMap().insert(std::pair(reinterpret_cast(allocation), physicalMemoryAllocation)); + *phPhysicalMemory = reinterpret_cast(allocation); return ZE_RESULT_SUCCESS; } @@ -1271,7 +1292,10 @@ ze_result_t ContextImp::mapVirtualMem(const void *ptr, return ZE_RESULT_ERROR_INVALID_ARGUMENT; } - if (this->driverHandle->getMemoryManager()->mapPhysicalToVirtualMemory(allocationNode->allocation, reinterpret_cast(ptr), size) == true) { + if (allocationNode->allocation->getAllocationType() == NEO::AllocationType::buffer) { + if (!this->driverHandle->getMemoryManager()->mapPhysicalDeviceMemoryToVirtualMemory(allocationNode->allocation, reinterpret_cast(ptr), size)) { + return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY; + } NEO::SvmAllocationData allocData(allocationNode->allocation->getRootDeviceIndex()); allocData.gpuAllocations.addAllocation(allocationNode->allocation); allocData.cpuAllocation = nullptr; @@ -1284,15 +1308,36 @@ ze_result_t ContextImp::mapVirtualMem(const void *ptr, NEO::MemoryMappedRange *mappedRange = new NEO::MemoryMappedRange; mappedRange->ptr = ptr; mappedRange->size = size; - mappedRange->mappedAllocation = allocationNode; + mappedRange->mappedAllocation = *allocationNode; virtualMemoryReservation->mappedAllocations.insert(std::pair(const_cast(ptr), mappedRange)); this->driverHandle->getSvmAllocsManager()->insertSVMAlloc(allocData); NEO::MemoryOperationsHandler *memoryOperationsIface = allocationNode->device->getRootDeviceEnvironment().memoryOperationsInterface.get(); auto success = memoryOperationsIface->makeResident(allocationNode->device, ArrayRef(&allocationNode->allocation, 1), false); ze_result_t res = changeMemoryOperationStatusToL0ResultType(success); return res; + } else { + RootDeviceIndicesContainer rootDeviceIndicesVector(this->rootDeviceIndices); + auto maxRootDeviceIndex = *std::max_element(rootDeviceIndicesVector.begin(), rootDeviceIndicesVector.end(), std::less()); + NEO::SvmAllocationData allocData(maxRootDeviceIndex); + if (!this->driverHandle->getMemoryManager()->mapPhysicalHostMemoryToVirtualMemory(rootDeviceIndices, allocData.gpuAllocations, allocationNode->allocation, castToUint64(ptr), size)) { + return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + allocData.cpuAllocation = nullptr; + allocData.device = nullptr; + allocData.size = size; + allocData.pageSizeForAlignment = MemoryConstants::pageSize2M; + allocData.setAllocId(++this->driverHandle->svmAllocsManager->allocationsCounter); + allocData.memoryType = InternalMemoryType::reservedHostMemory; + allocData.virtualReservationData = virtualMemoryReservation; + NEO::MemoryMappedRange *mappedRange = new NEO::MemoryMappedRange; + mappedRange->ptr = ptr; + mappedRange->size = size; + mappedRange->mappedAllocation = *allocationNode; + mappedRange->mappedAllocation.allocation = allocData.gpuAllocations.getGraphicsAllocation(allocationNode->allocation->getRootDeviceIndex()); + virtualMemoryReservation->mappedAllocations.insert(std::pair(const_cast(ptr), mappedRange)); + this->driverHandle->getSvmAllocsManager()->insertSVMAlloc(allocData); + return ZE_RESULT_SUCCESS; } - return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY; } ze_result_t ContextImp::unMapVirtualMem(const void *ptr, @@ -1305,13 +1350,19 @@ ze_result_t ContextImp::unMapVirtualMem(const void *ptr, std::map::iterator physicalMapIt; physicalMapIt = virtualMemoryReservation->mappedAllocations.find(const_cast(ptr)); if (physicalMapIt != virtualMemoryReservation->mappedAllocations.end()) { - NEO::PhysicalMemoryAllocation *physicalAllocation = physicalMapIt->second->mappedAllocation; + NEO::PhysicalMemoryAllocation *physicalAllocation = &physicalMapIt->second->mappedAllocation; NEO::SvmAllocationData *allocData = this->driverHandle->getSvmAllocsManager()->getSVMAlloc(reinterpret_cast(physicalAllocation->allocation->getGpuAddress())); - this->driverHandle->getSvmAllocsManager()->removeSVMAlloc(*allocData); - NEO::Device *device = physicalAllocation->device; - NEO::CommandStreamReceiver *csr = device->getDefaultEngine().commandStreamReceiver; - NEO::OsContext *osContext = &csr->getOsContext(); - this->driverHandle->getMemoryManager()->unMapPhysicalToVirtualMemory(physicalAllocation->allocation, reinterpret_cast(ptr), size, osContext, virtualMemoryReservation->rootDeviceIndex); + + if (physicalAllocation->allocation->getAllocationType() == NEO::AllocationType::buffer) { + this->driverHandle->getSvmAllocsManager()->removeSVMAlloc(*allocData); + NEO::OsContext *osContext = &physicalAllocation->device->getDefaultEngine().commandStreamReceiver->getOsContext(); + this->driverHandle->getMemoryManager()->unMapPhysicalDeviceMemoryFromVirtualMemory(physicalAllocation->allocation, castToUint64(ptr), size, osContext, virtualMemoryReservation->rootDeviceIndex); + } else { + auto gpuAllocations = allocData->gpuAllocations; + this->driverHandle->getSvmAllocsManager()->removeSVMAlloc(*allocData); + this->driverHandle->getMemoryManager()->unMapPhysicalHostMemoryFromVirtualMemory(gpuAllocations, physicalAllocation->allocation, castToUint64(ptr), size); + } + delete physicalMapIt->second; virtualMemoryReservation->mappedAllocations.erase(physicalMapIt); } diff --git a/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h b/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h index a8f3a4cae5..9583148ab2 100644 --- a/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h +++ b/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h @@ -224,8 +224,11 @@ class MemoryManagerIpcMock : public NEO::MemoryManager { NEO::GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const NEO::AllocationData &allocationData) override { return nullptr; }; GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; - void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; - bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; + GraphicsAllocation *allocatePhysicalHostMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + void unMapPhysicalDeviceMemoryFromVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; + void unMapPhysicalHostMemoryFromVirtualMemory(MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return; }; + bool mapPhysicalDeviceMemoryToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; + bool mapPhysicalHostMemoryToVirtualMemory(RootDeviceIndicesContainer &rootDeviceIndices, MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; NEO::GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const NEO::AllocationData &allocationData, std::unique_ptr gmm) override { return nullptr; }; NEO::GraphicsAllocation *allocateMemoryByKMD(const NEO::AllocationData &allocationData) override { return nullptr; }; @@ -352,8 +355,11 @@ class MemoryManagerIpcImplicitScalingMock : public NEO::MemoryManager { NEO::GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const NEO::AllocationData &allocationData) override { return nullptr; }; GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; - void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; - bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; + GraphicsAllocation *allocatePhysicalHostMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + void unMapPhysicalDeviceMemoryFromVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; + void unMapPhysicalHostMemoryFromVirtualMemory(MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return; }; + bool mapPhysicalDeviceMemoryToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; + bool mapPhysicalHostMemoryToVirtualMemory(RootDeviceIndicesContainer &rootDeviceIndices, MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; NEO::GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const NEO::AllocationData &allocationData, std::unique_ptr gmm) override { return nullptr; }; NEO::GraphicsAllocation *allocateMemoryByKMD(const NEO::AllocationData &allocationData) override { return nullptr; }; diff --git a/level_zero/core/test/unit_tests/sources/context/test_context.cpp b/level_zero/core/test/unit_tests/sources/context/test_context.cpp index 446deefb2a..2e56edd04a 100644 --- a/level_zero/core/test/unit_tests/sources/context/test_context.cpp +++ b/level_zero/core/test/unit_tests/sources/context/test_context.cpp @@ -1133,11 +1133,15 @@ TEST_F(ContextTest, whenCallingPhysicalMemCreateWithInvalisSizeThenUnsupportedSi res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); EXPECT_EQ(ZE_RESULT_SUCCESS, res); - ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, 0, 10}; + ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, ZE_PHYSICAL_MEM_FLAG_ALLOCATE_ON_DEVICE, 10}; ze_physical_mem_handle_t mem = {}; res = contextImp->createPhysicalMem(device, &descMem, &mem); EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_SIZE, res); + descMem.flags = ZE_PHYSICAL_MEM_FLAG_ALLOCATE_ON_HOST; + res = contextImp->createPhysicalMem(device, &descMem, &mem); + EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_SIZE, res); + res = contextImp->destroy(); EXPECT_EQ(ZE_RESULT_SUCCESS, res); } @@ -1174,7 +1178,7 @@ TEST_F(ContextTest, whenCallingDestroyPhysicalMemWithIncorrectPointerThenMemoryN EXPECT_EQ(ZE_RESULT_SUCCESS, res); } -TEST_F(ContextTest, whenCallingMappingVirtualInterfacesThenSuccessIsReturned) { +TEST_F(ContextTest, whenCallingMappingVirtualInterfacesOnPhysicalDeviceMemoryThenSuccessIsReturned) { ze_context_handle_t hContext; ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; @@ -1196,7 +1200,73 @@ TEST_F(ContextTest, whenCallingMappingVirtualInterfacesThenSuccessIsReturned) { EXPECT_EQ(ZE_RESULT_SUCCESS, res); EXPECT_GT(static_cast(driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size()), 0); - ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, 0, pagesize}; + ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, ZE_PHYSICAL_MEM_FLAG_ALLOCATE_ON_DEVICE, pagesize}; + ze_physical_mem_handle_t mem = {}; + res = contextImp->createPhysicalMem(device, &descMem, &mem); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ze_memory_access_attribute_t access = {ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE}; + size_t offset = 0; + + std::vector memoryAccessFlags = { + ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE, ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY, + ZE_MEMORY_ACCESS_ATTRIBUTE_NONE}; + + for (auto accessFlags : memoryAccessFlags) { + res = contextImp->mapVirtualMem(ptr, pagesize, mem, offset, accessFlags); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + res = contextImp->unMapVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + } + + res = contextImp->mapVirtualMem(ptr, pagesize, mem, offset, access); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->setVirtualMemAccessAttribute(ptr, pagesize, access); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ze_memory_access_attribute_t outAccess = {}; + size_t outSize = 0; + res = contextImp->getVirtualMemAccessAttribute(ptr, pagesize, &outAccess, &outSize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_EQ(pagesize, outSize); + + res = contextImp->unMapVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->destroyPhysicalMem(mem); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->freeVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + +TEST_F(ContextTest, whenCallingMappingVirtualInterfacesOnPhysicalHostMemoryThenSuccessIsReturned) { + ze_context_handle_t hContext; + ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + + ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[0]->memoryOperationsInterface = + std::make_unique(); + + ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + + void *pStart = 0x0; + size_t size = 4096u; + void *ptr = nullptr; + size_t pagesize = 0u; + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_GT(static_cast(driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size()), 0); + + ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, ZE_PHYSICAL_MEM_FLAG_ALLOCATE_ON_HOST, pagesize}; ze_physical_mem_handle_t mem = {}; res = contextImp->createPhysicalMem(device, &descMem, &mem); EXPECT_EQ(ZE_RESULT_SUCCESS, res); @@ -1444,8 +1514,17 @@ class ReserveMemoryManagerMock : public NEO::MemoryManager { NEO::GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const NEO::AllocationData &allocationData) override { return nullptr; }; GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; - void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; - bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { + GraphicsAllocation *allocatePhysicalHostMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + void unMapPhysicalDeviceMemoryFromVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; + void unMapPhysicalHostMemoryFromVirtualMemory(MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return; }; + bool mapPhysicalDeviceMemoryToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { + if (failMapVirtualMemory) { + return false; + } else { + return true; + } + }; + bool mapPhysicalHostMemoryToVirtualMemory(RootDeviceIndicesContainer &rootDeviceIndices, MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { if (failMapVirtualMemory) { return false; } else { @@ -2037,7 +2116,7 @@ TEST_F(ContextTest, whenCallingPhysicalMemoryAllocateWhenOutOfMemoryThenOutofMem EXPECT_EQ(ZE_RESULT_SUCCESS, res); } -TEST_F(ContextTest, whenCallingMapVirtualMemoryWithFailedMapThenOutOfMemoryreturned) { +TEST_F(ContextTest, whenCallingMapVirtualDeviceMemoryWithFailedMapThenOutOfMemoryreturned) { ze_context_handle_t hContext; std::unique_ptr mockMemoryManager; ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; @@ -2065,7 +2144,7 @@ TEST_F(ContextTest, whenCallingMapVirtualMemoryWithFailedMapThenOutOfMemoryretur EXPECT_EQ(ZE_RESULT_SUCCESS, res); EXPECT_GT(static_cast(driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size()), 0); - ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, 0, pagesize}; + ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, ZE_PHYSICAL_MEM_FLAG_ALLOCATE_ON_DEVICE, pagesize}; ze_physical_mem_handle_t mem = {}; res = contextImp->createPhysicalMem(device, &descMem, &mem); @@ -2089,6 +2168,58 @@ TEST_F(ContextTest, whenCallingMapVirtualMemoryWithFailedMapThenOutOfMemoryretur EXPECT_EQ(ZE_RESULT_SUCCESS, res); } +TEST_F(ContextTest, whenCallingMapVirtualHostMemoryWithFailedMapThenOutOfMemoryreturned) { + ze_context_handle_t hContext; + std::unique_ptr mockMemoryManager; + ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; + + ze_result_t res = driverHandle->createContext(&desc, 0u, nullptr, &hContext); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[0]->memoryOperationsInterface = + std::make_unique(); + + ContextImp *contextImp = static_cast(L0::Context::fromHandle(hContext)); + + mockMemoryManager = std::make_unique(); + + auto memoryManager = driverHandle->getMemoryManager(); + driverHandle->setMemoryManager(mockMemoryManager.get()); + + void *pStart = 0x0; + size_t size = 4096u; + void *ptr = nullptr; + size_t pagesize = 0u; + res = contextImp->queryVirtualMemPageSize(device, size, &pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + res = contextImp->reserveVirtualMem(pStart, pagesize, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_GT(static_cast(driverHandle->getMemoryManager()->getVirtualMemoryReservationMap().size()), 0); + + ze_physical_mem_desc_t descMem = {ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC, nullptr, ZE_PHYSICAL_MEM_FLAG_ALLOCATE_ON_HOST, pagesize}; + ze_physical_mem_handle_t mem = {}; + + res = contextImp->createPhysicalMem(device, &descMem, &mem); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + ze_memory_access_attribute_t access = {ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE}; + size_t offset = 0; + mockMemoryManager->failMapPhysicalToVirtualMemory = true; + res = contextImp->mapVirtualMem(ptr, pagesize, mem, offset, access); + EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY, res); + + res = contextImp->destroyPhysicalMem(mem); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + res = contextImp->freeVirtualMem(ptr, pagesize); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + driverHandle->setMemoryManager(memoryManager); + + res = contextImp->destroy(); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); +} + TEST_F(ContextTest, whenCallingMapVirtualMemoryWithInvalidValuesThenFailureReturned) { ze_context_handle_t hContext; ze_context_desc_t desc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0}; diff --git a/level_zero/core/test/unit_tests/sources/event/test_event.cpp b/level_zero/core/test/unit_tests/sources/event/test_event.cpp index ee494e004e..4b04214b30 100644 --- a/level_zero/core/test/unit_tests/sources/event/test_event.cpp +++ b/level_zero/core/test/unit_tests/sources/event/test_event.cpp @@ -100,8 +100,11 @@ class MemoryManagerEventPoolFailMock : public NEO::MemoryManager { NEO::GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const NEO::AllocationData &allocationData) override { return nullptr; }; GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; - void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; - bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; + GraphicsAllocation *allocatePhysicalHostMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + void unMapPhysicalDeviceMemoryFromVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; + void unMapPhysicalHostMemoryFromVirtualMemory(MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return; }; + bool mapPhysicalDeviceMemoryToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; + bool mapPhysicalHostMemoryToVirtualMemory(RootDeviceIndicesContainer &rootDeviceIndices, MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; NEO::GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const NEO::AllocationData &allocationData, std::unique_ptr gmm) override { return nullptr; }; NEO::GraphicsAllocation *allocateMemoryByKMD(const NEO::AllocationData &allocationData) override { return nullptr; }; diff --git a/level_zero/core/test/unit_tests/sources/module/test_module.cpp b/level_zero/core/test/unit_tests/sources/module/test_module.cpp index 370d9f2461..34e398a1ce 100644 --- a/level_zero/core/test/unit_tests/sources/module/test_module.cpp +++ b/level_zero/core/test/unit_tests/sources/module/test_module.cpp @@ -2906,12 +2906,12 @@ HWTEST_F(MultiDeviceModuleSetArgBufferTest, auto svmAllocsManager = device->getDriverHandle()->getSvmAllocsManager(); auto virtualAlloc = svmAllocsManager->getSVMAlloc(ptr); - virtualAlloc->virtualReservationData->mappedAllocations.at(offsetAddress)->mappedAllocation->allocation->setSize((MemoryConstants::gigaByte * 4) - MemoryConstants::pageSize64k); + virtualAlloc->virtualReservationData->mappedAllocations.at(offsetAddress)->mappedAllocation.allocation->setSize((MemoryConstants::gigaByte * 4) - MemoryConstants::pageSize64k); L0::KernelImp *kernel = reinterpret_cast(Kernel::fromHandle(kernelHandle)); kernel->setArgBuffer(0, sizeof(ptr), &ptr); - virtualAlloc->virtualReservationData->mappedAllocations.at(offsetAddress)->mappedAllocation->allocation->setSize(size); + virtualAlloc->virtualReservationData->mappedAllocations.at(offsetAddress)->mappedAllocation.allocation->setSize(size); bool phys1Resident = false; bool phys2Resident = false; diff --git a/shared/source/memory_manager/memory_manager.cpp b/shared/source/memory_manager/memory_manager.cpp index db9f608937..4c86dc0444 100644 --- a/shared/source/memory_manager/memory_manager.cpp +++ b/shared/source/memory_manager/memory_manager.cpp @@ -655,14 +655,24 @@ GraphicsAllocation *MemoryManager::allocatePhysicalGraphicsMemory(const Allocati getAllocationData(allocationData, properties, nullptr, createStorageInfoFromProperties(properties)); AllocationStatus status = AllocationStatus::Error; - if (this->localMemorySupported[allocationData.rootDeviceIndex]) { - allocation = allocatePhysicalLocalDeviceMemory(allocationData, status); - if (allocation) { - getLocalMemoryUsageBankSelector(properties.allocationType, properties.rootDeviceIndex)->reserveOnBanks(allocationData.storageInfo.getMemoryBanks(), allocation->getUnderlyingBufferSize()); - status = this->registerLocalMemAlloc(allocation, properties.rootDeviceIndex); + if (allocationData.flags.isUSMDeviceMemory) { + if (this->localMemorySupported[allocationData.rootDeviceIndex]) { + allocation = allocatePhysicalLocalDeviceMemory(allocationData, status); + if (allocation) { + getLocalMemoryUsageBankSelector(properties.allocationType, properties.rootDeviceIndex)->reserveOnBanks(allocationData.storageInfo.getMemoryBanks(), allocation->getUnderlyingBufferSize()); + status = this->registerLocalMemAlloc(allocation, properties.rootDeviceIndex); + } + } else { + allocation = allocatePhysicalDeviceMemory(allocationData, status); + if (allocation) { + status = this->registerSysMemAlloc(allocation); + } } } else { - allocation = allocatePhysicalDeviceMemory(allocationData, status); + allocation = allocatePhysicalHostMemory(allocationData, status); + if (allocation) { + status = this->registerSysMemAlloc(allocation); + } } if (allocation && status != AllocationStatus::Success) { freeGraphicsMemory(allocation); diff --git a/shared/source/memory_manager/memory_manager.h b/shared/source/memory_manager/memory_manager.h index ede69589f2..14824a5799 100644 --- a/shared/source/memory_manager/memory_manager.h +++ b/shared/source/memory_manager/memory_manager.h @@ -59,10 +59,15 @@ struct AddressRange { size_t size; }; +struct PhysicalMemoryAllocation { + GraphicsAllocation *allocation; + Device *device; +}; + struct MemoryMappedRange { const void *ptr; size_t size; - struct PhysicalMemoryAllocation *mappedAllocation; + PhysicalMemoryAllocation mappedAllocation; }; struct VirtualMemoryReservation { @@ -76,11 +81,6 @@ struct VirtualMemoryReservation { size_t reservationTotalSize; }; -struct PhysicalMemoryAllocation { - GraphicsAllocation *allocation; - Device *device; -}; - constexpr size_t paddingBufferSize = 2 * MemoryConstants::megaByte; namespace MemoryTransferHelper { @@ -302,8 +302,10 @@ class MemoryManager { [[nodiscard]] std::unique_lock lockVirtualMemoryReservationMap() { return std::unique_lock(this->virtualMemoryReservationMapMutex); }; std::map &getPhysicalMemoryAllocationMap() { return this->physicalMemoryAllocationMap; }; [[nodiscard]] std::unique_lock lockPhysicalMemoryAllocationMap() { return std::unique_lock(this->physicalMemoryAllocationMapMutex); }; - virtual bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) = 0; - virtual void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) = 0; + virtual bool mapPhysicalDeviceMemoryToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) = 0; + virtual bool mapPhysicalHostMemoryToVirtualMemory(RootDeviceIndicesContainer &rootDeviceIndices, MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) = 0; + virtual void unMapPhysicalDeviceMemoryFromVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) = 0; + virtual void unMapPhysicalHostMemoryFromVirtualMemory(MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) = 0; bool allocateBindlessSlot(GraphicsAllocation *allocation); static uint64_t adjustToggleBitFlagForGpuVa(AllocationType inputAllocationType, uint64_t gpuAddress); virtual bool allocateInterrupt(uint32_t &outHandle, uint32_t rootDeviceIndex) { return false; } @@ -350,6 +352,7 @@ class MemoryManager { virtual GraphicsAllocation *allocateMemoryByKMD(const AllocationData &allocationData) = 0; virtual GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) = 0; virtual GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) = 0; + virtual GraphicsAllocation *allocatePhysicalHostMemory(const AllocationData &allocationData, AllocationStatus &status) = 0; virtual void *lockResourceImpl(GraphicsAllocation &graphicsAllocation) = 0; virtual void unlockResourceImpl(GraphicsAllocation &graphicsAllocation) = 0; virtual void freeAssociatedResourceImpl(GraphicsAllocation &graphicsAllocation) { return unlockResourceImpl(graphicsAllocation); }; diff --git a/shared/source/memory_manager/os_agnostic_memory_manager.cpp b/shared/source/memory_manager/os_agnostic_memory_manager.cpp index d178c4b377..e3856135dd 100644 --- a/shared/source/memory_manager/os_agnostic_memory_manager.cpp +++ b/shared/source/memory_manager/os_agnostic_memory_manager.cpp @@ -25,6 +25,7 @@ #include "shared/source/memory_manager/gfx_partition.h" #include "shared/source/memory_manager/host_ptr_manager.h" #include "shared/source/memory_manager/memory_allocation.h" +#include "shared/source/memory_manager/multi_graphics_allocation.h" #include "shared/source/memory_manager/residency.h" #include "shared/source/os_interface/os_context.h" #include "shared/source/os_interface/product_helper.h" @@ -416,17 +417,36 @@ void OsAgnosticMemoryManager::cleanOsHandles(OsHandleStorage &handleStorage, uin } } -void OsAgnosticMemoryManager::unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) { +void OsAgnosticMemoryManager::unMapPhysicalDeviceMemoryFromVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) { physicalAllocation->setGpuPtr(0u); physicalAllocation->setReservedAddressRange(nullptr, 0u); } -bool OsAgnosticMemoryManager::mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { +void OsAgnosticMemoryManager::unMapPhysicalHostMemoryFromVirtualMemory(MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { + for (uint32_t i = 0; i < static_cast(multiGraphicsAllocation.getGraphicsAllocations().size()); i++) { + delete multiGraphicsAllocation.getGraphicsAllocation(i); + multiGraphicsAllocation.removeAllocation(i); + } +} + +bool OsAgnosticMemoryManager::mapPhysicalDeviceMemoryToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { physicalAllocation->setGpuPtr(gpuRange); physicalAllocation->setReservedAddressRange(reinterpret_cast(gpuRange), bufferSize); return true; } +bool OsAgnosticMemoryManager::mapPhysicalHostMemoryToVirtualMemory(RootDeviceIndicesContainer &rootDeviceIndices, MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { + for (size_t i = 0; i < rootDeviceIndices.size(); i++) { + auto allocation = new GraphicsAllocation(rootDeviceIndices[i], 1u, AllocationType::bufferHostMemory, addrToPtr(gpuRange), bufferSize, physicalAllocation->peekSharedHandle(), MemoryPool::systemCpuInaccessible, 1, gpuRange); + if (i == 0) { + allocation->setGpuPtr(gpuRange); + allocation->setReservedAddressRange(addrToPtr(gpuRange), bufferSize); + } + multiGraphicsAllocation.addAllocation(allocation); + } + return true; +} + GraphicsAllocation *OsAgnosticMemoryManager::allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) { status = AllocationStatus::Error; MemoryAllocation *allocation = nullptr; @@ -494,6 +514,34 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocatePhysicalDeviceMemory(const return alloc; } +GraphicsAllocation *OsAgnosticMemoryManager::allocatePhysicalHostMemory(const AllocationData &allocationData, AllocationStatus &status) { + status = AllocationStatus::Error; + + auto &productHelper = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHelper(); + GmmRequirements gmmRequirements{}; + gmmRequirements.allowLargePages = true; + gmmRequirements.preferCompressed = false; + auto gmm = std::make_unique(executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getGmmHelper(), nullptr, + allocationData.size, 0u, CacheSettingsHelper::getGmmUsageType(allocationData.type, allocationData.flags.uncacheable, productHelper), + allocationData.storageInfo, gmmRequirements); + + GraphicsAllocation *alloc = nullptr; + + auto ptr = allocateSystemMemory(alignUp(allocationData.size, MemoryConstants::pageSize), MemoryConstants::pageSize2M); + if (ptr != nullptr) { + alloc = new MemoryAllocation(allocationData.rootDeviceIndex, 1u /*num gmms*/, allocationData.type, ptr, ptr, 0u, allocationData.size, + counter, MemoryPool::system4KBPages, allocationData.flags.uncacheable, allocationData.flags.flushL3, maxOsContextCount); + counter++; + } + + if (alloc) { + alloc->setDefaultGmm(gmm.release()); + status = AllocationStatus::Success; + } + + return alloc; +} + GraphicsAllocation *OsAgnosticMemoryManager::allocateMemoryByKMD(const AllocationData &allocationData) { auto &productHelper = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHelper(); diff --git a/shared/source/memory_manager/os_agnostic_memory_manager.h b/shared/source/memory_manager/os_agnostic_memory_manager.h index 2101395e32..f2b24b0034 100644 --- a/shared/source/memory_manager/os_agnostic_memory_manager.h +++ b/shared/source/memory_manager/os_agnostic_memory_manager.h @@ -61,8 +61,11 @@ class OsAgnosticMemoryManager : public MemoryManager { GraphicsAllocation *allocateMemoryByKMD(const AllocationData &allocationData) override; GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; - bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; - void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override; + GraphicsAllocation *allocatePhysicalHostMemory(const AllocationData &allocationData, AllocationStatus &status) override; + bool mapPhysicalDeviceMemoryToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; + bool mapPhysicalHostMemoryToVirtualMemory(RootDeviceIndicesContainer &rootDeviceIndices, MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; + void unMapPhysicalDeviceMemoryFromVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override; + void unMapPhysicalHostMemoryFromVirtualMemory(MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr gmm) override; GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const AllocationData &allocationData) override; diff --git a/shared/source/os_interface/linux/drm_buffer_object.h b/shared/source/os_interface/linux/drm_buffer_object.h index 7e375e0afd..ccb7c006f5 100644 --- a/shared/source/os_interface/linux/drm_buffer_object.h +++ b/shared/source/os_interface/linux/drm_buffer_object.h @@ -235,6 +235,9 @@ class BufferObject { void setUserptr(uint64_t ptr) { this->userptr = ptr; }; uint64_t getUserptr() const { return this->userptr; }; + void setMmapOffset(uint64_t offset) { this->mmapOffset = offset; }; + uint64_t getMmapOffset() const { return this->mmapOffset; }; + static constexpr int gpuHangDetected{-7171}; uint32_t getOsContextId(OsContext *osContext); @@ -257,6 +260,7 @@ class BufferObject { uint64_t unmapSize = 0; uint64_t patIndex = CommonConstants::unsupportedPatIndex; uint64_t userptr = 0u; + uint64_t mmapOffset = 0u; size_t colourChunk = 0; uint64_t gpuAddress = 0llu; diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index bbaff98c69..2d59deb54a 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -27,6 +27,7 @@ #include "shared/source/memory_manager/allocation_properties.h" #include "shared/source/memory_manager/gfx_partition.h" #include "shared/source/memory_manager/host_ptr_manager.h" +#include "shared/source/memory_manager/local_memory_usage.h" #include "shared/source/memory_manager/memory_pool.h" #include "shared/source/memory_manager/multi_graphics_allocation.h" #include "shared/source/memory_manager/residency.h" @@ -638,7 +639,7 @@ GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemory64kb(const Allocatio return nullptr; } -void DrmMemoryManager::unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) { +void DrmMemoryManager::unMapPhysicalDeviceMemoryFromVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) { DrmAllocation *drmAllocation = reinterpret_cast(physicalAllocation); auto bufferObjects = drmAllocation->getBOs(); for (auto bufferObject : bufferObjects) { @@ -655,7 +656,43 @@ void DrmMemoryManager::unMapPhysicalToVirtualMemory(GraphicsAllocation *physical physicalAllocation->setReservedAddressRange(nullptr, 0u); } -bool DrmMemoryManager::mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { +void DrmMemoryManager::unMapPhysicalHostMemoryFromVirtualMemory(MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { + void *addressToUnmap = static_cast(multiGraphicsAllocation.getGraphicsAllocation(physicalAllocation->getRootDeviceIndex()))->getMmapPtr(); + size_t sizeToUnmap = static_cast(multiGraphicsAllocation.getGraphicsAllocation(physicalAllocation->getRootDeviceIndex()))->getMmapSize(); + + for (auto allocation : multiGraphicsAllocation.getGraphicsAllocations()) { + if (allocation == nullptr) { + continue; + } + + const bool isLocked = allocation->isLocked(); + if (isLocked) { + freeAssociatedResourceImpl(*allocation); + } + + auto rootDeviceIndex = allocation->getRootDeviceIndex(); + auto memoryOperationsInterface = static_cast(executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->memoryOperationsInterface.get()); + for (auto &engine : getRegisteredEngines(rootDeviceIndex)) { + memoryOperationsInterface->evictWithinOsContext(engine.osContext, *allocation); + } + + DrmAllocation *drmAlloc = static_cast(allocation); + if (allocation->getRootDeviceIndex() == physicalAllocation->getRootDeviceIndex()) { + drmAlloc->setMmapPtr(nullptr); + drmAlloc->setMmapSize(0u); + freeGraphicsMemory(allocation); + } else { + for (auto bo : drmAlloc->getBOs()) { + unreference(bo, false); + } + delete allocation; + } + } + + this->munmapFunction(addressToUnmap, sizeToUnmap); +} + +bool DrmMemoryManager::mapPhysicalDeviceMemoryToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { DrmAllocation *drmAllocation = reinterpret_cast(physicalAllocation); auto bufferObjects = drmAllocation->getBOs(); for (auto bufferObject : bufferObjects) { @@ -669,6 +706,75 @@ bool DrmMemoryManager::mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAl return true; } +bool DrmMemoryManager::mapPhysicalHostMemoryToVirtualMemory(RootDeviceIndicesContainer &rootDeviceIndices, MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { + auto drmPhysicalAllocation = static_cast(physicalAllocation); + auto &drm = this->getDrm(drmPhysicalAllocation->getRootDeviceIndex()); + BufferObject *physcialBo = drmPhysicalAllocation->getBO(); + uint64_t mmapOffset = physcialBo->getMmapOffset(); + uint64_t internalHandle = 0; + if ((rootDeviceIndices.size() > 1) && (physicalAllocation->peekInternalHandle(this, internalHandle) < 0)) { + return false; + } + + [[maybe_unused]] auto retPtr = this->mmapFunction(addrToPtr(gpuRange), bufferSize, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, drm.getFileDescriptor(), static_cast(mmapOffset)); + DEBUG_BREAK_IF(retPtr != addrToPtr(gpuRange)); + + int physicalBoHandle = physcialBo->peekHandle(); + auto physcialBoHandleWrapper = tryToGetBoHandleWrapperWithSharedOwnership(physicalBoHandle, physicalAllocation->getRootDeviceIndex()); + + auto memoryPool = MemoryPool::system4KBPages; + auto patIndex = drm.getPatIndex(nullptr, AllocationType::bufferHostMemory, CacheRegion::defaultRegion, CachePolicy::writeBack, false, MemoryPoolHelper::isSystemMemoryPool(memoryPool)); + auto bo = new BufferObject(physicalAllocation->getRootDeviceIndex(), &drm, patIndex, std::move(physcialBoHandleWrapper), bufferSize, maxOsContextCount); + + bo->setMmapOffset(mmapOffset); + bo->setAddress(gpuRange); + bo->bind(getDefaultOsContext(drmPhysicalAllocation->getRootDeviceIndex()), 0); + + auto drmAllocation = new DrmAllocation(drmPhysicalAllocation->getRootDeviceIndex(), 1u, AllocationType::bufferHostMemory, bo, addrToPtr(bo->peekAddress()), bo->peekSize(), + static_cast(internalHandle), memoryPool, getGmmHelper(drmPhysicalAllocation->getRootDeviceIndex())->canonize(bo->peekAddress())); + + drmAllocation->setUsmHostAllocation(true); + drmAllocation->setShareableHostMemory(true); + drmAllocation->setMmapPtr(addrToPtr(gpuRange)); + drmAllocation->setMmapSize(bufferSize); + drmAllocation->setCpuPtrAndGpuAddress(retPtr, gpuRange); + drmAllocation->setReservedAddressRange(addrToPtr(gpuRange), bufferSize); + multiGraphicsAllocation.addAllocation(drmAllocation); + this->registerSysMemAlloc(drmAllocation); + + for (size_t i = 0; i < rootDeviceIndices.size(); i++) { + if (rootDeviceIndices[i] == physicalAllocation->getRootDeviceIndex()) { + continue; + } + PrimeHandle openFd{}; + openFd.fileDescriptor = static_cast(internalHandle); + + auto &drmToImport = this->getDrm(rootDeviceIndices[i]); + + auto ioctlHelper = drm.getIoctlHelper(); + if (0 != ioctlHelper->ioctl(DrmIoctl::primeFdToHandle, &openFd)) { + for (auto allocation : multiGraphicsAllocation.getGraphicsAllocations()) { + if (allocation != nullptr) { + multiGraphicsAllocation.removeAllocation(allocation->getRootDeviceIndex()); + freeGraphicsMemory(allocation); + } + } + return false; + } + + auto bo = new BufferObject(rootDeviceIndices[i], &drmToImport, patIndex, openFd.handle, bufferSize, maxOsContextCount); + bo->setAddress(gpuRange); + + auto canonizedGpuAddress = getGmmHelper(rootDeviceIndices[i])->canonize(bo->peekAddress()); + auto allocation = new DrmAllocation(rootDeviceIndices[i], 1u, AllocationType::bufferHostMemory, bo, addrToPtr(bo->peekAddress()), bo->peekSize(), + static_cast(internalHandle), memoryPool, canonizedGpuAddress); + allocation->setImportedMmapPtr(addrToPtr(gpuRange)); + multiGraphicsAllocation.addAllocation(allocation); + } + + return true; +} + GraphicsAllocation *DrmMemoryManager::allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) { const auto memoryPool = MemoryPool::systemCpuInaccessible; StorageInfo systemMemoryStorageInfo = {}; @@ -1893,6 +1999,41 @@ GraphicsAllocation *DrmMemoryManager::allocatePhysicalLocalDeviceMemory(const Al return allocation.release(); } +GraphicsAllocation *DrmMemoryManager::allocatePhysicalHostMemory(const AllocationData &allocationData, AllocationStatus &status) { + const size_t alignedSize = alignUp(allocationData.size, MemoryConstants::pageSize); + + auto gmm = makeGmmIfSingleHandle(allocationData, alignedSize); + std::unique_ptr bo(this->createBufferObjectInMemoryRegion(allocationData.rootDeviceIndex, gmm.get(), + allocationData.type, 0u, alignedSize, + 0u, maxOsContextCount, -1, true, true)); + if (!bo) { + return nullptr; + } + + uint64_t offset = 0; + auto ioctlHelper = this->getDrm(allocationData.rootDeviceIndex).getIoctlHelper(); + uint64_t mmapOffsetWb = ioctlHelper->getDrmParamValue(DrmParam::mmapOffsetWb); + if (!retrieveMmapOffsetForBufferObject(allocationData.rootDeviceIndex, *bo, mmapOffsetWb, offset)) { + return nullptr; + } + bo->setMmapOffset(offset); + + auto allocation = std::make_unique(allocationData.rootDeviceIndex, 1u, allocationData.type, bo.get(), + nullptr, 0u, alignedSize, MemoryPool::system4KBPages); + allocation->setMmapPtr(nullptr); + allocation->setMmapSize(0u); + + registerSharedBoHandleAllocation(allocation.get()); + + bo.release(); + allocation->setDefaultGmm(gmm.release()); + allocation->setUsmHostAllocation(true); + allocation->setShareableHostMemory(true); + allocation->storageInfo = allocationData.storageInfo; + status = AllocationStatus::Success; + return allocation.release(); +} + GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) { status = AllocationStatus::RetryInNonDevicePool; if (!this->localMemorySupported[allocationData.rootDeviceIndex] || diff --git a/shared/source/os_interface/linux/drm_memory_manager.h b/shared/source/os_interface/linux/drm_memory_manager.h index 99745583af..ec584c1824 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.h +++ b/shared/source/os_interface/linux/drm_memory_manager.h @@ -160,8 +160,11 @@ class DrmMemoryManager : public MemoryManager { GraphicsAllocation *allocateMemoryByKMD(const AllocationData &allocationData) override; GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; - bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; - void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override; + GraphicsAllocation *allocatePhysicalHostMemory(const AllocationData &allocationData, AllocationStatus &status) override; + bool mapPhysicalDeviceMemoryToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; + bool mapPhysicalHostMemoryToVirtualMemory(RootDeviceIndicesContainer &rootDeviceIndices, MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; + void unMapPhysicalDeviceMemoryFromVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override; + void unMapPhysicalHostMemoryFromVirtualMemory(MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr gmm) override; GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const AllocationData &allocationData) override; GraphicsAllocation *createSharedUnifiedMemoryAllocation(const AllocationData &allocationData); diff --git a/shared/source/os_interface/windows/wddm_memory_manager.cpp b/shared/source/os_interface/windows/wddm_memory_manager.cpp index 400e00488f..b1cc9492f7 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.cpp +++ b/shared/source/os_interface/windows/wddm_memory_manager.cpp @@ -94,7 +94,7 @@ GfxMemoryAllocationMethod WddmMemoryManager::getPreferredAllocationMethod(const return preferredAllocationMethod; } -void WddmMemoryManager::unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) { +void WddmMemoryManager::unMapPhysicalDeviceMemoryFromVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) { const auto wddm = static_cast(osContext)->getWddm(); wddm->freeGpuVirtualAddress(gpuRange, bufferSize); auto gfxPartition = getGfxPartition(rootDeviceIndex); @@ -107,7 +107,10 @@ void WddmMemoryManager::unMapPhysicalToVirtualMemory(GraphicsAllocation *physica wddmAllocation->setMappedPhysicalMemoryReservation(false); } -bool WddmMemoryManager::mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { +void WddmMemoryManager::unMapPhysicalHostMemoryFromVirtualMemory(MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { +} + +bool WddmMemoryManager::mapPhysicalDeviceMemoryToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { WddmAllocation *wddmAllocation = reinterpret_cast(physicalAllocation); auto decanonizedAddress = getGmmHelper(physicalAllocation->getRootDeviceIndex())->decanonize(gpuRange); wddmAllocation->setMappedPhysicalMemoryReservation(mapGpuVirtualAddress(wddmAllocation, reinterpret_cast(decanonizedAddress))); @@ -116,6 +119,10 @@ bool WddmMemoryManager::mapPhysicalToVirtualMemory(GraphicsAllocation *physicalA return wddmAllocation->isMappedPhysicalMemoryReservation(); } +bool WddmMemoryManager::mapPhysicalHostMemoryToVirtualMemory(RootDeviceIndicesContainer &rootDeviceIndices, MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) { + return false; +} + GraphicsAllocation *WddmMemoryManager::allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) { auto &productHelper = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHelper(); StorageInfo systemMemoryStorageInfo = {}; @@ -635,7 +642,7 @@ GraphicsAllocation *WddmMemoryManager::createGraphicsAllocationFromSharedHandle( } status = mapGpuVirtualAddress(allocation.get(), allocation->getReservedAddressPtr()); } else { - status = mapPhysicalToVirtualMemory(allocation.get(), reinterpret_cast(mapPointer), size); + status = mapPhysicalDeviceMemoryToVirtualMemory(allocation.get(), reinterpret_cast(mapPointer), size); } DEBUG_BREAK_IF(!status); if (!status) { @@ -1385,6 +1392,10 @@ GraphicsAllocation *WddmMemoryManager::allocatePhysicalLocalDeviceMemory(const A return wddmAllocation.release(); } +GraphicsAllocation *WddmMemoryManager::allocatePhysicalHostMemory(const AllocationData &allocationData, AllocationStatus &status) { + return nullptr; +} + GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) { status = AllocationStatus::RetryInNonDevicePool; diff --git a/shared/source/os_interface/windows/wddm_memory_manager.h b/shared/source/os_interface/windows/wddm_memory_manager.h index a0016ed347..846a1f6815 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.h +++ b/shared/source/os_interface/windows/wddm_memory_manager.h @@ -88,8 +88,11 @@ class WddmMemoryManager : public MemoryManager { GraphicsAllocation *allocateMemoryByKMD(const AllocationData &allocationData) override; GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; - void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override; - bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; + GraphicsAllocation *allocatePhysicalHostMemory(const AllocationData &allocationData, AllocationStatus &status) override; + void unMapPhysicalDeviceMemoryFromVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override; + void unMapPhysicalHostMemoryFromVirtualMemory(MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; + bool mapPhysicalDeviceMemoryToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; + bool mapPhysicalHostMemoryToVirtualMemory(RootDeviceIndicesContainer &rootDeviceIndices, MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override; GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr gmm) override; GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const AllocationData &allocationData) override { return nullptr; } diff --git a/shared/source/unified_memory/unified_memory.h b/shared/source/unified_memory/unified_memory.h index 441d87e0ef..8b057ee065 100644 --- a/shared/source/unified_memory/unified_memory.h +++ b/shared/source/unified_memory/unified_memory.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2023 Intel Corporation + * Copyright (C) 2019-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -15,7 +15,8 @@ enum class InternalMemoryType : uint32_t { deviceUnifiedMemory = 0b10, hostUnifiedMemory = 0b100, sharedUnifiedMemory = 0b1000, - reservedDeviceMemory = 0b10000 + reservedDeviceMemory = 0b10000, + reservedHostMemory = 0b100000 }; enum class InternalIpcMemoryType : uint32_t { diff --git a/shared/test/common/mocks/linux/mock_drm_memory_manager.h b/shared/test/common/mocks/linux/mock_drm_memory_manager.h index dc16114f5b..4ad06b8cea 100644 --- a/shared/test/common/mocks/linux/mock_drm_memory_manager.h +++ b/shared/test/common/mocks/linux/mock_drm_memory_manager.h @@ -42,6 +42,7 @@ class TestedDrmMemoryManager : public MemoryManagerCreate { using DrmMemoryManager::allocateGraphicsMemoryWithHostPtr; using DrmMemoryManager::allocateMemoryByKMD; using DrmMemoryManager::allocatePhysicalDeviceMemory; + using DrmMemoryManager::allocatePhysicalHostMemory; using DrmMemoryManager::allocatePhysicalLocalDeviceMemory; using DrmMemoryManager::allocationTypeForCompletionFence; using DrmMemoryManager::allocUserptr; @@ -64,7 +65,8 @@ class TestedDrmMemoryManager : public MemoryManagerCreate { using DrmMemoryManager::localMemBanksCount; using DrmMemoryManager::lockBufferObject; using DrmMemoryManager::lockResourceImpl; - using DrmMemoryManager::mapPhysicalToVirtualMemory; + using DrmMemoryManager::mapPhysicalDeviceMemoryToVirtualMemory; + using DrmMemoryManager::mapPhysicalHostMemoryToVirtualMemory; using DrmMemoryManager::memoryForPinBBs; using DrmMemoryManager::mmapFunction; using DrmMemoryManager::munmapFunction; @@ -83,7 +85,8 @@ class TestedDrmMemoryManager : public MemoryManagerCreate { using DrmMemoryManager::supportsMultiStorageResources; using DrmMemoryManager::tryToGetBoHandleWrapperWithSharedOwnership; using DrmMemoryManager::unlockBufferObject; - using DrmMemoryManager::unMapPhysicalToVirtualMemory; + using DrmMemoryManager::unMapPhysicalDeviceMemoryFromVirtualMemory; + using DrmMemoryManager::unMapPhysicalHostMemoryFromVirtualMemory; using DrmMemoryManager::waitOnCompletionFence; using MemoryManager::allocateGraphicsMemoryInDevicePool; using MemoryManager::allRegisteredEngines; diff --git a/shared/test/common/mocks/mock_memory_manager.cpp b/shared/test/common/mocks/mock_memory_manager.cpp index 16c5996347..dfd038b4a7 100644 --- a/shared/test/common/mocks/mock_memory_manager.cpp +++ b/shared/test/common/mocks/mock_memory_manager.cpp @@ -134,6 +134,10 @@ GraphicsAllocation *MockMemoryManager::allocatePhysicalLocalDeviceMemory(const A return OsAgnosticMemoryManager::allocatePhysicalLocalDeviceMemory(allocationData, status); } +GraphicsAllocation *MockMemoryManager::allocatePhysicalHostMemory(const AllocationData &allocationData, AllocationStatus &status) { + return OsAgnosticMemoryManager::allocatePhysicalHostMemory(allocationData, status); +} + GraphicsAllocation *MockMemoryManager::allocateGraphicsMemory64kb(const AllocationData &allocationData) { allocation64kbPageCreated = true; preferCompressedFlagPassed = forceCompressed ? true : allocationData.flags.preferCompressed; diff --git a/shared/test/common/mocks/mock_memory_manager.h b/shared/test/common/mocks/mock_memory_manager.h index 0e0c19040a..c254a748cb 100644 --- a/shared/test/common/mocks/mock_memory_manager.h +++ b/shared/test/common/mocks/mock_memory_manager.h @@ -47,7 +47,8 @@ class MockMemoryManager : public MemoryManagerCreate { using MemoryManager::pageFaultManager; using MemoryManager::prefetchManager; using MemoryManager::supportsMultiStorageResources; - using MemoryManager::unMapPhysicalToVirtualMemory; + using MemoryManager::unMapPhysicalDeviceMemoryFromVirtualMemory; + using MemoryManager::unMapPhysicalHostMemoryFromVirtualMemory; using MemoryManager::useNonSvmHostPtrAlloc; using OsAgnosticMemoryManager::allocateGraphicsMemoryForImageFromHostPtr; using MemoryManagerCreate::MemoryManagerCreate; @@ -83,6 +84,7 @@ class MockMemoryManager : public MemoryManagerCreate { GraphicsAllocation *allocateMemoryByKMD(const AllocationData &allocationData) override; GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override; + GraphicsAllocation *allocatePhysicalHostMemory(const AllocationData &allocationData, AllocationStatus &status) override; int redundancyRatio = 1; GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override; @@ -217,6 +219,7 @@ class MockMemoryManager : public MemoryManagerCreate { } return allocateGraphicsMemoryForNonSvmHostPtrResult; } + bool allowIndirectAllocationsAsPack(uint32_t rootDeviceIndex) override { if (overrideAllocateAsPackReturn != -1) { return !!overrideAllocateAsPackReturn; @@ -225,12 +228,19 @@ class MockMemoryManager : public MemoryManagerCreate { } } - bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { + bool mapPhysicalDeviceMemoryToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { if (failMapPhysicalToVirtualMemory) { return false; } - return OsAgnosticMemoryManager::mapPhysicalToVirtualMemory(physicalAllocation, gpuRange, bufferSize); - }; + return OsAgnosticMemoryManager::mapPhysicalDeviceMemoryToVirtualMemory(physicalAllocation, gpuRange, bufferSize); + } + + bool mapPhysicalHostMemoryToVirtualMemory(RootDeviceIndicesContainer &rootDeviceIndices, MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { + if (failMapPhysicalToVirtualMemory) { + return false; + } + return OsAgnosticMemoryManager::mapPhysicalHostMemoryToVirtualMemory(rootDeviceIndices, multiGraphicsAllocation, physicalAllocation, gpuRange, bufferSize); + } void registerIpcExportedAllocation(GraphicsAllocation *graphicsAllocation) override { registerIpcExportedAllocationCalled++; diff --git a/shared/test/common/os_interface/linux/device_command_stream_fixture.cpp b/shared/test/common/os_interface/linux/device_command_stream_fixture.cpp index e0dab1592c..8bb28274e3 100644 --- a/shared/test/common/os_interface/linux/device_command_stream_fixture.cpp +++ b/shared/test/common/os_interface/linux/device_command_stream_fixture.cpp @@ -117,6 +117,11 @@ int DrmMockCustom::ioctl(DrmIoctl request, void *arg) { primeToHandleParams->handle = outputHandle; inputFd = primeToHandleParams->fileDescriptor; ioctlCnt.primeFdToHandle++; + if (failOnSecondPrimeFdToHandle == true) { + failOnSecondPrimeFdToHandle = false; + failOnPrimeFdToHandle = true; + break; + } if (failOnPrimeFdToHandle == true) { return -1; } @@ -131,6 +136,9 @@ int DrmMockCustom::ioctl(DrmIoctl request, void *arg) { outputFd++; } ioctlCnt.handleToPrimeFd++; + if (failOnPrimeHandleToFd == true) { + return -1; + } } break; case DrmIoctl::gemSetDomain: { auto setDomainParams = static_cast(arg); diff --git a/shared/test/common/os_interface/linux/device_command_stream_fixture.h b/shared/test/common/os_interface/linux/device_command_stream_fixture.h index f2084240ea..1214deba53 100644 --- a/shared/test/common/os_interface/linux/device_command_stream_fixture.h +++ b/shared/test/common/os_interface/linux/device_command_stream_fixture.h @@ -268,6 +268,8 @@ struct DrmMockCustom : public Drm { uint64_t mmapOffsetFlags = 0; bool failOnMmapOffset = false; bool failOnPrimeFdToHandle = false; + bool failOnSecondPrimeFdToHandle = false; + bool failOnPrimeHandleToFd = false; // DRM_IOCTL_I915_GEM_CREATE_EXT uint64_t createExtSize = 0; diff --git a/shared/test/common/os_interface/linux/drm_memory_manager_fixture.h b/shared/test/common/os_interface/linux/drm_memory_manager_fixture.h index 8c898fdf46..8e508c03f3 100644 --- a/shared/test/common/os_interface/linux/drm_memory_manager_fixture.h +++ b/shared/test/common/os_interface/linux/drm_memory_manager_fixture.h @@ -130,7 +130,7 @@ class DrmMemoryManagerFixtureWithoutQuietIoctlExpectation { DrmMockCustom::IoctlResExt ioctlResExt = {0, 0}; DebugManagerStateRestore restore; const uint32_t rootDeviceIndex = 1u; - const uint32_t numRootDevices = 2u; + const uint32_t numRootDevices = 3u; }; class DrmMemoryManagerFixtureWithLocalMemoryAndWithoutQuietIoctlExpectation : public DrmMemoryManagerFixtureWithoutQuietIoctlExpectation { diff --git a/shared/test/common/os_interface/windows/mock_wddm_memory_manager.h b/shared/test/common/os_interface/windows/mock_wddm_memory_manager.h index cc2395db8d..8b0c476eb1 100644 --- a/shared/test/common/os_interface/windows/mock_wddm_memory_manager.h +++ b/shared/test/common/os_interface/windows/mock_wddm_memory_manager.h @@ -24,6 +24,7 @@ class MockWddmMemoryManager : public MemoryManagerCreate { using BaseClass::allocateGraphicsMemoryWithProperties; using BaseClass::allocateMemoryByKMD; using BaseClass::allocatePhysicalDeviceMemory; + using BaseClass::allocatePhysicalHostMemory; using BaseClass::allocatePhysicalLocalDeviceMemory; using BaseClass::allRegisteredEngines; using BaseClass::createGraphicsAllocation; @@ -32,9 +33,11 @@ class MockWddmMemoryManager : public MemoryManagerCreate { using BaseClass::getWddm; using BaseClass::gfxPartitions; using BaseClass::localMemorySupported; - using BaseClass::mapPhysicalToVirtualMemory; + using BaseClass::mapPhysicalDeviceMemoryToVirtualMemory; + using BaseClass::mapPhysicalHostMemoryToVirtualMemory; using BaseClass::supportsMultiStorageResources; - using BaseClass::unMapPhysicalToVirtualMemory; + using BaseClass::unMapPhysicalDeviceMemoryFromVirtualMemory; + using BaseClass::unMapPhysicalHostMemoryFromVirtualMemory; using MemoryManagerCreate::MemoryManagerCreate; using BaseClass::executionEnvironment; using BaseClass::getHugeGfxMemoryChunkSize; diff --git a/shared/test/unit_test/device/neo_device_tests.cpp b/shared/test/unit_test/device/neo_device_tests.cpp index fe174ce6af..f0e332db44 100644 --- a/shared/test/unit_test/device/neo_device_tests.cpp +++ b/shared/test/unit_test/device/neo_device_tests.cpp @@ -549,8 +549,11 @@ TEST_F(DeviceGetCapsTest, givenFlagEnabled64kbPagesWhenCallConstructorMemoryMana GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const AllocationData &allocationData) override { return nullptr; }; GraphicsAllocation *allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; GraphicsAllocation *allocatePhysicalLocalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; - void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; - bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; + GraphicsAllocation *allocatePhysicalHostMemory(const AllocationData &allocationData, AllocationStatus &status) override { return nullptr; }; + void unMapPhysicalDeviceMemoryFromVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) override { return; }; + void unMapPhysicalHostMemoryFromVirtualMemory(MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return; }; + bool mapPhysicalDeviceMemoryToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; + bool mapPhysicalHostMemoryToVirtualMemory(RootDeviceIndicesContainer &rootDeviceIndices, MultiGraphicsAllocation &multiGraphicsAllocation, GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) override { return false; }; GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr gmm) override { return nullptr; }; GraphicsAllocation *allocateMemoryByKMD(const AllocationData &allocationData) override { return nullptr; }; diff --git a/shared/test/unit_test/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp b/shared/test/unit_test/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp index ae8bb1ff97..f7c35d9c37 100644 --- a/shared/test/unit_test/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp +++ b/shared/test/unit_test/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp @@ -22,8 +22,6 @@ #include "shared/test/common/mocks/ult_device_factory.h" #include "shared/test/common/test_macros/hw_test.h" -#include "hw_cmds.h" - using namespace NEO; TEST(MemoryManagerTest, givenSetUseSytemMemoryWhenGraphicsAllocationInDevicePoolIsAllocatedThenNullptrIsReturned) { @@ -350,12 +348,26 @@ TEST(BaseMemoryManagerTest, givenDebugVariableSetWhenCompressedBufferIsCreatedTh memoryManager.freeGraphicsMemory(allocationBufferCompressed); } -TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsMemoryThenPhysicalAllocationReturned) { +TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsDeviceMemoryThenPhysicalAllocationReturned) { MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); executionEnvironment.initGmm(); MemoryManagerCreate memoryManager(false, true, executionEnvironment); AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::buffer, mockDeviceBitfield); + allocPropertiesBuffer.flags.isUSMDeviceAllocation = true; + + auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); + EXPECT_NE(nullptr, allocationBuffer); + memoryManager.freeGraphicsMemory(allocationBuffer); +} + +TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsHostMemoryThenPhysicalAllocationReturned) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + executionEnvironment.initGmm(); + MemoryManagerCreate memoryManager(false, true, executionEnvironment); + + AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::buffer, mockDeviceBitfield); + allocPropertiesBuffer.flags.isUSMDeviceAllocation = false; auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); EXPECT_NE(nullptr, allocationBuffer); @@ -365,8 +377,8 @@ TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsMemoryThenPhysica class MockMemoryManagerLocalMemory : public OsAgnosticMemoryManager { public: using OsAgnosticMemoryManager::localMemorySupported; - using OsAgnosticMemoryManager::mapPhysicalToVirtualMemory; - using OsAgnosticMemoryManager::unMapPhysicalToVirtualMemory; + using OsAgnosticMemoryManager::mapPhysicalDeviceMemoryToVirtualMemory; + using OsAgnosticMemoryManager::unMapPhysicalDeviceMemoryFromVirtualMemory; MockMemoryManagerLocalMemory(ExecutionEnvironment &executionEnvironment) : OsAgnosticMemoryManager(executionEnvironment) {} }; @@ -377,6 +389,7 @@ TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsMemoryWithoutLoca memoryManager.localMemorySupported[0] = 0; AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::buffer, mockDeviceBitfield); + allocPropertiesBuffer.flags.isUSMDeviceAllocation = true; auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); EXPECT_NE(nullptr, allocationBuffer); @@ -392,25 +405,39 @@ class MockMemoryManagerNoLocalMemoryFail : public OsAgnosticMemoryManager { } }; -TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsMemoryWithoutLocalMemoryThenNullptrReturned) { +TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsDeviceMemoryWithoutLocalMemoryThenNullptrReturned) { MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); executionEnvironment.initGmm(); MemoryManagerCreate memoryManager(false, true, executionEnvironment); memoryManager.localMemorySupported[0] = 0; AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::buffer, mockDeviceBitfield); + allocPropertiesBuffer.flags.isUSMDeviceAllocation = true; auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); EXPECT_EQ(nullptr, allocationBuffer); } -TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsMemoryWithLocalMemoryThenNullptrReturned) { +TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsDeviceMemoryWithLocalMemoryThenNullptrReturned) { MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); executionEnvironment.initGmm(); MemoryManagerCreate memoryManager(false, true, executionEnvironment); memoryManager.localMemorySupported[0] = 1; AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::buffer, mockDeviceBitfield); + allocPropertiesBuffer.flags.isUSMDeviceAllocation = true; + + auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); + EXPECT_EQ(nullptr, allocationBuffer); +} + +TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsHostMemoryThenNullptrReturned) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + executionEnvironment.initGmm(); + MemoryManagerCreate memoryManager(false, true, executionEnvironment); + + AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::bufferHostMemory, mockDeviceBitfield); + allocPropertiesBuffer.flags.isUSMHostAllocation = true; auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); EXPECT_EQ(nullptr, allocationBuffer); @@ -418,8 +445,8 @@ TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsMemoryWithLocalMe class MockAgnosticMemoryManager : public OsAgnosticMemoryManager { public: - using OsAgnosticMemoryManager::mapPhysicalToVirtualMemory; - using OsAgnosticMemoryManager::unMapPhysicalToVirtualMemory; + using OsAgnosticMemoryManager::mapPhysicalDeviceMemoryToVirtualMemory; + using OsAgnosticMemoryManager::unMapPhysicalDeviceMemoryFromVirtualMemory; MockAgnosticMemoryManager(ExecutionEnvironment &executionEnvironment) : OsAgnosticMemoryManager(executionEnvironment) {} }; @@ -429,14 +456,15 @@ TEST(BaseMemoryManagerTest, givenCalltoMapAndUnMapThenVirtialAddressSetUnSetOnPh MemoryManagerCreate memoryManager(false, true, executionEnvironment); AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::buffer, mockDeviceBitfield); + allocPropertiesBuffer.flags.isUSMDeviceAllocation = true; auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); EXPECT_NE(nullptr, allocationBuffer); uint64_t gpuAddress = 0x1234; size_t size = 4096; - memoryManager.mapPhysicalToVirtualMemory(allocationBuffer, gpuAddress, size); + memoryManager.mapPhysicalDeviceMemoryToVirtualMemory(allocationBuffer, gpuAddress, size); EXPECT_EQ(gpuAddress, allocationBuffer->getGpuAddress()); - memoryManager.unMapPhysicalToVirtualMemory(allocationBuffer, gpuAddress, size, nullptr, 0u); + memoryManager.unMapPhysicalDeviceMemoryFromVirtualMemory(allocationBuffer, gpuAddress, size, nullptr, 0u); EXPECT_NE(gpuAddress, allocationBuffer->getGpuAddress()); memoryManager.freeGraphicsMemory(allocationBuffer); } @@ -464,6 +492,7 @@ TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsMemoryWithFailedR memoryManager.localMemorySupported[0] = 1; AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::buffer, mockDeviceBitfield); + allocPropertiesBuffer.flags.isUSMDeviceAllocation = true; auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); EXPECT_EQ(nullptr, allocationBuffer); @@ -477,6 +506,7 @@ TEST(BaseMemoryManagerTest, givenCalltoAllocatePhysicalGraphicsMemoryWithFailedL memoryManager.failAllocate = true; AllocationProperties allocPropertiesBuffer(mockRootDeviceIndex, 1, AllocationType::buffer, mockDeviceBitfield); + allocPropertiesBuffer.flags.isUSMDeviceAllocation = true; auto allocationBuffer = memoryManager.allocatePhysicalGraphicsMemory(allocPropertiesBuffer); EXPECT_EQ(nullptr, allocationBuffer); @@ -941,4 +971,4 @@ TEST(MemoryManagerTest, givenDebugVariableToToggleGpuVaBitsWhenAllocatingResourc memoryManager.freeGraphicsMemory(allocation); } -} \ No newline at end of file +} diff --git a/shared/test/unit_test/memory_manager/memory_manager_tests.cpp b/shared/test/unit_test/memory_manager/memory_manager_tests.cpp index 455540f338..f84695f3f0 100644 --- a/shared/test/unit_test/memory_manager/memory_manager_tests.cpp +++ b/shared/test/unit_test/memory_manager/memory_manager_tests.cpp @@ -98,7 +98,7 @@ TEST(MemoryManagerTest, givenAllocationWithNullCpuPtrThenMemoryCopyToAllocationR EXPECT_FALSE(memoryManager.copyMemoryToAllocation(&allocation, offset, nullptr, 0)); } -TEST(MemoryManagerTest, givenGraphicsAllocationWhenMapCalledThenDontResetCpuAddress) { +TEST(MemoryManagerTest, givenDeviceGraphicsAllocationWhenMapCalledThenDontResetCpuAddress) { MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); MockMemoryManager memoryManager(false, false, executionEnvironment); @@ -106,15 +106,42 @@ TEST(MemoryManagerTest, givenGraphicsAllocationWhenMapCalledThenDontResetCpuAddr MockGraphicsAllocation allocation{&allocationStorage, 1}; EXPECT_NE(nullptr, allocation.getUnderlyingBuffer()); - EXPECT_TRUE(memoryManager.mapPhysicalToVirtualMemory(&allocation, 0x12300, 0)); + EXPECT_TRUE(memoryManager.mapPhysicalDeviceMemoryToVirtualMemory(&allocation, 0x12300, 0)); EXPECT_EQ(&allocationStorage, allocation.getUnderlyingBuffer()); EXPECT_EQ(0x12300u, allocation.getGpuAddress()); - memoryManager.unMapPhysicalToVirtualMemory(&allocation, 1, 1, nullptr, 0); + memoryManager.unMapPhysicalDeviceMemoryFromVirtualMemory(&allocation, 1, 1, nullptr, 0); EXPECT_EQ(&allocationStorage, allocation.getUnderlyingBuffer()); EXPECT_EQ(0u, allocation.getGpuAddress()); } +TEST(MemoryManagerTest, givenHostGraphicsAllocationWhenMapCalledThenDontResetCpuAddress) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + MockMemoryManager memoryManager(false, false, executionEnvironment); + + uint8_t allocationStorage = 0; + MockGraphicsAllocation allocation{&allocationStorage, 1}; + EXPECT_NE(nullptr, allocation.getUnderlyingBuffer()); + + RootDeviceIndicesContainer rootDeviceIndices; + rootDeviceIndices.pushUnique(0); + rootDeviceIndices.pushUnique(1); + MultiGraphicsAllocation multiGraphicsAllocation{static_cast(rootDeviceIndices.size())}; + EXPECT_TRUE(memoryManager.mapPhysicalHostMemoryToVirtualMemory(rootDeviceIndices, multiGraphicsAllocation, &allocation, 0x12300, 0)); + EXPECT_EQ(&allocationStorage, allocation.getUnderlyingBuffer()); + EXPECT_NE(0x12300u, allocation.getGpuAddress()); + for (size_t i = 0; i < rootDeviceIndices.size(); i++) { + EXPECT_NE(multiGraphicsAllocation.getGraphicsAllocation(static_cast(i)), nullptr); + } + + memoryManager.unMapPhysicalHostMemoryFromVirtualMemory(multiGraphicsAllocation, static_cast(&allocation), 0x12300, 0); + EXPECT_EQ(&allocationStorage, allocation.getUnderlyingBuffer()); + EXPECT_NE(0x12300u, allocation.getGpuAddress()); + for (size_t i = 0; i < rootDeviceIndices.size(); i++) { + EXPECT_EQ(multiGraphicsAllocation.getGraphicsAllocation(static_cast(i)), nullptr); + } +} + TEST(MemoryManagerTest, givenDefaultMemoryManagerWhenItIsAskedForBudgetExhaustionThenFalseIsReturned) { MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); MockMemoryManager memoryManager(false, false, executionEnvironment); @@ -3335,4 +3362,4 @@ TEST(MemoryManagerTest, WhenGettingExtraDevicePropertiesThenPropertiesRemainUnch EXPECT_EQ(moduleId, 0u); EXPECT_EQ(serverType, 0u); -} \ No newline at end of file +} diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index 681979293a..9473680a8d 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -212,6 +212,56 @@ TEST_F(DrmMemoryManagerTest, GivenAllocatePhysicalDeviceMemoryThenSuccessReturne memoryManager->freeGraphicsMemory(allocation); } +TEST_F(DrmMemoryManagerTest, GivenAllocatePhysicalHostMemoryWithoutMemoryInfoThenNullptrIsReturned) { + allocationData.size = MemoryConstants::pageSize; + allocationData.flags.shareable = true; + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + auto allocation = memoryManager->allocatePhysicalHostMemory(allocationData, status); + EXPECT_EQ(status, MemoryManager::AllocationStatus::Error); + EXPECT_EQ(nullptr, allocation); +} + +TEST_F(DrmMemoryManagerTest, GivenAllocatePhysicalHostMemoryAndMmapOffsetFailsThenNullptrIsReturned) { + mock->ioctlExpected.gemCreateExt = 1; + mock->ioctlExpected.gemMmapOffset = 1; + mock->ioctlExpected.gemClose = 1; + + mock->failOnMmapOffset = true; + + std::vector regionInfo(1); + regionInfo[0].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_SYSTEM, 0}; + this->mock->memoryInfo.reset(new MemoryInfo(regionInfo, *mock)); + this->mock->memoryInfoQueried = true; + + allocationData.size = MemoryConstants::pageSize; + allocationData.flags.shareable = true; + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + auto allocation = memoryManager->allocatePhysicalHostMemory(allocationData, status); + EXPECT_EQ(status, MemoryManager::AllocationStatus::Error); + EXPECT_EQ(nullptr, allocation); +} + +TEST_F(DrmMemoryManagerTest, GivenAllocatePhysicalHostMemoryThenSuccessReturnedAndNoVirtualMemoryAssigned) { + mock->ioctlExpected.gemWait = 1; + mock->ioctlExpected.gemCreateExt = 1; + mock->ioctlExpected.gemMmapOffset = 1; + mock->ioctlExpected.gemClose = 1; + + std::vector regionInfo(1); + regionInfo[0].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_SYSTEM, 0}; + this->mock->memoryInfo.reset(new MemoryInfo(regionInfo, *mock)); + this->mock->memoryInfoQueried = true; + + allocationData.size = MemoryConstants::pageSize; + allocationData.flags.shareable = true; + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + auto allocation = memoryManager->allocatePhysicalHostMemory(allocationData, status); + EXPECT_EQ(status, MemoryManager::AllocationStatus::Success); + EXPECT_NE(nullptr, allocation); + EXPECT_EQ(0u, allocation->getGpuAddress()); + memoryManager->freeGraphicsMemory(allocation); +} + TEST_F(DrmMemoryManagerTest, whenCallingCheckUnexpectedGpuPagedfaultThenAllEnginesWereChecked) { mock->ioctlExpected.total = -1; // don't care memoryManager->checkUnexpectedGpuPageFault(); @@ -6554,7 +6604,7 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenAllocati EXPECT_EQ(MemoryPool::localMemory, allocation->getMemoryPool()); EXPECT_EQ(0u, allocation->getGpuAddress()); EXPECT_EQ(EngineLimits::maxHandleCount, allocation->getNumGmms()); - memoryManager->mapPhysicalToVirtualMemory(allocation, gpuAddress, allocData.size); + memoryManager->mapPhysicalDeviceMemoryToVirtualMemory(allocation, gpuAddress, allocData.size); EXPECT_EQ(gpuAddress, allocation->getGpuAddress()); auto drmAllocation = static_cast(allocation); @@ -6570,7 +6620,7 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenAllocati boAddress += boSize; } auto osContext = device->getDefaultEngine().osContext; - memoryManager->unMapPhysicalToVirtualMemory(allocation, gpuAddress, allocData.size, osContext, 0u); + memoryManager->unMapPhysicalDeviceMemoryFromVirtualMemory(allocation, gpuAddress, allocData.size, osContext, 0u); EXPECT_EQ(0u, allocation->getGpuAddress()); memoryManager->freeGraphicsMemory(allocation); } @@ -6591,7 +6641,7 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenAllocati EXPECT_NE(nullptr, kernelIsaAllocation); - memoryManager->mapPhysicalToVirtualMemory(kernelIsaAllocation, gpuAddress, allocData.size); + memoryManager->mapPhysicalDeviceMemoryToVirtualMemory(kernelIsaAllocation, gpuAddress, allocData.size); auto gpuAddressReserved = kernelIsaAllocation->getGpuAddress(); auto &bos = kernelIsaAllocation->getBOs(); @@ -6610,7 +6660,7 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenAllocati EXPECT_EQ(0b1011u, storageInfo.memoryBanks.to_ulong()); auto osContext = device->getDefaultEngine().osContext; - memoryManager->unMapPhysicalToVirtualMemory(kernelIsaAllocation, gpuAddress, allocData.size, osContext, 0u); + memoryManager->unMapPhysicalDeviceMemoryFromVirtualMemory(kernelIsaAllocation, gpuAddress, allocData.size, osContext, 0u); memoryManager->freeGraphicsMemory(kernelIsaAllocation); } @@ -6633,7 +6683,7 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenAllocati EXPECT_EQ(MemoryPool::localMemory, allocation->getMemoryPool()); EXPECT_EQ(0u, allocation->getGpuAddress()); EXPECT_EQ(EngineLimits::maxHandleCount, allocation->getNumGmms()); - memoryManager->mapPhysicalToVirtualMemory(allocation, gpuAddress, allocData.size); + memoryManager->mapPhysicalDeviceMemoryToVirtualMemory(allocation, gpuAddress, allocData.size); EXPECT_EQ(gpuAddress, allocation->getGpuAddress()); auto drmAllocation = static_cast(allocation); @@ -6649,7 +6699,7 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenAllocati boAddress += boSize; } auto osContext = device->getDefaultEngine().osContext; - memoryManager->unMapPhysicalToVirtualMemory(allocation, gpuAddress, allocData.size, osContext, 0u); + memoryManager->unMapPhysicalDeviceMemoryFromVirtualMemory(allocation, gpuAddress, allocData.size, osContext, 0u); for (auto handleId = 0u; handleId < EngineLimits::maxHandleCount; handleId++) { auto bo = bos[handleId]; auto contextId = bo->getOsContextId(osContext); @@ -6759,6 +6809,210 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenAllocati memoryManager->freeGraphicsMemory(kernelIsaAllocation); } +TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDeviceBitfieldWithHolesWhenMappingPhysicalHostMemoryThenMappingIsSuccessful) { + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize; + allocData.flags.allocateMemory = true; + allocData.flags.isUSMHostAllocation = true; + allocData.type = AllocationType::bufferHostMemory; + allocData.storageInfo.multiStorage = false; + allocData.rootDeviceIndex = rootDeviceIndex; + uint64_t gpuAddress = 0x1234; + + auto physicalAllocation = static_cast(memoryManager->allocatePhysicalHostMemory(allocData, status)); + EXPECT_NE(nullptr, physicalAllocation); + + RootDeviceIndicesContainer rootDeviceIndices; + rootDeviceIndices.pushUnique(0); + rootDeviceIndices.pushUnique(1); + rootDeviceIndices.pushUnique(2); + MultiGraphicsAllocation multiGraphicsAllocation{static_cast(rootDeviceIndices.size())}; + EXPECT_TRUE(memoryManager->mapPhysicalHostMemoryToVirtualMemory(rootDeviceIndices, multiGraphicsAllocation, physicalAllocation, gpuAddress, allocData.size)); + for (uint32_t i = 0; i < static_cast(rootDeviceIndices.size()); i++) { + EXPECT_NE(nullptr, multiGraphicsAllocation.getGraphicsAllocation(i)); + } + + memoryManager->unMapPhysicalHostMemoryFromVirtualMemory(multiGraphicsAllocation, physicalAllocation, gpuAddress, allocData.size); + memoryManager->freeGraphicsMemory(physicalAllocation); +} + +TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDeviceBitfieldWithHolesAndPrimeHandleToFdFailsWhenMappingPhysicalHostMemoryThenMappingFails) { + mock->failOnPrimeHandleToFd = true; + + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize; + allocData.flags.allocateMemory = true; + allocData.flags.isUSMHostAllocation = true; + allocData.type = AllocationType::bufferHostMemory; + allocData.storageInfo.multiStorage = false; + allocData.rootDeviceIndex = rootDeviceIndex; + uint64_t gpuAddress = 0x1234; + + auto physicalAllocation = static_cast(memoryManager->allocatePhysicalHostMemory(allocData, status)); + EXPECT_NE(nullptr, physicalAllocation); + + RootDeviceIndicesContainer rootDeviceIndices; + rootDeviceIndices.pushUnique(0); + rootDeviceIndices.pushUnique(1); + rootDeviceIndices.pushUnique(2); + MultiGraphicsAllocation multiGraphicsAllocation{static_cast(rootDeviceIndices.size())}; + EXPECT_FALSE(memoryManager->mapPhysicalHostMemoryToVirtualMemory(rootDeviceIndices, multiGraphicsAllocation, physicalAllocation, gpuAddress, allocData.size)); + + for (uint32_t i = 0; i < static_cast(rootDeviceIndices.size()); i++) { + EXPECT_EQ(nullptr, multiGraphicsAllocation.getGraphicsAllocation(i)); + } + + memoryManager->freeGraphicsMemory(physicalAllocation); +} + +TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDeviceBitfieldWithHolesAndPrimeFdToHandleFailsWhenMappingPhysicalHostMemoryThenMappingFails) { + mock->failOnPrimeFdToHandle = true; + + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize; + allocData.flags.allocateMemory = true; + allocData.flags.isUSMHostAllocation = true; + allocData.type = AllocationType::bufferHostMemory; + allocData.storageInfo.multiStorage = false; + allocData.rootDeviceIndex = rootDeviceIndex; + uint64_t gpuAddress = 0x1234; + + auto physicalAllocation = static_cast(memoryManager->allocatePhysicalHostMemory(allocData, status)); + EXPECT_NE(nullptr, physicalAllocation); + + RootDeviceIndicesContainer rootDeviceIndices; + rootDeviceIndices.pushUnique(0); + rootDeviceIndices.pushUnique(1); + rootDeviceIndices.pushUnique(2); + MultiGraphicsAllocation multiGraphicsAllocation{static_cast(rootDeviceIndices.size())}; + EXPECT_FALSE(memoryManager->mapPhysicalHostMemoryToVirtualMemory(rootDeviceIndices, multiGraphicsAllocation, physicalAllocation, gpuAddress, allocData.size)); + + for (uint32_t i = 0; i < static_cast(rootDeviceIndices.size()); i++) { + EXPECT_EQ(nullptr, multiGraphicsAllocation.getGraphicsAllocation(i)); + } + + memoryManager->freeGraphicsMemory(physicalAllocation); +} + +TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenDeviceBitfieldWithHolesAndSecondPrimeFdToHandleFailsWhenMappingPhysicalHostMemoryThenMappingFails) { + mock->failOnSecondPrimeFdToHandle = true; + + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize; + allocData.flags.allocateMemory = true; + allocData.flags.isUSMHostAllocation = true; + allocData.type = AllocationType::bufferHostMemory; + allocData.storageInfo.multiStorage = false; + allocData.rootDeviceIndex = rootDeviceIndex; + uint64_t gpuAddress = 0x1234; + + auto physicalAllocation = static_cast(memoryManager->allocatePhysicalHostMemory(allocData, status)); + EXPECT_NE(nullptr, physicalAllocation); + + RootDeviceIndicesContainer rootDeviceIndices; + rootDeviceIndices.pushUnique(0); + rootDeviceIndices.pushUnique(1); + rootDeviceIndices.pushUnique(2); + MultiGraphicsAllocation multiGraphicsAllocation{static_cast(rootDeviceIndices.size())}; + EXPECT_FALSE(memoryManager->mapPhysicalHostMemoryToVirtualMemory(rootDeviceIndices, multiGraphicsAllocation, physicalAllocation, gpuAddress, allocData.size)); + + for (uint32_t i = 0; i < static_cast(rootDeviceIndices.size()); i++) { + EXPECT_EQ(nullptr, multiGraphicsAllocation.getGraphicsAllocation(i)); + } + + memoryManager->freeGraphicsMemory(physicalAllocation); +} + +TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenSingleRootDeviceWhenMappingPhysicalHostMemoryThenMappingIsSuccessful) { + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize; + allocData.flags.allocateMemory = true; + allocData.flags.isUSMHostAllocation = true; + allocData.type = AllocationType::bufferHostMemory; + allocData.storageInfo.multiStorage = false; + allocData.rootDeviceIndex = rootDeviceIndex; + uint64_t gpuAddress = 0x1234; + + auto physicalAllocation = static_cast(memoryManager->allocatePhysicalHostMemory(allocData, status)); + EXPECT_NE(nullptr, physicalAllocation); + + RootDeviceIndicesContainer rootDeviceIndices; + rootDeviceIndices.pushUnique(rootDeviceIndex); + MultiGraphicsAllocation multiGraphicsAllocation{1u}; + EXPECT_TRUE(memoryManager->mapPhysicalHostMemoryToVirtualMemory(rootDeviceIndices, multiGraphicsAllocation, physicalAllocation, gpuAddress, allocData.size)); + EXPECT_NE(nullptr, multiGraphicsAllocation.getDefaultGraphicsAllocation()); + EXPECT_NE(nullptr, multiGraphicsAllocation.getGraphicsAllocation(rootDeviceIndex)); + + memoryManager->unMapPhysicalHostMemoryFromVirtualMemory(multiGraphicsAllocation, physicalAllocation, gpuAddress, allocData.size); + memoryManager->freeGraphicsMemory(physicalAllocation); +} + +TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenSingleRootDeviceWhenMappingLockedPhysicalHostMemoryThenMappingIsSuccessful) { + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize; + allocData.flags.allocateMemory = true; + allocData.flags.isUSMHostAllocation = true; + allocData.type = AllocationType::bufferHostMemory; + allocData.storageInfo.multiStorage = false; + allocData.rootDeviceIndex = rootDeviceIndex; + uint64_t gpuAddress = 0x1234; + + auto physicalAllocation = static_cast(memoryManager->allocatePhysicalHostMemory(allocData, status)); + EXPECT_NE(nullptr, physicalAllocation); + physicalAllocation->lock(addrToPtr(gpuAddress)); + + RootDeviceIndicesContainer rootDeviceIndices; + rootDeviceIndices.pushUnique(rootDeviceIndex); + MultiGraphicsAllocation multiGraphicsAllocation{1u}; + EXPECT_TRUE(memoryManager->mapPhysicalHostMemoryToVirtualMemory(rootDeviceIndices, multiGraphicsAllocation, physicalAllocation, gpuAddress, allocData.size)); + EXPECT_NE(nullptr, multiGraphicsAllocation.getDefaultGraphicsAllocation()); + EXPECT_NE(nullptr, multiGraphicsAllocation.getGraphicsAllocation(rootDeviceIndex)); + multiGraphicsAllocation.getGraphicsAllocation(rootDeviceIndex)->lock(addrToPtr(gpuAddress)); + + memoryManager->unMapPhysicalHostMemoryFromVirtualMemory(multiGraphicsAllocation, physicalAllocation, gpuAddress, allocData.size); + memoryManager->freeGraphicsMemory(physicalAllocation); +} + +TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenSingleRootDeviceAndPrimeHandleToFdFailsWhenMappingPhysicalHostMemoryThenMappingIsSuccessful) { + mock->failOnPrimeHandleToFd = true; + + MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error; + AllocationData allocData; + allocData.allFlags = 0; + allocData.size = MemoryConstants::pageSize; + allocData.flags.allocateMemory = true; + allocData.flags.isUSMHostAllocation = true; + allocData.type = AllocationType::bufferHostMemory; + allocData.storageInfo.multiStorage = false; + allocData.rootDeviceIndex = rootDeviceIndex; + uint64_t gpuAddress = 0x1234; + + auto physicalAllocation = static_cast(memoryManager->allocatePhysicalHostMemory(allocData, status)); + EXPECT_NE(nullptr, physicalAllocation); + + RootDeviceIndicesContainer rootDeviceIndices; + rootDeviceIndices.pushUnique(rootDeviceIndex); + MultiGraphicsAllocation multiGraphicsAllocation{1u}; + EXPECT_TRUE(memoryManager->mapPhysicalHostMemoryToVirtualMemory(rootDeviceIndices, multiGraphicsAllocation, physicalAllocation, gpuAddress, allocData.size)); + EXPECT_NE(nullptr, multiGraphicsAllocation.getDefaultGraphicsAllocation()); + EXPECT_NE(nullptr, multiGraphicsAllocation.getGraphicsAllocation(rootDeviceIndex)); + + memoryManager->unMapPhysicalHostMemoryFromVirtualMemory(multiGraphicsAllocation, physicalAllocation, gpuAddress, allocData.size); + memoryManager->freeGraphicsMemory(physicalAllocation); +} + struct DrmMemoryManagerLocalMemoryAlignmentTest : DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest { std::unique_ptr createMemoryManager() { return std::make_unique(true, false, false, *executionEnvironment); diff --git a/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp b/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp index 683eb55893..687192c8a3 100644 --- a/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp +++ b/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp @@ -83,8 +83,8 @@ struct MockWddmLinuxMemoryManager : NEO::WddmMemoryManager { using WddmMemoryManager::allocatePhysicalLocalDeviceMemory; using WddmMemoryManager::createPhysicalAllocation; using WddmMemoryManager::localMemorySupported; - using WddmMemoryManager::mapPhysicalToVirtualMemory; - using WddmMemoryManager::unMapPhysicalToVirtualMemory; + using WddmMemoryManager::mapPhysicalDeviceMemoryToVirtualMemory; + using WddmMemoryManager::unMapPhysicalDeviceMemoryFromVirtualMemory; using WddmMemoryManager::WddmMemoryManager; NTSTATUS createInternalNTHandle(D3DKMT_HANDLE *resourceHandle, HANDLE *ntHandle, uint32_t rootDeviceIndex) override { if (failCreateInternalNTHandle) { diff --git a/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp index 877c8f2902..e8cf8f532d 100644 --- a/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp @@ -1156,7 +1156,7 @@ TEST_F(WddmMemoryManagerSimpleTest, GivenShareableEnabledAndHugeSizeWhenAskedToC memoryManager->freeGraphicsMemory(allocation); } -TEST_F(WddmMemoryManagerSimpleTest, GivenPhysicalMemoryAndVirtualMemoryThenMapSucceeds) { +TEST_F(WddmMemoryManagerSimpleTest, GivenPhysicalDeviceMemoryAndVirtualMemoryThenMapSucceeds) { memoryManager.reset(new MockWddmMemoryManager(false, false, executionEnvironment)); memoryManager->hugeGfxMemoryChunkSize = MemoryConstants::pageSize64k; AllocationData allocationData; @@ -1165,12 +1165,34 @@ TEST_F(WddmMemoryManagerSimpleTest, GivenPhysicalMemoryAndVirtualMemoryThenMapSu MemoryManager::AllocationStatus status; auto allocation = memoryManager->allocatePhysicalLocalDeviceMemory(allocationData, status); EXPECT_NE(nullptr, allocation); - auto res = memoryManager->mapPhysicalToVirtualMemory(allocation, gpuRange, allocationData.size); + auto res = memoryManager->mapPhysicalDeviceMemoryToVirtualMemory(allocation, gpuRange, allocationData.size); EXPECT_TRUE(res); - memoryManager->unMapPhysicalToVirtualMemory(allocation, gpuRange, allocationData.size, osContext, 0u); + memoryManager->unMapPhysicalDeviceMemoryFromVirtualMemory(allocation, gpuRange, allocationData.size, osContext, 0u); memoryManager->freeGraphicsMemory(allocation); } +TEST_F(WddmMemoryManagerSimpleTest, GivenPhysicalHostMemoryAndVirtualMemoryThenMapFails) { + memoryManager.reset(new MockWddmMemoryManager(false, false, executionEnvironment)); + AllocationData allocationData; + allocationData.allFlags = 0; + allocationData.size = MemoryConstants::pageSize; + allocationData.flags.allocateMemory = true; + allocationData.flags.useSystemMemory = true; + allocationData.flags.isUSMHostAllocation = true; + uint64_t gpuRange = 0x1234; + MemoryManager::AllocationStatus status; + auto allocation = memoryManager->allocatePhysicalHostMemory(allocationData, status); + EXPECT_EQ(nullptr, allocation); + + RootDeviceIndicesContainer rootDeviceIndices; + rootDeviceIndices.pushUnique(0); + rootDeviceIndices.pushUnique(1); + MultiGraphicsAllocation multiGraphicsAllocations{2}; + auto res = memoryManager->mapPhysicalHostMemoryToVirtualMemory(rootDeviceIndices, multiGraphicsAllocations, allocation, gpuRange, allocationData.size); + EXPECT_FALSE(res); + memoryManager->unMapPhysicalHostMemoryFromVirtualMemory(multiGraphicsAllocations, allocation, gpuRange, allocationData.size); +} + TEST_F(WddmMemoryManagerSimpleTest, givenZeroFenceValueOnSingleEngineRegisteredWhenHandleFenceCompletionIsCalledThenDoNotWaitOnCpu) { ASSERT_EQ(1u, memoryManager->getRegisteredEngines(0).size());