diff --git a/level_zero/core/source/context/context_imp.cpp b/level_zero/core/source/context/context_imp.cpp index ca2e7b2955..297de8c0eb 100644 --- a/level_zero/core/source/context/context_imp.cpp +++ b/level_zero/core/source/context/context_imp.cpp @@ -178,8 +178,8 @@ ze_result_t ContextImp::allocHostMem(const ze_host_mem_alloc_desc_t *hostDesc, return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - if (false == lookupTable.exportMemory) { - if (auto usmPtrFromPool = this->driverHandle->usmHostMemAllocPool.createUnifiedMemoryAllocation(size, unifiedMemoryProperties)) { + if (false == lookupTable.exportMemory && this->driverHandle->usmHostMemAllocPool) { + if (auto usmPtrFromPool = this->driverHandle->usmHostMemAllocPool->createUnifiedMemoryAllocation(size, unifiedMemoryProperties)) { *ptr = usmPtrFromPool; return ZE_RESULT_SUCCESS; } @@ -494,8 +494,9 @@ NEO::UsmMemAllocPool *ContextImp::getUsmPoolOwningPtr(const void *ptr, NEO::SvmA NEO::UsmMemAllocPool *usmPool = nullptr; if (InternalMemoryType::hostUnifiedMemory == svmData->memoryType && - driverHandle->usmHostMemAllocPool.isInPool(ptr)) { - usmPool = &driverHandle->usmHostMemAllocPool; + driverHandle->usmHostMemAllocPool && + driverHandle->usmHostMemAllocPool->isInPool(ptr)) { + usmPool = driverHandle->usmHostMemAllocPool.get(); } else if (InternalMemoryType::deviceUnifiedMemory == svmData->memoryType) { usmPool = svmData->device->getUsmPoolOwningPtr(ptr); } diff --git a/level_zero/core/source/driver/driver_handle_imp.cpp b/level_zero/core/source/driver/driver_handle_imp.cpp index 3a1bd42b49..f2d7919dbd 100644 --- a/level_zero/core/source/driver/driver_handle_imp.cpp +++ b/level_zero/core/source/driver/driver_handle_imp.cpp @@ -240,7 +240,9 @@ DriverHandleImp::~DriverHandleImp() { memoryManager->peekExecutionEnvironment().prepareForCleanup(); if (this->svmAllocsManager) { this->svmAllocsManager->cleanupUSMAllocCaches(); - this->usmHostMemAllocPool.cleanup(); + if (this->usmHostMemAllocPool) { + this->usmHostMemAllocPool->cleanup(); + } } } @@ -384,7 +386,8 @@ void DriverHandleImp::initHostUsmAllocPool() { if (usmHostAllocPoolingEnabled) { NEO::SVMAllocsManager::UnifiedMemoryProperties memoryProperties(InternalMemoryType::hostUnifiedMemory, MemoryConstants::pageSize2M, rootDeviceIndices, deviceBitfields); - usmHostMemAllocPool.initialize(svmAllocsManager, memoryProperties, poolParams.poolSize, poolParams.minServicedSize, poolParams.maxServicedSize); + usmHostMemAllocPool.reset(new NEO::UsmMemAllocPool); + usmHostMemAllocPool->initialize(svmAllocsManager, memoryProperties, poolParams.poolSize, poolParams.minServicedSize, poolParams.maxServicedSize); } } diff --git a/level_zero/core/source/driver/driver_handle_imp.h b/level_zero/core/source/driver/driver_handle_imp.h index 07c30a7423..be44139f76 100644 --- a/level_zero/core/source/driver/driver_handle_imp.h +++ b/level_zero/core/source/driver/driver_handle_imp.h @@ -137,7 +137,7 @@ struct DriverHandleImp : public DriverHandle { NEO::MemoryManager *memoryManager = nullptr; NEO::SVMAllocsManager *svmAllocsManager = nullptr; - NEO::UsmMemAllocPool usmHostMemAllocPool; + std::unique_ptr usmHostMemAllocPool; ze_context_handle_t defaultContext = nullptr; std::unique_ptr stagingBufferManager; diff --git a/level_zero/core/test/common/ult_helpers_l0.h b/level_zero/core/test/common/ult_helpers_l0.h index 11c1d76fc2..ffe822c73c 100644 --- a/level_zero/core/test/common/ult_helpers_l0.h +++ b/level_zero/core/test/common/ult_helpers_l0.h @@ -14,7 +14,10 @@ namespace L0 { namespace ult { struct L0UltHelper { static void cleanupUsmAllocPoolsAndReuse(DriverHandleImp *driverHandle) { - driverHandle->usmHostMemAllocPool.cleanup(); + if (driverHandle->usmHostMemAllocPool) { + driverHandle->usmHostMemAllocPool->cleanup(); + driverHandle->usmHostMemAllocPool.reset(nullptr); + } for (auto device : driverHandle->devices) { device->getNEODevice()->cleanupUsmAllocationPool(); } diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp index 523b398157..50c02b0cd7 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp @@ -539,13 +539,13 @@ TEST_F(MemoryTest, givenHostPointerThenDriverGetAllocPropertiesReturnsExpectedPr EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_EQ(memoryProperties.type, ZE_MEMORY_TYPE_HOST); - auto usmPool = &driverHandle->usmHostMemAllocPool; + auto usmPool = driverHandle->usmHostMemAllocPool.get(); auto alloc = context->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); EXPECT_NE(alloc, nullptr); EXPECT_NE(alloc->pageSizeForAlignment, 0u); EXPECT_EQ(alloc->pageSizeForAlignment, memoryProperties.pageSize); - if (usmPool->isInPool(ptr)) { + if (usmPool && usmPool->isInPool(ptr)) { EXPECT_EQ(memoryProperties.id, alloc->getAllocId()); } else { EXPECT_EQ(memoryProperties.id, @@ -2789,11 +2789,7 @@ TEST_F(FreeExtTests, TEST_F(FreeExtTests, whenAllocMemFailsWithDeferredFreeAllocationThenMemoryFreedAndRetrySucceeds) { // does not make sense for usm pooling, disable for test - driverHandle->usmHostMemAllocPool.cleanup(); - if (auto deviceUsmPool = neoDevice->getUsmMemAllocPool()) { - deviceUsmPool->cleanup(); - neoDevice->usmMemAllocPool.reset(nullptr); - } + L0UltHelper::cleanupUsmAllocPoolsAndReuse(driverHandle.get()); size_t size = 1024; size_t alignment = 1u; @@ -5792,7 +5788,7 @@ struct AllocHostMemoryTest : public ::testing::Test { TEST_F(AllocHostMemoryTest, whenCallingAllocHostMemThenAllocateGraphicsMemoryWithPropertiesIsCalledTheNumberOfTimesOfRootDevices) { - driverHandle->usmHostMemAllocPool.cleanup(); + L0UltHelper::cleanupUsmAllocPoolsAndReuse(driverHandle.get()); void *ptr = nullptr; static_cast(driverHandle->getMemoryManager())->isMockHostMemoryManager = true; @@ -5828,7 +5824,7 @@ TEST_F(AllocHostMemoryTest, TEST_F(AllocHostMemoryTest, whenCallingAllocHostMemAndFailingOnCreatingGraphicsAllocationWithHostPointerThenNullIsReturned) { - driverHandle->usmHostMemAllocPool.cleanup(); + L0UltHelper::cleanupUsmAllocPoolsAndReuse(driverHandle.get()); static_cast(driverHandle->getMemoryManager())->isMockHostMemoryManager = true; static_cast(driverHandle->getMemoryManager())->forceFailureInAllocationWithHostPointer = true; diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp index eb96e724b5..844c563289 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp @@ -20,6 +20,7 @@ #include "level_zero/core/source/device/device_imp.h" #include "level_zero/core/source/driver/driver_handle_imp.h" #include "level_zero/core/source/gfx_core_helpers/l0_gfx_core_helper.h" +#include "level_zero/core/test/common/ult_helpers_l0.h" #include "level_zero/core/test/unit_tests/mock.h" #include "level_zero/core/test/unit_tests/mocks/mock_driver_handle.h" namespace L0 { @@ -94,51 +95,57 @@ struct AllocUsmPoolMemoryTest : public ::testing::Test { using AllocUsmHostDefaultMemoryTest = AllocUsmPoolMemoryTest<-1, -1, 0, true>; TEST_F(AllocUsmHostDefaultMemoryTest, givenDriverHandleWhenCallinginitHostUsmAllocPoolThenInitIfEnabledForAllDevicesAndNoDebugger) { - driverHandle->usmHostMemAllocPool.cleanup(); + L0UltHelper::cleanupUsmAllocPoolsAndReuse(driverHandle.get()); mockProductHelpers[0]->isHostUsmPoolAllocatorSupportedResult = false; mockProductHelpers[1]->isHostUsmPoolAllocatorSupportedResult = false; driverHandle->initHostUsmAllocPool(); - EXPECT_FALSE(driverHandle->usmHostMemAllocPool.isInitialized()); + EXPECT_EQ(nullptr, driverHandle->usmHostMemAllocPool.get()); - driverHandle->usmHostMemAllocPool.cleanup(); + L0UltHelper::cleanupUsmAllocPoolsAndReuse(driverHandle.get()); mockProductHelpers[0]->isHostUsmPoolAllocatorSupportedResult = true; mockProductHelpers[1]->isHostUsmPoolAllocatorSupportedResult = false; driverHandle->initHostUsmAllocPool(); - EXPECT_FALSE(driverHandle->usmHostMemAllocPool.isInitialized()); + EXPECT_EQ(nullptr, driverHandle->usmHostMemAllocPool.get()); - driverHandle->usmHostMemAllocPool.cleanup(); + L0UltHelper::cleanupUsmAllocPoolsAndReuse(driverHandle.get()); mockProductHelpers[0]->isHostUsmPoolAllocatorSupportedResult = true; mockProductHelpers[1]->isHostUsmPoolAllocatorSupportedResult = true; driverHandle->initHostUsmAllocPool(); - EXPECT_TRUE(driverHandle->usmHostMemAllocPool.isInitialized()); + ASSERT_NE(nullptr, driverHandle->usmHostMemAllocPool.get()); + EXPECT_TRUE(driverHandle->usmHostMemAllocPool->isInitialized()); for (int32_t csrType = 0; csrType < static_cast(CommandStreamReceiverType::typesNum); ++csrType) { DebugManagerStateRestore restorer; debugManager.flags.SetCommandStreamReceiver.set(static_cast(csrType)); - driverHandle->usmHostMemAllocPool.cleanup(); + L0UltHelper::cleanupUsmAllocPoolsAndReuse(driverHandle.get()); mockProductHelpers[0]->isHostUsmPoolAllocatorSupportedResult = true; mockProductHelpers[1]->isHostUsmPoolAllocatorSupportedResult = true; driverHandle->initHostUsmAllocPool(); - EXPECT_EQ(NEO::DeviceFactory::isHwModeSelected(), driverHandle->usmHostMemAllocPool.isInitialized()); + if (NEO::DeviceFactory::isHwModeSelected()) { + ASSERT_NE(nullptr, driverHandle->usmHostMemAllocPool.get()); + EXPECT_TRUE(driverHandle->usmHostMemAllocPool->isInitialized()); + } else { + EXPECT_EQ(nullptr, driverHandle->usmHostMemAllocPool.get()); + } } - driverHandle->usmHostMemAllocPool.cleanup(); + L0UltHelper::cleanupUsmAllocPoolsAndReuse(driverHandle.get()); auto debuggerL0 = DebuggerL0::create(l0Devices[1]->getNEODevice()); executionEnvironment->rootDeviceEnvironments[1]->debugger.reset(debuggerL0.release()); driverHandle->initHostUsmAllocPool(); - EXPECT_FALSE(driverHandle->usmHostMemAllocPool.isInitialized()); + EXPECT_EQ(nullptr, driverHandle->usmHostMemAllocPool.get()); } using AllocUsmHostDisabledMemoryTest = AllocUsmPoolMemoryTest<0, -1>; TEST_F(AllocUsmHostDisabledMemoryTest, givenDriverHandleWhenCallingAllocHostMemThenDoNotUsePool) { - EXPECT_FALSE(driverHandle->usmHostMemAllocPool.isInitialized()); + EXPECT_EQ(nullptr, driverHandle->usmHostMemAllocPool.get()); void *ptr = nullptr; ze_host_mem_alloc_desc_t hostDesc = {}; ze_result_t result = context->allocHostMem(&hostDesc, 1u, 0u, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptr); - EXPECT_FALSE(driverHandle->usmHostMemAllocPool.isInitialized()); + EXPECT_EQ(nullptr, driverHandle->usmHostMemAllocPool.get()); result = context->freeMem(ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, result); } @@ -146,8 +153,9 @@ TEST_F(AllocUsmHostDisabledMemoryTest, givenDriverHandleWhenCallingAllocHostMemT using AllocUsmHostEnabledMemoryTest = AllocUsmPoolMemoryTest<1, -1, 0>; TEST_F(AllocUsmHostEnabledMemoryTest, givenDriverHandleWhenCallingAllocHostMemWithVariousParametersThenUsePoolIfAllowed) { - auto mockHostMemAllocPool = reinterpret_cast(&driverHandle->usmHostMemAllocPool); - EXPECT_TRUE(driverHandle->usmHostMemAllocPool.isInitialized()); + auto mockHostMemAllocPool = reinterpret_cast(driverHandle->usmHostMemAllocPool.get()); + ASSERT_NE(nullptr, driverHandle->usmHostMemAllocPool.get()); + EXPECT_TRUE(driverHandle->usmHostMemAllocPool->isInitialized()); auto poolAllocationData = driverHandle->svmAllocsManager->getSVMAlloc(mockHostMemAllocPool->pool); void *ptr1Byte = nullptr; @@ -155,7 +163,7 @@ TEST_F(AllocUsmHostEnabledMemoryTest, givenDriverHandleWhenCallingAllocHostMemWi ze_result_t result = context->allocHostMem(&hostDesc, 1u, 0u, &ptr1Byte); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptr1Byte); - EXPECT_TRUE(driverHandle->usmHostMemAllocPool.isInPool(ptr1Byte)); + EXPECT_TRUE(driverHandle->usmHostMemAllocPool->isInPool(ptr1Byte)); EXPECT_EQ(1u, mockHostMemAllocPool->allocations.getNumAllocs()); EXPECT_EQ(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptr1Byte)); result = context->freeMem(ptr1Byte); @@ -166,7 +174,7 @@ TEST_F(AllocUsmHostEnabledMemoryTest, givenDriverHandleWhenCallingAllocHostMemWi result = context->allocHostMem(&hostDesc, poolAllocationThreshold, 0u, &ptrThreshold); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptrThreshold); - EXPECT_TRUE(driverHandle->usmHostMemAllocPool.isInPool(ptrThreshold)); + EXPECT_TRUE(driverHandle->usmHostMemAllocPool->isInPool(ptrThreshold)); EXPECT_EQ(1u, mockHostMemAllocPool->allocations.getNumAllocs()); EXPECT_EQ(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptrThreshold)); result = context->freeMem(ptrThreshold); @@ -177,7 +185,7 @@ TEST_F(AllocUsmHostEnabledMemoryTest, givenDriverHandleWhenCallingAllocHostMemWi result = context->allocHostMem(&hostDesc, poolAllocationThreshold + 1u, 0u, &ptrOverThreshold); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptrOverThreshold); - EXPECT_FALSE(driverHandle->usmHostMemAllocPool.isInPool(ptrOverThreshold)); + EXPECT_FALSE(driverHandle->usmHostMemAllocPool->isInPool(ptrOverThreshold)); EXPECT_EQ(0u, mockHostMemAllocPool->allocations.getNumAllocs()); EXPECT_NE(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptrOverThreshold)); result = context->freeMem(ptrOverThreshold); @@ -188,7 +196,7 @@ TEST_F(AllocUsmHostEnabledMemoryTest, givenDriverHandleWhenCallingAllocHostMemWi result = context->allocHostMem(&hostDesc, poolAllocationThreshold, 0u, &ptrFreeMemExt); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptrFreeMemExt); - EXPECT_TRUE(driverHandle->usmHostMemAllocPool.isInPool(ptrFreeMemExt)); + EXPECT_TRUE(driverHandle->usmHostMemAllocPool->isInPool(ptrFreeMemExt)); EXPECT_EQ(1u, mockHostMemAllocPool->allocations.getNumAllocs()); EXPECT_EQ(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptrFreeMemExt)); ze_memory_free_ext_desc_t memFreeDesc = {}; @@ -205,7 +213,7 @@ TEST_F(AllocUsmHostEnabledMemoryTest, givenDriverHandleWhenCallingAllocHostMemWi result = context->allocHostMem(&hostDesc, poolAllocationThreshold, 0u, &ptrExportMemory); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptrExportMemory); - EXPECT_FALSE(driverHandle->usmHostMemAllocPool.isInPool(ptrExportMemory)); + EXPECT_FALSE(driverHandle->usmHostMemAllocPool->isInPool(ptrExportMemory)); EXPECT_EQ(0u, mockHostMemAllocPool->allocations.getNumAllocs()); EXPECT_NE(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptrExportMemory)); result = context->freeMem(ptrExportMemory); @@ -219,7 +227,7 @@ TEST_F(AllocUsmHostEnabledMemoryTest, givenDriverHandleWhenCallingAllocHostMemWi } TEST_F(AllocUsmHostEnabledMemoryTest, givenPooledAllocationWhenCallingGetMemAddressRangeThenCorrectValuesAreReturned) { - auto pool = &driverHandle->usmHostMemAllocPool; + auto pool = driverHandle->usmHostMemAllocPool.get(); { auto mockDeviceMemAllocPool = reinterpret_cast(pool); @@ -250,8 +258,9 @@ TEST_F(AllocUsmHostEnabledMemoryTest, givenPooledAllocationWhenCallingGetMemAddr } TEST_F(AllocUsmHostEnabledMemoryTest, givenDrmDriverModelWhenOpeningIpcHandleFromPooledAllocationThenOffsetIsApplied) { - auto mockHostMemAllocPool = reinterpret_cast(&driverHandle->usmHostMemAllocPool); - EXPECT_TRUE(driverHandle->usmHostMemAllocPool.isInitialized()); + ASSERT_NE(nullptr, driverHandle->usmHostMemAllocPool.get()); + auto mockHostMemAllocPool = reinterpret_cast(driverHandle->usmHostMemAllocPool.get()); + EXPECT_TRUE(driverHandle->usmHostMemAllocPool->isInitialized()); auto poolAllocationData = driverHandle->svmAllocsManager->getSVMAlloc(mockHostMemAllocPool->pool); executionEnvironment->rootDeviceEnvironments[0]->osInterface.reset(new NEO::OSInterface()); executionEnvironment->rootDeviceEnvironments[0]->osInterface->setDriverModel(std::make_unique()); @@ -261,7 +270,7 @@ TEST_F(AllocUsmHostEnabledMemoryTest, givenDrmDriverModelWhenOpeningIpcHandleFro ze_result_t result = context->allocHostMem(&hostDesc, 1u, 0u, &pooledAllocation); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, pooledAllocation); - EXPECT_TRUE(driverHandle->usmHostMemAllocPool.isInPool(pooledAllocation)); + EXPECT_TRUE(driverHandle->usmHostMemAllocPool->isInPool(pooledAllocation)); EXPECT_EQ(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(pooledAllocation)); const auto pooledAllocationOffset = ptrDiff(mockHostMemAllocPool->allocations.get(pooledAllocation)->address, castToUint64(mockHostMemAllocPool->pool)); EXPECT_NE(0u, pooledAllocationOffset); diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory_wddm.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory_wddm.cpp index 4dcf2a9f47..8917607e63 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory_wddm.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory_wddm.cpp @@ -387,8 +387,9 @@ struct HostUsmPoolMemoryOpenIpcHandleTest : public MemoryIPCTests { TEST_F(HostUsmPoolMemoryOpenIpcHandleTest, givenCallToOpenIpcMemHandleItIsSuccessfullyOpenedAndClosed) { - auto mockHostMemAllocPool = reinterpret_cast(&driverHandle->usmHostMemAllocPool); - EXPECT_TRUE(driverHandle->usmHostMemAllocPool.isInitialized()); + ASSERT_NE(nullptr, driverHandle->usmHostMemAllocPool.get()); + auto mockHostMemAllocPool = reinterpret_cast(driverHandle->usmHostMemAllocPool.get()); + EXPECT_TRUE(driverHandle->usmHostMemAllocPool->isInitialized()); size_t size = 1; size_t alignment = 0u; void *ptr = nullptr; @@ -398,7 +399,7 @@ TEST_F(HostUsmPoolMemoryOpenIpcHandleTest, size, alignment, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptr); - EXPECT_TRUE(driverHandle->usmHostMemAllocPool.isInPool(ptr)); + EXPECT_TRUE(driverHandle->usmHostMemAllocPool->isInPool(ptr)); const auto pooledAllocationOffset = ptrDiff(mockHostMemAllocPool->allocations.get(ptr)->address, castToUint64(mockHostMemAllocPool->pool)); EXPECT_GT(pooledAllocationOffset, 0u);