diff --git a/level_zero/core/source/context/context_imp.h b/level_zero/core/source/context/context_imp.h index e23f0483a0..c1e7313032 100644 --- a/level_zero/core/source/context/context_imp.h +++ b/level_zero/core/source/context/context_imp.h @@ -222,6 +222,7 @@ struct ContextImp : Context, NEO::NonCopyableAndNonMovableClass { return numDevices; } ze_result_t systemBarrier(ze_device_handle_t hDevice) override; + NEO::UsmMemAllocPool *getUsmPoolOwningPtr(const void *ptr, NEO::SvmAllocationData *svmData); protected: ze_result_t getIpcMemHandlesImpl(const void *ptr, uint32_t *numIpcHandles, ze_ipc_mem_handle_t *pIpcHandles); @@ -276,7 +277,6 @@ struct ContextImp : Context, NEO::NonCopyableAndNonMovableClass { } size_t getPageAlignedSizeRequired(const void *pStart, size_t size, NEO::HeapIndex *heapRequired, size_t *pageSizeRequired); - NEO::UsmMemAllocPool *getUsmPoolOwningPtr(const void *ptr, NEO::SvmAllocationData *svmData); bool tryFreeViaPooling(const void *ptr, NEO::SvmAllocationData *svmData, NEO::UsmMemAllocPool *usmPool); std::map devices; diff --git a/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.cpp b/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.cpp index 4d2966122a..b9ecf12090 100644 --- a/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.cpp +++ b/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.cpp @@ -8,7 +8,6 @@ #include "level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h" #include "shared/source/gmm_helper/gmm_helper.h" -#include "shared/test/common/helpers/debug_manager_state_restore.h" #include "shared/test/common/mocks/mock_device.h" #include "shared/test/common/mocks/mock_memory_manager.h" @@ -503,6 +502,9 @@ NEO::GraphicsAllocation *MemoryManagerIpcImplicitScalingMock::createGraphicsAllo } void MemoryExportImportImplicitScalingTest::SetUp() { + // flags to use during test run + debugManager.flags.EnableDeviceUsmAllocationPool.set(0); + debugManager.flags.EnableHostUsmAllocationPool.set(0); DebugManagerStateRestore restorer; debugManager.flags.EnableImplicitScaling.set(1); debugManager.flags.EnableWalkerPartition.set(1); @@ -534,7 +536,6 @@ void MemoryExportImportImplicitScalingTest::SetUp() { } void MemoryExportImportImplicitScalingTest::TearDown() { - L0UltHelper::cleanupUsmAllocPoolsAndReuse(driverHandle.get()); driverHandle->svmAllocsManager = prevSvmAllocsManager; delete currSvmAllocsManager; driverHandle->setMemoryManager(prevMemoryManager); diff --git a/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h b/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h index 0014af7585..3dfbf7715f 100644 --- a/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h +++ b/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h @@ -11,6 +11,7 @@ #include "shared/source/memory_manager/gfx_partition.h" #include "shared/source/memory_manager/memory_allocation.h" #include "shared/source/memory_manager/memory_manager.h" +#include "shared/test/common/helpers/debug_manager_state_restore.h" #include "level_zero/core/source/context/context_imp.h" #include "level_zero/core/source/driver/driver_handle_imp.h" @@ -438,6 +439,7 @@ struct MemoryExportImportImplicitScalingTest : public ::testing::Test { NEO::MockDevice *neoDevice = nullptr; L0::Device *device = nullptr; std::unique_ptr context; + DebugManagerStateRestore restorer; }; struct MemoryGetIpcHandlePidfdTest : public ::testing::Test { diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_5.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_5.cpp index 1dd7af804d..acfc89b171 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_5.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_5.cpp @@ -119,6 +119,8 @@ HWTEST_F(AppendQueryKernelTimestamps, givenCommandListWhenAppendQueryKernelTimes } HWTEST_F(AppendQueryKernelTimestamps, givenCommandListWhenAppendQueryKernelTimestampsWithOffsetsThenProperBuiltinWasAdded) { + DebugManagerStateRestore restorer; + NEO::debugManager.flags.EnableDeviceUsmAllocationPool.set(0); auto testDevice = std::make_unique(device->getNEODevice(), driverHandle.get()); testDevice->builtins.reset(new MockBuiltinFunctionsLibImplTimestamps(testDevice.get(), testDevice->getNEODevice()->getBuiltIns())); testDevice->getBuiltinFunctionsLib()->initBuiltinKernel(L0::Builtin::queryKernelTimestamps); diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_memory_extension.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_memory_extension.cpp index 2e4f6e706e..e6a2d73d00 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_memory_extension.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_memory_extension.cpp @@ -30,6 +30,8 @@ namespace ult { class CommandListWaitOnMemFixture : public DeviceFixture { public: void setUp() { + DebugManagerStateRestore restorer; + NEO::debugManager.flags.EnableDeviceUsmAllocationPool.set(0); DeviceFixture::setUp(); ze_result_t returnValue; commandList.reset(CommandList::whiteboxCast(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false))); @@ -954,6 +956,8 @@ HWTEST_F(CommandListAppendWriteToMem, givenAppendWriteToMemWithScopeThenPipeCont class ImmediateCommandListWaitOnMemFixture : public DeviceFixture { public: void setUp() { + DebugManagerStateRestore restorer; + NEO::debugManager.flags.EnableDeviceUsmAllocationPool.set(0); DeviceFixture::setUp(); ze_result_t returnValue; ze_command_queue_desc_t queueDesc{}; diff --git a/level_zero/core/test/unit_tests/sources/driver/test_driver.cpp b/level_zero/core/test/unit_tests/sources/driver/test_driver.cpp index 7318afe93c..455ef2c8b5 100644 --- a/level_zero/core/test/unit_tests/sources/driver/test_driver.cpp +++ b/level_zero/core/test/unit_tests/sources/driver/test_driver.cpp @@ -141,6 +141,8 @@ TEST_F(DriverHandleImpTest, givenDriverImpWhenCallingupdateRootDeviceBitFieldsTh } TEST_F(DriverHandleImpTest, givenDriverWhenFindAllocationDataForRangeWithDifferentAllocationsThenReturnFailure) { + DebugManagerStateRestore restorer; + NEO::debugManager.flags.EnableDeviceUsmAllocationPool.set(0); ze_device_mem_alloc_desc_t devDesc = {}; devDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; void *ptr1 = nullptr; diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp index 0fb3cbb20e..ae0d4b5dc7 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp @@ -505,11 +505,11 @@ TEST_F(MemoryTest, givenDevicePointerThenDriverGetAllocPropertiesReturnsExpected EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_EQ(memoryProperties.type, ZE_MEMORY_TYPE_DEVICE); EXPECT_EQ(deviceHandle, device->toHandle()); - auto usmPool = device->getNEODevice()->getUsmMemAllocPool(); auto alloc = context->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); EXPECT_NE(alloc, nullptr); EXPECT_NE(alloc->pageSizeForAlignment, 0u); EXPECT_EQ(alloc->pageSizeForAlignment, memoryProperties.pageSize); + auto usmPool = context->getUsmPoolOwningPtr(ptr, alloc); if (usmPool && usmPool->isInPool(ptr)) { @@ -703,6 +703,8 @@ TEST_F(MemoryTest, givenForceExtendedUSMBufferSizeDebugFlagWhenUSMAllocationIsCr } TEST_F(MemoryTest, givenHostPointerThenDriverGetAllocPropertiesReturnsMemoryId) { + DebugManagerStateRestore restorer; + NEO::debugManager.flags.EnableHostUsmAllocationPool.set(0); size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; @@ -712,6 +714,9 @@ TEST_F(MemoryTest, givenHostPointerThenDriverGetAllocPropertiesReturnsMemoryId) size, alignment, &ptr); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptr); + auto alloc = context->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr); + EXPECT_NE(alloc, nullptr); + auto usmPool = context->getUsmPoolOwningPtr(ptr, alloc); ze_memory_allocation_properties_t memoryProperties = {}; ze_device_handle_t deviceHandle; @@ -721,8 +726,13 @@ TEST_F(MemoryTest, givenHostPointerThenDriverGetAllocPropertiesReturnsMemoryId) EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_EQ(memoryProperties.type, ZE_MEMORY_TYPE_HOST); EXPECT_EQ(deviceHandle, nullptr); - EXPECT_EQ(memoryProperties.id, - context->getDriverHandle()->getSvmAllocsManager()->allocationsCounter); + if (usmPool && + usmPool->isInPool(ptr)) { + EXPECT_EQ(memoryProperties.id, alloc->getAllocId()); + } else { + EXPECT_EQ(memoryProperties.id, + context->getDriverHandle()->getSvmAllocsManager()->allocationsCounter); + } result = context->freeMem(ptr); ASSERT_EQ(result, ZE_RESULT_SUCCESS); @@ -2454,6 +2464,8 @@ TEST_F(FreeExtTests, TEST_F(FreeExtTests, whenFreeMemExtIsCalledWithBlockingFreePolicyThenBlockingCallIsMade) { + DebugManagerStateRestore restorer; + NEO::debugManager.flags.EnableHostUsmAllocationPool.set(0); size_t size = 1024; size_t alignment = 1u; void *ptr = nullptr; @@ -2474,6 +2486,8 @@ TEST_F(FreeExtTests, TEST_F(FreeExtTests, whenFreeMemExtIsCalledWithDeferFreePolicyThenBlockingCallIsNotMade) { + DebugManagerStateRestore restorer; + NEO::debugManager.flags.EnableHostUsmAllocationPool.set(0); size_t size = 1024; size_t alignment = 1u; void *ptr = nullptr; @@ -2505,6 +2519,8 @@ TEST_F(FreeExtTests, TEST_F(FreeExtTests, whenFreeMemExtIsCalledWithDeferFreePolicyAndAllocationNotInUseThenMemoryFreeNotDeferred) { + DebugManagerStateRestore restorer; + NEO::debugManager.flags.EnableHostUsmAllocationPool.set(0); size_t size = 1024; size_t alignment = 1u; void *ptr = nullptr; @@ -2528,6 +2544,8 @@ TEST_F(FreeExtTests, TEST_F(FreeExtTests, whenFreeMemExtIsCalledWithDeferFreePolicyAndAllocationInUseThenMemoryFreeDeferred) { + DebugManagerStateRestore restorer; + NEO::debugManager.flags.EnableHostUsmAllocationPool.set(0); size_t size = 1024; size_t alignment = 1u; void *ptr = nullptr; @@ -2582,6 +2600,8 @@ TEST_F(FreeExtTests, TEST_F(FreeExtTests, whenFreeMemExtIsCalledMultipleTimesForSameAllocationWithDeferFreePolicyAndAllocationInUseThenMemoryFreeDeferredOnlyOnce) { + DebugManagerStateRestore restorer; + NEO::debugManager.flags.EnableHostUsmAllocationPool.set(0); size_t size = 1024; size_t alignment = 1u; void *ptr = nullptr; @@ -2621,6 +2641,8 @@ TEST_F(FreeExtTests, TEST_F(FreeExtTests, whenFreeMemIsCalledWithDeferredFreeAllocationThenMemoryFreed) { + DebugManagerStateRestore restorer; + NEO::debugManager.flags.EnableHostUsmAllocationPool.set(0); size_t size = 1024; size_t alignment = 1u; void *ptr = nullptr; @@ -2654,6 +2676,9 @@ TEST_F(FreeExtTests, TEST_F(FreeExtTests, whenAllocMemFailsWithDeferredFreeAllocationThenMemoryFreed) { + DebugManagerStateRestore restorer; + NEO::debugManager.flags.EnableHostUsmAllocationPool.set(0); + NEO::debugManager.flags.EnableDeviceUsmAllocationPool.set(0); size_t size = 1024; size_t alignment = 1u; void *ptr = nullptr; @@ -2741,6 +2766,7 @@ TEST_F(FreeExtTests, // does not make sense for usm pooling, disable for test DebugManagerStateRestore restorer; NEO::debugManager.flags.EnableDeviceUsmAllocationPool.set(0); + NEO::debugManager.flags.EnableHostUsmAllocationPool.set(0); size_t size = 1024; size_t alignment = 1u; @@ -2823,6 +2849,8 @@ TEST_F(FreeExtTests, TEST_F(FreeExtTests, whenDestroyContextAnyRemainingDeferFreeMemoryAllocationsAreFreed) { + DebugManagerStateRestore restorer; + NEO::debugManager.flags.EnableHostUsmAllocationPool.set(0); size_t size = 1024; size_t alignment = 1u; void *ptr = nullptr; diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp index 982e5a48c9..7ba3923a70 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp @@ -598,7 +598,10 @@ TEST_F(AllocUsmDeviceEnabledSinglePoolMemoryTest, givenDrmDriverModelWhenOpening void *allocation = nullptr; ze_device_mem_alloc_desc_t deviceDesc = {}; + auto mockDevice = static_cast(l0Devices[0]->getNEODevice()); + auto tempDeviceMemAllocPool = mockDevice->usmMemAllocPool.release(); ze_result_t result = context->allocDeviceMem(l0Devices[0], &deviceDesc, 1u, 1u, &allocation); + mockDevice->usmMemAllocPool.reset(tempDeviceMemAllocPool); auto allocationData = driverHandle->svmAllocsManager->getSVMAlloc(mockDeviceMemAllocPool->pool); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, allocation); diff --git a/shared/source/memory_manager/unified_memory_pooling.cpp b/shared/source/memory_manager/unified_memory_pooling.cpp index 9e53c58b2d..69071b8057 100644 --- a/shared/source/memory_manager/unified_memory_pooling.cpp +++ b/shared/source/memory_manager/unified_memory_pooling.cpp @@ -70,7 +70,7 @@ void UsmMemAllocPool::cleanup() { } bool UsmMemAllocPool::alignmentIsAllowed(size_t alignment) { - return alignment % chunkAlignment == 0 && alignment <= poolAlignment; + return alignment <= poolAlignment; } bool UsmMemAllocPool::sizeIsAllowed(size_t size) { @@ -83,7 +83,8 @@ bool UsmMemAllocPool::flagsAreAllowed(const UnifiedMemoryProperties &memoryPrope flagsWithoutCompression.flags.uncompressedHint = 0u; return flagsWithoutCompression.allFlags == 0u && - memoryProperties.allocationFlags.allAllocFlags == 0u; + memoryProperties.allocationFlags.allAllocFlags == 0u && + memoryProperties.allocationFlags.hostptr == 0u; } double UsmMemAllocPool::getPercentOfFreeMemoryForRecycling(InternalMemoryType memoryType) { diff --git a/shared/test/unit_test/memory_manager/unified_memory_pooling_tests.cpp b/shared/test/unit_test/memory_manager/unified_memory_pooling_tests.cpp index f6abc4f07d..bdf45b3552 100644 --- a/shared/test/unit_test/memory_manager/unified_memory_pooling_tests.cpp +++ b/shared/test/unit_test/memory_manager/unified_memory_pooling_tests.cpp @@ -29,10 +29,11 @@ TEST_F(UnifiedMemoryPoolingStaticTest, givenUsmAllocPoolWhenCallingStaticMethods EXPECT_EQ(0.02, UsmMemAllocPool::getPercentOfFreeMemoryForRecycling(InternalMemoryType::hostUnifiedMemory)); EXPECT_EQ(0.00, UsmMemAllocPool::getPercentOfFreeMemoryForRecycling(InternalMemoryType::sharedUnifiedMemory)); + EXPECT_TRUE(UsmMemAllocPool::alignmentIsAllowed(UsmMemAllocPool::chunkAlignment / 2)); EXPECT_TRUE(UsmMemAllocPool::alignmentIsAllowed(UsmMemAllocPool::chunkAlignment)); EXPECT_TRUE(UsmMemAllocPool::alignmentIsAllowed(UsmMemAllocPool::chunkAlignment * 2)); - EXPECT_FALSE(UsmMemAllocPool::alignmentIsAllowed(UsmMemAllocPool::chunkAlignment / 2)); - EXPECT_FALSE(UsmMemAllocPool::alignmentIsAllowed(UsmMemAllocPool::poolAlignment + UsmMemAllocPool::chunkAlignment)); + EXPECT_TRUE(UsmMemAllocPool::alignmentIsAllowed(UsmMemAllocPool::poolAlignment)); + EXPECT_FALSE(UsmMemAllocPool::alignmentIsAllowed(UsmMemAllocPool::poolAlignment * 2)); const RootDeviceIndicesContainer rootDeviceIndices; const std::map deviceBitfields; @@ -44,6 +45,10 @@ TEST_F(UnifiedMemoryPoolingStaticTest, givenUsmAllocPoolWhenCallingStaticMethods unifiedMemoryProperties.allocationFlags.allFlags = 0u; unifiedMemoryProperties.allocationFlags.allAllocFlags = 1u; EXPECT_FALSE(UsmMemAllocPool::flagsAreAllowed(unifiedMemoryProperties)); + unifiedMemoryProperties.allocationFlags.allFlags = 0u; + unifiedMemoryProperties.allocationFlags.allAllocFlags = 0u; + unifiedMemoryProperties.allocationFlags.hostptr = 0x1u; + EXPECT_FALSE(UsmMemAllocPool::flagsAreAllowed(unifiedMemoryProperties)); } using UnifiedMemoryPoolingTest = Test>; @@ -276,7 +281,7 @@ TEST_F(InitializedHostUnifiedMemoryPoolingTest, givenDifferentAllocationSizesWhe EXPECT_FALSE(usmMemAllocPool.canBePooled(poolAllocationThreshold + 1, memoryProperties)); memoryProperties.allocationFlags.allAllocFlags = 0u; - constexpr auto notAllowedAlignment = UsmMemAllocPool::chunkAlignment / 2; + constexpr auto notAllowedAlignment = UsmMemAllocPool::poolAlignment * 2; memoryProperties.alignment = notAllowedAlignment; EXPECT_FALSE(usmMemAllocPool.canBePooled(poolAllocationThreshold, memoryProperties)); EXPECT_FALSE(usmMemAllocPool.canBePooled(poolAllocationThreshold + 1, memoryProperties)); @@ -300,10 +305,11 @@ TEST_F(InitializedHostUnifiedMemoryPoolingTest, givenVariousPointersWhenCallingI } TEST_F(InitializedHostUnifiedMemoryPoolingTest, givenAlignmentsWhenCallingAlignmentIsAllowedThenCorrectValueIsReturned) { - EXPECT_FALSE(usmMemAllocPool.alignmentIsAllowed(UsmMemAllocPool::chunkAlignment / 2)); + EXPECT_TRUE(usmMemAllocPool.alignmentIsAllowed(UsmMemAllocPool::chunkAlignment / 2)); EXPECT_TRUE(usmMemAllocPool.alignmentIsAllowed(UsmMemAllocPool::chunkAlignment)); - EXPECT_FALSE(usmMemAllocPool.alignmentIsAllowed(UsmMemAllocPool::chunkAlignment + UsmMemAllocPool::chunkAlignment / 2)); EXPECT_TRUE(usmMemAllocPool.alignmentIsAllowed(UsmMemAllocPool::chunkAlignment * 2)); + EXPECT_TRUE(usmMemAllocPool.alignmentIsAllowed(UsmMemAllocPool::poolAlignment)); + EXPECT_FALSE(usmMemAllocPool.alignmentIsAllowed(UsmMemAllocPool::poolAlignment * 2)); } TEST_F(InitializedHostUnifiedMemoryPoolingTest, givenPoolableAllocationWhenUsingPoolThenAllocationIsPooledUnlessPoolIsFull) { @@ -512,7 +518,7 @@ TEST_P(UnifiedMemoryPoolingManagerTest, givenUsmMemAllocPoolsManagerWhenCallingC EXPECT_TRUE(usmMemAllocPoolsManager->canBePooled(maxPoolableSize, unifiedMemoryProperties)); EXPECT_FALSE(usmMemAllocPoolsManager->canBePooled(maxPoolableSize + 1, unifiedMemoryProperties)); - unifiedMemoryProperties.alignment = UsmMemAllocPool::chunkAlignment / 2; + unifiedMemoryProperties.alignment = UsmMemAllocPool::poolAlignment * 2; EXPECT_FALSE(usmMemAllocPoolsManager->canBePooled(maxPoolableSize, unifiedMemoryProperties)); unifiedMemoryProperties.alignment = UsmMemAllocPool::chunkAlignment;