From f7939735dacab03bfcf5d9da50030e5e70f5bda0 Mon Sep 17 00:00:00 2001 From: Lukasz Jobczyk Date: Mon, 14 Apr 2025 15:44:58 +0000 Subject: [PATCH] refactor: Adjust USM cleaner to ULLS light Related-To: NEO-13922 Signed-off-by: Lukasz Jobczyk --- .../unified_memory_reuse_cleaner_tests_mt.cpp | 4 +- .../execution_environment.cpp | 6 +-- .../execution_environment.h | 2 +- .../memory_manager/unified_memory_manager.cpp | 4 +- .../unified_memory_reuse_cleaner.cpp | 4 +- .../unified_memory_reuse_cleaner.h | 4 +- .../mocks/mock_usm_memory_reuse_cleaner.h | 1 + .../execution_environment_tests.cpp | 2 +- .../unified_memory_manager_cache_tests.cpp | 45 ++++++++++++++++++- 9 files changed, 58 insertions(+), 14 deletions(-) diff --git a/opencl/test/unit_test/mt_tests/memory_manager/unified_memory_reuse_cleaner_tests_mt.cpp b/opencl/test/unit_test/mt_tests/memory_manager/unified_memory_reuse_cleaner_tests_mt.cpp index a545fa063f..97fcb2ca5e 100644 --- a/opencl/test/unit_test/mt_tests/memory_manager/unified_memory_reuse_cleaner_tests_mt.cpp +++ b/opencl/test/unit_test/mt_tests/memory_manager/unified_memory_reuse_cleaner_tests_mt.cpp @@ -10,7 +10,7 @@ namespace NEO { TEST(UnifiedMemoryReuseCleanerTestsMt, givenUnifiedMemoryReuseCleanerWhenSleepExpiredThenTrimOldInCachesIsCalled) { - MockUnifiedMemoryReuseCleaner cleaner; + MockUnifiedMemoryReuseCleaner cleaner(false); cleaner.callBaseStartThread = true; cleaner.callBaseTrimOldInCaches = false; EXPECT_EQ(nullptr, cleaner.unifiedMemoryReuseCleanerThread); @@ -33,7 +33,7 @@ TEST(UnifiedMemoryReuseCleanerTestsMt, givenUnifiedMemoryReuseCleanerWhenSleepEx } TEST(UnifiedMemoryReuseCleanerTestsMt, givenUnifiedMemoryReuseCleanerWithNotStartedCleaningWhenShuttingDownThenNoHang) { - MockUnifiedMemoryReuseCleaner cleaner; + MockUnifiedMemoryReuseCleaner cleaner(false); cleaner.callBaseStartThread = true; cleaner.callBaseTrimOldInCaches = false; cleaner.startThread(); diff --git a/shared/source/execution_environment/execution_environment.cpp b/shared/source/execution_environment/execution_environment.cpp index 1e979ba400..799580fe67 100644 --- a/shared/source/execution_environment/execution_environment.cpp +++ b/shared/source/execution_environment/execution_environment.cpp @@ -152,16 +152,16 @@ DirectSubmissionController *ExecutionEnvironment::initializeDirectSubmissionCont return directSubmissionController.get(); } -void ExecutionEnvironment::initializeUnifiedMemoryReuseCleaner(bool enable) { +void ExecutionEnvironment::initializeUnifiedMemoryReuseCleaner(bool isAnyDirectSubmissionLightEnabled) { std::lock_guard lock(initializeUnifiedMemoryReuseCleanerMutex); - auto initializeUnifiedMemoryReuseCleaner = UnifiedMemoryReuseCleaner::isSupported() && enable; + auto initializeUnifiedMemoryReuseCleaner = UnifiedMemoryReuseCleaner::isSupported() && !isAnyDirectSubmissionLightEnabled; if (debugManager.flags.ExperimentalUSMAllocationReuseCleaner.get() != -1) { initializeUnifiedMemoryReuseCleaner = debugManager.flags.ExperimentalUSMAllocationReuseCleaner.get() == 1; } if (initializeUnifiedMemoryReuseCleaner && nullptr == this->unifiedMemoryReuseCleaner) { - this->unifiedMemoryReuseCleaner = std::make_unique(); + this->unifiedMemoryReuseCleaner = std::make_unique(isAnyDirectSubmissionLightEnabled); this->unifiedMemoryReuseCleaner->startThread(); } } diff --git a/shared/source/execution_environment/execution_environment.h b/shared/source/execution_environment/execution_environment.h index 2a9bdd63f2..989f7171cd 100644 --- a/shared/source/execution_environment/execution_environment.h +++ b/shared/source/execution_environment/execution_environment.h @@ -66,7 +66,7 @@ class ExecutionEnvironment : public ReferenceTrackedObject bool isFP64EmulationEnabled() const { return fp64EmulationEnabled; } DirectSubmissionController *initializeDirectSubmissionController(); - void initializeUnifiedMemoryReuseCleaner(bool enable); + void initializeUnifiedMemoryReuseCleaner(bool isAnyDirectSubmissionLightEnabled); std::unique_ptr memoryManager; std::unique_ptr unifiedMemoryReuseCleaner; diff --git a/shared/source/memory_manager/unified_memory_manager.cpp b/shared/source/memory_manager/unified_memory_manager.cpp index 7263a2b1e2..808b5aa12b 100644 --- a/shared/source/memory_manager/unified_memory_manager.cpp +++ b/shared/source/memory_manager/unified_memory_manager.cpp @@ -936,7 +936,7 @@ void SVMAllocsManager::initUsmAllocationsCaches(Device &device) { usmDeviceAllocationsCacheEnabled = !!debugManager.flags.ExperimentalEnableDeviceAllocationCache.get(); } if (usmDeviceAllocationsCacheEnabled) { - device.getExecutionEnvironment()->initializeUnifiedMemoryReuseCleaner(!device.isAnyDirectSubmissionLightEnabled()); + device.getExecutionEnvironment()->initializeUnifiedMemoryReuseCleaner(device.isAnyDirectSubmissionLightEnabled()); this->initUsmDeviceAllocationsCache(device); } @@ -945,7 +945,7 @@ void SVMAllocsManager::initUsmAllocationsCaches(Device &device) { usmHostAllocationsCacheEnabled = !!debugManager.flags.ExperimentalEnableHostAllocationCache.get(); } if (usmHostAllocationsCacheEnabled) { - device.getExecutionEnvironment()->initializeUnifiedMemoryReuseCleaner(!device.isAnyDirectSubmissionLightEnabled()); + device.getExecutionEnvironment()->initializeUnifiedMemoryReuseCleaner(device.isAnyDirectSubmissionLightEnabled()); this->initUsmHostAllocationsCache(); } } diff --git a/shared/source/memory_manager/unified_memory_reuse_cleaner.cpp b/shared/source/memory_manager/unified_memory_reuse_cleaner.cpp index aa83e9f413..7b2254b79c 100644 --- a/shared/source/memory_manager/unified_memory_reuse_cleaner.cpp +++ b/shared/source/memory_manager/unified_memory_reuse_cleaner.cpp @@ -15,7 +15,7 @@ #include namespace NEO { -UnifiedMemoryReuseCleaner::UnifiedMemoryReuseCleaner() { +UnifiedMemoryReuseCleaner::UnifiedMemoryReuseCleaner(bool trimAllAllocations) : trimAllAllocations(trimAllAllocations) { } UnifiedMemoryReuseCleaner::~UnifiedMemoryReuseCleaner() { @@ -75,7 +75,7 @@ void UnifiedMemoryReuseCleaner::trimOldInCaches() { } } } - svmAllocCache->trimOldAllocs(trimTimePoint, shouldLimitReuse); + svmAllocCache->trimOldAllocs(trimTimePoint, shouldLimitReuse || this->trimAllAllocations); } } diff --git a/shared/source/memory_manager/unified_memory_reuse_cleaner.h b/shared/source/memory_manager/unified_memory_reuse_cleaner.h index 179ac2e27e..485e585f04 100644 --- a/shared/source/memory_manager/unified_memory_reuse_cleaner.h +++ b/shared/source/memory_manager/unified_memory_reuse_cleaner.h @@ -23,7 +23,7 @@ class UnifiedMemoryReuseCleaner : NEO::NonCopyableAndNonMovableClass { static constexpr auto sleepTime = std::chrono::milliseconds(15u); static constexpr auto maxHoldTime = std::chrono::seconds(10u); static constexpr auto limitedHoldTime = std::chrono::seconds(2u); - UnifiedMemoryReuseCleaner(); + UnifiedMemoryReuseCleaner(bool trimAllAllocations); virtual ~UnifiedMemoryReuseCleaner(); MOCKABLE_VIRTUAL void startThread(); @@ -45,6 +45,8 @@ class UnifiedMemoryReuseCleaner : NEO::NonCopyableAndNonMovableClass { std::atomic_bool runCleaning = false; std::atomic_bool keepCleaning = true; + + bool trimAllAllocations = false; }; static_assert(NEO::NonCopyableAndNonMovable); diff --git a/shared/test/common/mocks/mock_usm_memory_reuse_cleaner.h b/shared/test/common/mocks/mock_usm_memory_reuse_cleaner.h index 2e646d05f6..57a4813388 100644 --- a/shared/test/common/mocks/mock_usm_memory_reuse_cleaner.h +++ b/shared/test/common/mocks/mock_usm_memory_reuse_cleaner.h @@ -13,6 +13,7 @@ struct MockUnifiedMemoryReuseCleaner : public UnifiedMemoryReuseCleaner { using UnifiedMemoryReuseCleaner::keepCleaning; using UnifiedMemoryReuseCleaner::runCleaning; using UnifiedMemoryReuseCleaner::svmAllocationCaches; + using UnifiedMemoryReuseCleaner::UnifiedMemoryReuseCleaner; using UnifiedMemoryReuseCleaner::unifiedMemoryReuseCleanerThread; void trimOldInCaches() override { diff --git a/shared/test/unit_test/execution_environment/execution_environment_tests.cpp b/shared/test/unit_test/execution_environment/execution_environment_tests.cpp index 62cb1a8623..c4ab6ae5ca 100644 --- a/shared/test/unit_test/execution_environment/execution_environment_tests.cpp +++ b/shared/test/unit_test/execution_environment/execution_environment_tests.cpp @@ -431,7 +431,7 @@ TEST(ExecutionEnvironment, givenExecutionEnvironmentWithVariousMembersWhenItIsDe } }; struct UnifiedMemoryReuseCleanerMock : public DestructorCounted { - UnifiedMemoryReuseCleanerMock(uint32_t &destructorId) : DestructorCounted(destructorId) {} + UnifiedMemoryReuseCleanerMock(uint32_t &destructorId) : DestructorCounted(destructorId, false) {} }; struct DirectSubmissionControllerMock : public DestructorCounted { DirectSubmissionControllerMock(uint32_t &destructorId) : DestructorCounted(destructorId) {} diff --git a/shared/test/unit_test/memory_manager/unified_memory_manager_cache_tests.cpp b/shared/test/unit_test/memory_manager/unified_memory_manager_cache_tests.cpp index 581cffa93f..8cbdb73289 100644 --- a/shared/test/unit_test/memory_manager/unified_memory_manager_cache_tests.cpp +++ b/shared/test/unit_test/memory_manager/unified_memory_manager_cache_tests.cpp @@ -911,7 +911,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenUsmReuseCleanerWhenTrimOldInCachesCall debugManager.flags.ExperimentalEnableHostAllocationCache.set(0); auto device = deviceFactory->rootDevices[0]; auto svmManager = std::make_unique(device->getMemoryManager(), false); - device->executionEnvironment->unifiedMemoryReuseCleaner.reset(new MockUnifiedMemoryReuseCleaner); + device->executionEnvironment->unifiedMemoryReuseCleaner.reset(new MockUnifiedMemoryReuseCleaner(false)); auto mockUnifiedMemoryReuseCleaner = reinterpret_cast(device->executionEnvironment->unifiedMemoryReuseCleaner.get()); EXPECT_EQ(0u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size()); device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); @@ -955,6 +955,47 @@ TEST_F(SvmDeviceAllocationCacheTest, givenUsmReuseCleanerWhenTrimOldInCachesCall EXPECT_EQ(0u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size()); } +TEST_F(SvmDeviceAllocationCacheTest, givenDirectSubmissionLightWhenTrimOldInCachesCalledThenAllOldAllocationsAreRemoved) { + std::unique_ptr deviceFactory(new UltDeviceFactory(1, 1)); + RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex}; + std::map deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}}; + DebugManagerStateRestore restore; + debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1); + debugManager.flags.ExperimentalEnableHostAllocationCache.set(0); + auto device = deviceFactory->rootDevices[0]; + auto svmManager = std::make_unique(device->getMemoryManager(), false); + device->executionEnvironment->unifiedMemoryReuseCleaner.reset(new MockUnifiedMemoryReuseCleaner(true)); + auto mockUnifiedMemoryReuseCleaner = reinterpret_cast(device->executionEnvironment->unifiedMemoryReuseCleaner.get()); + EXPECT_EQ(0u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size()); + device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); + svmManager->initUsmAllocationsCaches(*device); + EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache); + EXPECT_EQ(1u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size()); + EXPECT_EQ(svmManager->usmDeviceAllocationsCache.get(), mockUnifiedMemoryReuseCleaner->svmAllocationCaches[0]); + + SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::deviceUnifiedMemory, 1, rootDeviceIndices, deviceBitfields); + unifiedMemoryProperties.device = device; + auto allocation = svmManager->createUnifiedMemoryAllocation(10u, unifiedMemoryProperties); + auto allocation2 = svmManager->createUnifiedMemoryAllocation(10u, unifiedMemoryProperties); + EXPECT_NE(allocation, nullptr); + EXPECT_NE(allocation2, nullptr); + svmManager->freeSVMAlloc(allocation); + svmManager->freeSVMAlloc(allocation2); + EXPECT_EQ(svmManager->usmDeviceAllocationsCache->allocations.size(), 2u); + + const auto baseTimePoint = std::chrono::high_resolution_clock::now(); + const auto oldTimePoint = baseTimePoint - UnifiedMemoryReuseCleaner::maxHoldTime; + + svmManager->usmDeviceAllocationsCache->allocations[0].saveTime = oldTimePoint; + svmManager->usmDeviceAllocationsCache->allocations[1].saveTime = oldTimePoint; + + mockUnifiedMemoryReuseCleaner->trimOldInCaches(); + EXPECT_EQ(0u, svmManager->usmDeviceAllocationsCache->allocations.size()); + + svmManager->cleanupUSMAllocCaches(); + EXPECT_EQ(0u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size()); +} + TEST_F(SvmDeviceAllocationCacheTest, givenUsmReuseCleanerWhenTrimOldInCachesCalledAndShouldLimitUsmReuseThenAllOldAllocationsAreRemovedEvenIfDeferredDeleterHasWork) { std::unique_ptr deviceFactory(new UltDeviceFactory(1, 1)); RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex}; @@ -965,7 +1006,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenUsmReuseCleanerWhenTrimOldInCachesCall auto device = deviceFactory->rootDevices[0]; auto memoryManager = reinterpret_cast(device->getMemoryManager()); auto svmManager = std::make_unique(memoryManager, false); - device->executionEnvironment->unifiedMemoryReuseCleaner.reset(new MockUnifiedMemoryReuseCleaner); + device->executionEnvironment->unifiedMemoryReuseCleaner.reset(new MockUnifiedMemoryReuseCleaner(false)); auto mockUnifiedMemoryReuseCleaner = reinterpret_cast(device->executionEnvironment->unifiedMemoryReuseCleaner.get()); EXPECT_EQ(0u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size()); device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);