From 915d657420ac78d1cd697ed497f62e21c4bb97f8 Mon Sep 17 00:00:00 2001 From: Dominik Dabek Date: Wed, 26 Mar 2025 14:09:38 +0000 Subject: [PATCH] fix: flag to limit usm reuse based on memory usage Host usm and device usm for igfx checks system memory usage. Device usm for dgfx checks local memory usage. If used memory is above limit threshold: - no new allocations will be saved for reuse - cleaner will use shorter hold time of 2 seconds - cleaner will free all eligible allocations, regardless of async deleter thread having work Motivation: in case of gfx memory being full, making resident new allocations will require evictions which leads to massive slowdown on enqueue calls. This change aims to minimize cases where extra memory usage from usm reuse mechanism leads to above situation. Related-To: NEO-6893, NEO-14160 Signed-off-by: Dominik Dabek --- .../debug_settings/debug_variables_base.inl | 1 + shared/source/device/device.cpp | 26 +- shared/source/device/device.h | 27 +- .../execution_environment.cpp | 2 +- shared/source/memory_manager/CMakeLists.txt | 1 + .../source/memory_manager/memory_manager.cpp | 22 +- shared/source/memory_manager/memory_manager.h | 27 +- .../memory_manager/unified_memory_manager.cpp | 53 +-- .../memory_manager/unified_memory_manager.h | 2 +- .../memory_manager/unified_memory_reuse.h | 52 +++ .../unified_memory_reuse_cleaner.cpp | 16 +- .../unified_memory_reuse_cleaner.h | 1 + shared/test/common/mocks/mock_device.h | 4 +- .../test/common/mocks/mock_memory_manager.h | 1 - .../test/common/mocks/ult_device_factory.cpp | 2 +- shared/test/common/test_files/igdrcl.config | 1 + .../execution_environment_tests.cpp | 2 +- .../unified_memory_manager_cache_tests.cpp | 303 +++++++++++++----- 18 files changed, 373 insertions(+), 170 deletions(-) create mode 100644 shared/source/memory_manager/unified_memory_reuse.h diff --git a/shared/source/debug_settings/debug_variables_base.inl b/shared/source/debug_settings/debug_variables_base.inl index b65550553a..553703d1fd 100644 --- a/shared/source/debug_settings/debug_variables_base.inl +++ b/shared/source/debug_settings/debug_variables_base.inl @@ -592,6 +592,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalEnableDeviceAllocationCache, -1, "Ex DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalEnableHostAllocationCache, -1, "Experimentally enable host usm allocation cache. Use X% of shared system memory.") DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalUSMAllocationReuseVersion, -1, "Version of mechanism to use for usm allocation reuse.") DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalUSMAllocationReuseCleaner, -1, "Enable usm allocation reuse cleaner. -1: default, 0: disable, 1:enable") +DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalUSMAllocationReuseLimitThreshold, -1, "Threshold of used memory to limit usm reuse. -1: default, 0: disable, >0:X% of shared/device memory") DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalH2DCpuCopyThreshold, -1, "Override default threshold (in bytes) for H2D CPU copy.") DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalD2HCpuCopyThreshold, -1, "Override default threshold (in bytes) for D2H CPU copy.") DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalCopyThroughLock, -1, "Experimentally copy memory through locked ptr. -1: default 0: disable 1: enable ") diff --git a/shared/source/device/device.cpp b/shared/source/device/device.cpp index b854fcfd32..c9689ad6c8 100644 --- a/shared/source/device/device.cpp +++ b/shared/source/device/device.cpp @@ -210,10 +210,9 @@ void Device::initializeCommonResources() { deviceBitfields.emplace(getRootDeviceIndex(), getDeviceBitfield()); deviceUsmMemAllocPoolsManager.reset(new UsmMemAllocPoolsManager(getMemoryManager(), rootDeviceIndices, deviceBitfields, this, InternalMemoryType::deviceUnifiedMemory)); } - initUsmReuseMaxSize(); } -void Device::initUsmReuseMaxSize() { +void Device::initUsmReuseLimits() { const bool usmDeviceAllocationsCacheEnabled = NEO::ApiSpecificConfig::isDeviceAllocationCacheEnabled() && this->getProductHelper().isDeviceUsmAllocationReuseSupported(); auto ailConfiguration = this->getAilConfigurationHelper(); const bool limitDeviceMemoryForReuse = ailConfiguration && ailConfiguration->limitAmountOfDeviceMemoryForRecycling(); @@ -222,7 +221,27 @@ void Device::initUsmReuseMaxSize() { fractionOfTotalMemoryForRecycling = 0.01 * std::min(100, debugManager.flags.ExperimentalEnableDeviceAllocationCache.get()); } const auto totalDeviceMemory = this->getGlobalMemorySize(static_cast(this->getDeviceBitfield().to_ulong())); - this->maxAllocationsSavedForReuseSize = static_cast(fractionOfTotalMemoryForRecycling * totalDeviceMemory); + auto maxAllocationsSavedForReuseSize = static_cast(fractionOfTotalMemoryForRecycling * totalDeviceMemory); + + auto limitAllocationsReuseThreshold = UsmReuseInfo::notLimited; + const auto limitFlagValue = debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.get(); + if (limitFlagValue != -1) { + if (limitFlagValue == 0) { + limitAllocationsReuseThreshold = UsmReuseInfo::notLimited; + } else { + const auto fractionOfTotalMemoryToLimitReuse = limitFlagValue / 100.0; + limitAllocationsReuseThreshold = static_cast(fractionOfTotalMemoryToLimitReuse * totalDeviceMemory); + } + } + this->usmReuseInfo.init(maxAllocationsSavedForReuseSize, limitAllocationsReuseThreshold); +} + +bool Device::shouldLimitAllocationsReuse() const { + const bool isIntegratedDevice = getHardwareInfo().capabilityTable.isIntegratedDevice; + if (isIntegratedDevice) { + return getMemoryManager()->shouldLimitAllocationsReuse(); + } + return getMemoryManager()->getUsedLocalMemorySize(getRootDeviceIndex()) >= this->usmReuseInfo.getLimitAllocationsReuseThreshold(); } bool Device::initDeviceFully() { @@ -269,6 +288,7 @@ bool Device::initDeviceFully() { createBindlessHeapsHelper(); uuid.isValid = false; + initUsmReuseLimits(); if (getRootDeviceEnvironment().osInterface == nullptr) { return true; diff --git a/shared/source/device/device.h b/shared/source/device/device.h index 106503f120..23b50174b7 100644 --- a/shared/source/device/device.h +++ b/shared/source/device/device.h @@ -13,6 +13,7 @@ #include "shared/source/helpers/engine_node_helper.h" #include "shared/source/helpers/non_copyable_or_moveable.h" #include "shared/source/helpers/options.h" +#include "shared/source/memory_manager/unified_memory_reuse.h" #include "shared/source/os_interface/performance_counters.h" #include "shared/source/os_interface/product_helper.h" #include "shared/source/utilities/isa_pool_allocator.h" @@ -222,25 +223,8 @@ class Device : public ReferenceTrackedObject, NEO::NonCopyableAndNonMova void stopDirectSubmissionForCopyEngine(); - uint64_t getMaxAllocationsSavedForReuseSize() const { - return maxAllocationsSavedForReuseSize; - } + bool shouldLimitAllocationsReuse() const; - std::unique_lock obtainAllocationsReuseLock() const { - return std::unique_lock(allocationsReuseMtx); - } - - void recordAllocationSaveForReuse(size_t size) { - allocationsSavedForReuseSize += size; - } - - void recordAllocationGetFromReuse(size_t size) { - allocationsSavedForReuseSize -= size; - } - - uint64_t getAllocationsSavedForReuseSize() const { - return allocationsSavedForReuseSize; - } uint32_t getMicrosecondResolution() const { return microsecondResolution; } @@ -262,6 +246,8 @@ class Device : public ReferenceTrackedObject, NEO::NonCopyableAndNonMova bufferPoolCount -= size; } + UsmReuseInfo usmReuseInfo; + protected: Device() = delete; Device(ExecutionEnvironment *executionEnvironment, const uint32_t rootDeviceIndex); @@ -281,7 +267,7 @@ class Device : public ReferenceTrackedObject, NEO::NonCopyableAndNonMova bool initDeviceWithEngines(); void initializeCommonResources(); bool initDeviceFully(); - void initUsmReuseMaxSize(); + void initUsmReuseLimits(); virtual bool createEngines(); void addEngineToEngineGroup(EngineControl &engine); @@ -339,11 +325,8 @@ class Device : public ReferenceTrackedObject, NEO::NonCopyableAndNonMova ISAPoolAllocator isaPoolAllocator; std::unique_ptr deviceUsmMemAllocPoolsManager; - uint64_t allocationsSavedForReuseSize = 0u; - uint64_t maxAllocationsSavedForReuseSize = 0u; std::atomic_uint32_t bufferPoolCount = 0u; uint32_t maxBufferPoolCount = 0u; - mutable std::mutex allocationsReuseMtx; uint32_t microsecondResolution = 1000u; struct { diff --git a/shared/source/execution_environment/execution_environment.cpp b/shared/source/execution_environment/execution_environment.cpp index 08d97f14d5..5702d41daa 100644 --- a/shared/source/execution_environment/execution_environment.cpp +++ b/shared/source/execution_environment/execution_environment.cpp @@ -88,7 +88,7 @@ bool ExecutionEnvironment::initializeMemoryManager() { } break; } - memoryManager->initUsmReuseMaxSize(); + memoryManager->initUsmReuseLimits(); return memoryManager->isInitialized(); } diff --git a/shared/source/memory_manager/CMakeLists.txt b/shared/source/memory_manager/CMakeLists.txt index 7e106cc1bf..e351358a9c 100644 --- a/shared/source/memory_manager/CMakeLists.txt +++ b/shared/source/memory_manager/CMakeLists.txt @@ -59,6 +59,7 @@ set(NEO_CORE_MEMORY_MANAGER ${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_manager.h ${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_pooling.cpp ${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_pooling.h + ${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_reuse.h ${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_reuse_cleaner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_reuse_cleaner.h ${CMAKE_CURRENT_SOURCE_DIR}/page_table.cpp diff --git a/shared/source/memory_manager/memory_manager.cpp b/shared/source/memory_manager/memory_manager.cpp index 7999c41d12..6b3e3064d4 100644 --- a/shared/source/memory_manager/memory_manager.cpp +++ b/shared/source/memory_manager/memory_manager.cpp @@ -389,13 +389,25 @@ uint32_t MemoryManager::getFirstContextIdForRootDevice(uint32_t rootDeviceIndex) return 0; } -void MemoryManager::initUsmReuseMaxSize() { - const auto totalSystemMemory = this->getSystemSharedMemory(0u); - auto fractionOfTotalMemoryForRecycling = 0.02; +void MemoryManager::initUsmReuseLimits() { + const auto systemSharedMemorySize = this->getSystemSharedMemory(0u); + auto fractionOfTotalMemoryForReuse = 0.02; if (debugManager.flags.ExperimentalEnableHostAllocationCache.get() != -1) { - fractionOfTotalMemoryForRecycling = 0.01 * std::min(100, debugManager.flags.ExperimentalEnableHostAllocationCache.get()); + fractionOfTotalMemoryForReuse = 0.01 * std::min(100, debugManager.flags.ExperimentalEnableHostAllocationCache.get()); } - this->maxAllocationsSavedForReuseSize = static_cast(fractionOfTotalMemoryForRecycling * totalSystemMemory); + auto maxAllocationsSavedForReuseSize = static_cast(fractionOfTotalMemoryForReuse * systemSharedMemorySize); + + auto limitAllocationsReuseThreshold = std::numeric_limits::max(); + const auto limitFlagValue = debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.get(); + if (limitFlagValue != -1) { + if (limitFlagValue == 0) { + limitAllocationsReuseThreshold = std::numeric_limits::max(); + } else { + const auto fractionOfTotalMemoryToLimitReuse = limitFlagValue / 100.0; + limitAllocationsReuseThreshold = static_cast(fractionOfTotalMemoryToLimitReuse * systemSharedMemorySize); + } + } + this->usmReuseInfo.init(maxAllocationsSavedForReuseSize, limitAllocationsReuseThreshold); } OsContext *MemoryManager::createAndRegisterOsContext(CommandStreamReceiver *commandStreamReceiver, diff --git a/shared/source/memory_manager/memory_manager.h b/shared/source/memory_manager/memory_manager.h index f119499261..2d3d8548cb 100644 --- a/shared/source/memory_manager/memory_manager.h +++ b/shared/source/memory_manager/memory_manager.h @@ -12,6 +12,7 @@ #include "shared/source/memory_manager/alignment_selector.h" #include "shared/source/memory_manager/graphics_allocation.h" #include "shared/source/memory_manager/memadvise_flags.h" +#include "shared/source/memory_manager/unified_memory_reuse.h" #include "shared/source/os_interface/os_memory.h" #include "shared/source/utilities/stackvec.h" @@ -338,26 +339,11 @@ class MemoryManager { virtual void getExtraDeviceProperties(uint32_t rootDeviceIndex, uint32_t *moduleId, uint16_t *serverType) { return; } - std::unique_lock obtainHostAllocationsReuseLock() const { - return std::unique_lock(hostAllocationsReuseMtx); - } + void initUsmReuseLimits(); + UsmReuseInfo usmReuseInfo; - void initUsmReuseMaxSize(); - - uint64_t getMaxAllocationsSavedForReuseSize() const { - return maxAllocationsSavedForReuseSize; - } - - void recordHostAllocationSaveForReuse(uint64_t size) { - hostAllocationsSavedForReuseSize += size; - } - - void recordHostAllocationGetFromReuse(uint64_t size) { - hostAllocationsSavedForReuseSize -= size; - } - - uint64_t getHostAllocationsSavedForReuseSize() const { - return hostAllocationsSavedForReuseSize; + bool shouldLimitAllocationsReuse() const { + return getUsedSystemMemorySize() >= usmReuseInfo.getLimitAllocationsReuseThreshold(); } void addCustomHeapAllocatorConfig(AllocationType allocationType, bool isFrontWindowPool, const CustomHeapAllocatorConfig &config); @@ -436,9 +422,6 @@ class MemoryManager { std::mutex physicalMemoryAllocationMapMutex; std::unique_ptr[]> localMemAllocsSize; std::atomic sysMemAllocsSize; - uint64_t maxAllocationsSavedForReuseSize = 0u; - uint64_t hostAllocationsSavedForReuseSize = 0u; - mutable std::mutex hostAllocationsReuseMtx; std::map, CustomHeapAllocatorConfig> customHeapAllocators; }; diff --git a/shared/source/memory_manager/unified_memory_manager.cpp b/shared/source/memory_manager/unified_memory_manager.cpp index 5b8f9d7614..73daf20aa2 100644 --- a/shared/source/memory_manager/unified_memory_manager.cpp +++ b/shared/source/memory_manager/unified_memory_manager.cpp @@ -64,21 +64,24 @@ bool SVMAllocsManager::SvmAllocationCache::insert(size_t size, void *ptr, SvmAll if (false == sizeAllowed(size)) { return false; } + if (svmData->device ? svmData->device->shouldLimitAllocationsReuse() : memoryManager->shouldLimitAllocationsReuse()) { + return false; + } std::lock_guard lock(this->mtx); bool isSuccess = true; if (auto device = svmData->device) { - auto lock = device->obtainAllocationsReuseLock(); - if (size + device->getAllocationsSavedForReuseSize() > device->getMaxAllocationsSavedForReuseSize()) { + auto lock = device->usmReuseInfo.obtainAllocationsReuseLock(); + if (size + device->usmReuseInfo.getAllocationsSavedForReuseSize() > device->usmReuseInfo.getMaxAllocationsSavedForReuseSize()) { isSuccess = false; } else { - device->recordAllocationSaveForReuse(size); + device->usmReuseInfo.recordAllocationSaveForReuse(size); } } else { - auto lock = memoryManager->obtainHostAllocationsReuseLock(); - if (size + memoryManager->getHostAllocationsSavedForReuseSize() > memoryManager->getMaxAllocationsSavedForReuseSize()) { + auto lock = memoryManager->usmReuseInfo.obtainAllocationsReuseLock(); + if (size + memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize() > memoryManager->usmReuseInfo.getMaxAllocationsSavedForReuseSize()) { isSuccess = false; } else { - memoryManager->recordHostAllocationSaveForReuse(size); + memoryManager->usmReuseInfo.recordAllocationSaveForReuse(size); } } if (isSuccess) { @@ -133,11 +136,11 @@ void *SVMAllocsManager::SvmAllocationCache::get(size_t size, const UnifiedMemory svmData->allocationFlagsProperty.allAllocFlags == unifiedMemoryProperties.allocationFlags.allAllocFlags && false == isInUse(svmData)) { if (svmData->device) { - auto lock = svmData->device->obtainAllocationsReuseLock(); - svmData->device->recordAllocationGetFromReuse(allocationIter->allocationSize); + auto lock = svmData->device->usmReuseInfo.obtainAllocationsReuseLock(); + svmData->device->usmReuseInfo.recordAllocationGetFromReuse(allocationIter->allocationSize); } else { - auto lock = memoryManager->obtainHostAllocationsReuseLock(); - memoryManager->recordHostAllocationGetFromReuse(allocationIter->allocationSize); + auto lock = memoryManager->usmReuseInfo.obtainAllocationsReuseLock(); + memoryManager->usmReuseInfo.recordAllocationGetFromReuse(allocationIter->allocationSize); } if (enablePerformanceLogging) { logCacheOperation({.allocationSize = allocationIter->allocationSize, @@ -167,11 +170,11 @@ void SVMAllocsManager::SvmAllocationCache::trim() { SvmAllocationData *svmData = svmAllocsManager->getSVMAlloc(cachedAllocationInfo.allocation); UNRECOVERABLE_IF(nullptr == svmData); if (svmData->device) { - auto lock = svmData->device->obtainAllocationsReuseLock(); - svmData->device->recordAllocationGetFromReuse(cachedAllocationInfo.allocationSize); + auto lock = svmData->device->usmReuseInfo.obtainAllocationsReuseLock(); + svmData->device->usmReuseInfo.recordAllocationGetFromReuse(cachedAllocationInfo.allocationSize); } else { - auto lock = memoryManager->obtainHostAllocationsReuseLock(); - memoryManager->recordHostAllocationGetFromReuse(cachedAllocationInfo.allocationSize); + auto lock = memoryManager->usmReuseInfo.obtainAllocationsReuseLock(); + memoryManager->usmReuseInfo.recordAllocationGetFromReuse(cachedAllocationInfo.allocationSize); } if (enablePerformanceLogging) { logCacheOperation({.allocationSize = cachedAllocationInfo.allocationSize, @@ -234,7 +237,7 @@ void SVMAllocsManager::SvmAllocationCache::logCacheOperation(const SvmAllocation isSuccessString); } -void SVMAllocsManager::SvmAllocationCache::trimOldAllocs(std::chrono::high_resolution_clock::time_point trimTimePoint) { +void SVMAllocsManager::SvmAllocationCache::trimOldAllocs(std::chrono::high_resolution_clock::time_point trimTimePoint, bool shouldLimitReuse) { if (this->allocations.empty()) { return; } @@ -248,11 +251,11 @@ void SVMAllocsManager::SvmAllocationCache::trimOldAllocs(std::chrono::high_resol SvmAllocationData *svmData = svmAllocsManager->getSVMAlloc(allocationPtr); UNRECOVERABLE_IF(nullptr == svmData); if (svmData->device) { - auto lock = svmData->device->obtainAllocationsReuseLock(); - svmData->device->recordAllocationGetFromReuse(allocCleanCandidate->allocationSize); + auto lock = svmData->device->usmReuseInfo.obtainAllocationsReuseLock(); + svmData->device->usmReuseInfo.recordAllocationGetFromReuse(allocCleanCandidate->allocationSize); } else { - auto lock = memoryManager->obtainHostAllocationsReuseLock(); - memoryManager->recordHostAllocationGetFromReuse(allocCleanCandidate->allocationSize); + auto lock = memoryManager->usmReuseInfo.obtainAllocationsReuseLock(); + memoryManager->usmReuseInfo.recordAllocationGetFromReuse(allocCleanCandidate->allocationSize); } if (enablePerformanceLogging) { logCacheOperation({.allocationSize = allocCleanCandidate->allocationSize, @@ -262,8 +265,12 @@ void SVMAllocsManager::SvmAllocationCache::trimOldAllocs(std::chrono::high_resol .isSuccess = true}); } svmAllocsManager->freeSVMAllocImpl(allocCleanCandidate->allocation, FreePolicyType::defer, svmData); - allocations.erase(allocCleanCandidate); - break; + if (shouldLimitReuse) { + allocCleanCandidate = allocations.erase(allocCleanCandidate); + } else { + allocations.erase(allocCleanCandidate); + break; + } } } @@ -901,7 +908,7 @@ void SVMAllocsManager::freeZeroCopySvmAllocation(SvmAllocationData *svmData) { void SVMAllocsManager::initUsmDeviceAllocationsCache(Device &device) { this->usmDeviceAllocationsCache.reset(new SvmAllocationCache); - if (device.getMaxAllocationsSavedForReuseSize() > 0u) { + if (device.usmReuseInfo.getMaxAllocationsSavedForReuseSize() > 0u) { this->usmDeviceAllocationsCache->allocations.reserve(128u); this->usmDeviceAllocationsCache->svmAllocsManager = this; this->usmDeviceAllocationsCache->memoryManager = memoryManager; @@ -913,7 +920,7 @@ void SVMAllocsManager::initUsmDeviceAllocationsCache(Device &device) { void SVMAllocsManager::initUsmHostAllocationsCache() { this->usmHostAllocationsCache.reset(new SvmAllocationCache); - if (memoryManager->getMaxAllocationsSavedForReuseSize() > 0u) { + if (memoryManager->usmReuseInfo.getMaxAllocationsSavedForReuseSize() > 0u) { this->usmHostAllocationsCache->allocations.reserve(128u); this->usmHostAllocationsCache->svmAllocsManager = this; this->usmHostAllocationsCache->memoryManager = memoryManager; diff --git a/shared/source/memory_manager/unified_memory_manager.h b/shared/source/memory_manager/unified_memory_manager.h index 7e0cc30409..4de2995acd 100644 --- a/shared/source/memory_manager/unified_memory_manager.h +++ b/shared/source/memory_manager/unified_memory_manager.h @@ -194,7 +194,7 @@ class SVMAllocsManager { bool isInUse(SvmAllocationData *svmData); void *get(size_t size, const UnifiedMemoryProperties &unifiedMemoryProperties); void trim(); - void trimOldAllocs(std::chrono::high_resolution_clock::time_point trimTimePoint); + void trimOldAllocs(std::chrono::high_resolution_clock::time_point trimTimePoint, bool shouldLimitReuse); void cleanup(); void logCacheOperation(const SvmAllocationCachePerfInfo &cachePerfEvent) const; diff --git a/shared/source/memory_manager/unified_memory_reuse.h b/shared/source/memory_manager/unified_memory_reuse.h new file mode 100644 index 0000000000..c4ce9d4729 --- /dev/null +++ b/shared/source/memory_manager/unified_memory_reuse.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once +#include +#include + +namespace NEO { +struct UsmReuseInfo { + uint64_t getAllocationsSavedForReuseSize() const { + return allocationsSavedForReuseSize; + } + + uint64_t getMaxAllocationsSavedForReuseSize() const { + return maxAllocationsSavedForReuseSize; + } + + uint64_t getLimitAllocationsReuseThreshold() const { + return limitAllocationsReuseThreshold; + } + + std::unique_lock obtainAllocationsReuseLock() const { + return std::unique_lock(allocationsReuseMtx); + } + + void recordAllocationSaveForReuse(size_t size) { + allocationsSavedForReuseSize += size; + } + + void recordAllocationGetFromReuse(size_t size) { + allocationsSavedForReuseSize -= size; + } + + void init(uint64_t maxAllocationsSavedForReuseSize, uint64_t limitAllocationsReuseThreshold) { + this->maxAllocationsSavedForReuseSize = maxAllocationsSavedForReuseSize; + this->limitAllocationsReuseThreshold = limitAllocationsReuseThreshold; + } + + static constexpr uint64_t notLimited = std::numeric_limits::max(); + + protected: + uint64_t maxAllocationsSavedForReuseSize = 0u; + uint64_t limitAllocationsReuseThreshold = 0u; + uint64_t allocationsSavedForReuseSize = 0u; + mutable std::mutex allocationsReuseMtx; +}; + +} // namespace NEO diff --git a/shared/source/memory_manager/unified_memory_reuse_cleaner.cpp b/shared/source/memory_manager/unified_memory_reuse_cleaner.cpp index f50e962b6b..aa83e9f413 100644 --- a/shared/source/memory_manager/unified_memory_reuse_cleaner.cpp +++ b/shared/source/memory_manager/unified_memory_reuse_cleaner.cpp @@ -61,15 +61,21 @@ void UnifiedMemoryReuseCleaner::unregisterSvmAllocationCache(SvmAllocationCache } void UnifiedMemoryReuseCleaner::trimOldInCaches() { - const std::chrono::high_resolution_clock::time_point trimTimePoint = std::chrono::high_resolution_clock::now() - maxHoldTime; + bool shouldLimitReuse = false; + auto trimTimePoint = std::chrono::high_resolution_clock::now() - maxHoldTime; std::lock_guard lockSvmAllocationCaches(this->svmAllocationCachesMutex); for (auto svmAllocCache : this->svmAllocationCaches) { - if (auto deferredDeleter = svmAllocCache->memoryManager->getDeferredDeleter()) { - if (false == deferredDeleter->areElementsReleased(false)) { - continue; + shouldLimitReuse |= svmAllocCache->memoryManager->shouldLimitAllocationsReuse(); + if (shouldLimitReuse) { + trimTimePoint = std::chrono::high_resolution_clock::now() - limitedHoldTime; + } else { + if (auto deferredDeleter = svmAllocCache->memoryManager->getDeferredDeleter()) { + if (false == deferredDeleter->areElementsReleased(false)) { + continue; + } } } - svmAllocCache->trimOldAllocs(trimTimePoint); + svmAllocCache->trimOldAllocs(trimTimePoint, shouldLimitReuse); } } diff --git a/shared/source/memory_manager/unified_memory_reuse_cleaner.h b/shared/source/memory_manager/unified_memory_reuse_cleaner.h index 4919321663..179ac2e27e 100644 --- a/shared/source/memory_manager/unified_memory_reuse_cleaner.h +++ b/shared/source/memory_manager/unified_memory_reuse_cleaner.h @@ -22,6 +22,7 @@ class UnifiedMemoryReuseCleaner : NEO::NonCopyableAndNonMovableClass { public: static constexpr auto sleepTime = std::chrono::milliseconds(15u); static constexpr auto maxHoldTime = std::chrono::seconds(10u); + static constexpr auto limitedHoldTime = std::chrono::seconds(2u); UnifiedMemoryReuseCleaner(); virtual ~UnifiedMemoryReuseCleaner(); diff --git a/shared/test/common/mocks/mock_device.h b/shared/test/common/mocks/mock_device.h index 34fdb73d20..19c511c731 100644 --- a/shared/test/common/mocks/mock_device.h +++ b/shared/test/common/mocks/mock_device.h @@ -33,7 +33,6 @@ extern CommandStreamReceiver *createCommandStream(ExecutionEnvironment &executio struct MockSubDevice : public SubDevice { using Device::allEngines; using Device::createEngines; - using Device::maxAllocationsSavedForReuseSize; using SubDevice::getDeviceBitfield; using SubDevice::getGlobalMemorySize; using SubDevice::SubDevice; @@ -67,8 +66,7 @@ class MockDevice : public RootDevice { using Device::generateUuidFromPciBusInfo; using Device::getGlobalMemorySize; using Device::initializeCaps; - using Device::initUsmReuseMaxSize; - using Device::maxAllocationsSavedForReuseSize; + using Device::initUsmReuseLimits; using Device::maxBufferPoolCount; using Device::microsecondResolution; using Device::preemptionMode; diff --git a/shared/test/common/mocks/mock_memory_manager.h b/shared/test/common/mocks/mock_memory_manager.h index 743dee8ecd..01d04dc434 100644 --- a/shared/test/common/mocks/mock_memory_manager.h +++ b/shared/test/common/mocks/mock_memory_manager.h @@ -62,7 +62,6 @@ class MockMemoryManager : public MemoryManagerCreate { using MemoryManager::latestContextId; using MemoryManager::localMemAllocsSize; using MemoryManager::localMemorySupported; - using MemoryManager::maxAllocationsSavedForReuseSize; using MemoryManager::reservedMemory; using MemoryManager::secondaryEngines; diff --git a/shared/test/common/mocks/ult_device_factory.cpp b/shared/test/common/mocks/ult_device_factory.cpp index f2639ecdcf..f954f812f0 100644 --- a/shared/test/common/mocks/ult_device_factory.cpp +++ b/shared/test/common/mocks/ult_device_factory.cpp @@ -94,7 +94,7 @@ bool UltDeviceFactory::initializeMemoryManager(ExecutionEnvironment &executionEn bool enableLocalMemory = gfxCoreHelper.getEnableLocalMemory(*defaultHwInfo); bool aubUsage = (testMode == TestMode::aubTests) || (testMode == TestMode::aubTestsWithTbx); executionEnvironment.memoryManager.reset(new MockMemoryManager(false, enableLocalMemory, aubUsage, executionEnvironment)); - executionEnvironment.memoryManager->initUsmReuseMaxSize(); + executionEnvironment.memoryManager->initUsmReuseLimits(); } return true; } diff --git a/shared/test/common/test_files/igdrcl.config b/shared/test/common/test_files/igdrcl.config index cb8aa31e6f..6b3d644bcb 100644 --- a/shared/test/common/test_files/igdrcl.config +++ b/shared/test/common/test_files/igdrcl.config @@ -668,4 +668,5 @@ EnableDeferBacking = 0 SetMaxBVHLevels = -1 GetSipBinaryFromExternalLib = -1 LogUsmReuse = 0 +ExperimentalUSMAllocationReuseLimitThreshold = -1 # Please don't edit below this line diff --git a/shared/test/unit_test/execution_environment/execution_environment_tests.cpp b/shared/test/unit_test/execution_environment/execution_environment_tests.cpp index a409c29096..c69921a26f 100644 --- a/shared/test/unit_test/execution_environment/execution_environment_tests.cpp +++ b/shared/test/unit_test/execution_environment/execution_environment_tests.cpp @@ -398,7 +398,7 @@ TEST(ExecutionEnvironment, givenExecutionEnvironmentWhenInitializeMemoryManagerI executionEnvironment.initializeMemoryManager(); ASSERT_NE(nullptr, executionEnvironment.memoryManager); EXPECT_TRUE(executionEnvironment.memoryManager->isInitialized()); - EXPECT_NE(0u, executionEnvironment.memoryManager->getMaxAllocationsSavedForReuseSize()); + EXPECT_NE(0u, executionEnvironment.memoryManager->usmReuseInfo.getMaxAllocationsSavedForReuseSize()); } static_assert(sizeof(ExecutionEnvironment) == sizeof(std::unique_ptr) + diff --git a/shared/test/unit_test/memory_manager/unified_memory_manager_cache_tests.cpp b/shared/test/unit_test/memory_manager/unified_memory_manager_cache_tests.cpp index 8b29d72c47..0ecd971e90 100644 --- a/shared/test/unit_test/memory_manager/unified_memory_manager_cache_tests.cpp +++ b/shared/test/unit_test/memory_manager/unified_memory_manager_cache_tests.cpp @@ -148,6 +148,7 @@ struct SvmAllocationCacheTestFixture { void tearDown() { } static constexpr size_t allocationSizeBasis = MemoryConstants::pageSize64k; + static constexpr uint64_t alwaysLimited = 0u; MockExecutionEnvironment executionEnvironment; }; @@ -166,11 +167,11 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheDisabledWhenCheckingIfE TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledAndMaxSizeZeroWhenCallingCleanupThenNoError) { DebugManagerStateRestore restorer; + debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1); std::unique_ptr deviceFactory(new UltDeviceFactory(1, 1)); auto device = deviceFactory->rootDevices[0]; auto svmManager = std::make_unique(device->getMemoryManager(), false); - debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1); - device->maxAllocationsSavedForReuseSize = 0u; + device->usmReuseInfo.init(0, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache); svmManager->cleanupUSMAllocCaches(); @@ -185,8 +186,8 @@ HWTEST_F(SvmDeviceAllocationCacheTest, givenOclApiSpecificConfigWhenCheckingIfEn device->mockAilConfigurationHelper = &mockAilConfigurationHelper; { raii.mockProductHelper->isDeviceUsmAllocationReuseSupportedResult = false; - device->initUsmReuseMaxSize(); - EXPECT_EQ(0u, device->getMaxAllocationsSavedForReuseSize()); + device->initUsmReuseLimits(); + EXPECT_EQ(0u, device->usmReuseInfo.getMaxAllocationsSavedForReuseSize()); auto svmManager = std::make_unique(device->getMemoryManager(), false); EXPECT_EQ(nullptr, svmManager->usmDeviceAllocationsCache); svmManager->initUsmAllocationsCaches(*device); @@ -194,23 +195,23 @@ HWTEST_F(SvmDeviceAllocationCacheTest, givenOclApiSpecificConfigWhenCheckingIfEn } { raii.mockProductHelper->isDeviceUsmAllocationReuseSupportedResult = true; - device->initUsmReuseMaxSize(); + device->initUsmReuseLimits(); auto svmManager = std::make_unique(device->getMemoryManager(), false); EXPECT_EQ(nullptr, svmManager->usmDeviceAllocationsCache); svmManager->initUsmAllocationsCaches(*device); EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache); const auto expectedMaxSize = static_cast(0.08 * device->getGlobalMemorySize(static_cast(device->getDeviceBitfield().to_ullong()))); - EXPECT_EQ(expectedMaxSize, device->getMaxAllocationsSavedForReuseSize()); + EXPECT_EQ(expectedMaxSize, device->usmReuseInfo.getMaxAllocationsSavedForReuseSize()); } { raii.mockProductHelper->isDeviceUsmAllocationReuseSupportedResult = true; mockAilConfigurationHelper.limitAmountOfDeviceMemoryForRecyclingReturn = true; - device->initUsmReuseMaxSize(); + device->initUsmReuseLimits(); auto svmManager = std::make_unique(device->getMemoryManager(), false); EXPECT_EQ(nullptr, svmManager->usmDeviceAllocationsCache); svmManager->initUsmAllocationsCaches(*device); EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache); - EXPECT_EQ(0u, device->getMaxAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, device->usmReuseInfo.getMaxAllocationsSavedForReuseSize()); } } @@ -231,6 +232,36 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledWhenDirectSubmis EXPECT_FALSE(device->getExecutionEnvironment()->unifiedMemoryReuseCleaner.get()); } +TEST_F(SvmDeviceAllocationCacheTest, givenReuseLimitFlagWhenInitUsmReuseLimitCalledThenLimitThresholdSetCorrectly) { + std::unique_ptr deviceFactory(new UltDeviceFactory(1, 1)); + RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex}; + std::map deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}}; + auto device = deviceFactory->rootDevices[0]; + + { + DebugManagerStateRestore restore; + debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.set(0); + device->initUsmReuseLimits(); + const auto expectedLimitThreshold = std::numeric_limits::max(); + EXPECT_EQ(expectedLimitThreshold, device->usmReuseInfo.getLimitAllocationsReuseThreshold()); + } + { + DebugManagerStateRestore restore; + debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.set(80); + device->initUsmReuseLimits(); + const auto totalDeviceMemory = device->getGlobalMemorySize(static_cast(device->getDeviceBitfield().to_ulong())); + const auto expectedLimitThreshold = static_cast(0.8 * totalDeviceMemory); + EXPECT_EQ(expectedLimitThreshold, device->usmReuseInfo.getLimitAllocationsReuseThreshold()); + } + { + DebugManagerStateRestore restore; + debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.set(-1); + device->initUsmReuseLimits(); + const auto expectedLimitThreshold = std::numeric_limits::max(); + EXPECT_EQ(expectedLimitThreshold, device->usmReuseInfo.getLimitAllocationsReuseThreshold()); + } +} + struct SvmDeviceAllocationCacheSimpleTestDataType { size_t allocationSize; void *allocation; @@ -244,7 +275,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingDevic debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1); auto device = deviceFactory->rootDevices[0]; auto svmManager = std::make_unique(device->getMemoryManager(), false); - device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache); @@ -294,7 +325,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledWhenInitializedT ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache); auto expectedMaxSize = static_cast(device->getGlobalMemorySize(static_cast(mockDeviceBitfield.to_ulong())) * 0.02); - EXPECT_EQ(expectedMaxSize, device->getMaxAllocationsSavedForReuseSize()); + EXPECT_EQ(expectedMaxSize, device->usmReuseInfo.getMaxAllocationsSavedForReuseSize()); } TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingDeviceAllocationThenItIsPutIntoCacheOnlyIfMaxSizeWillNotBeExceeded) { @@ -306,7 +337,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingDevic auto device = deviceFactory->rootDevices[0]; auto svmManager = std::make_unique(device->getMemoryManager(), false); constexpr auto allocationSize = MemoryConstants::pageSize64k; - device->maxAllocationsSavedForReuseSize = allocationSize; + device->usmReuseInfo.init(allocationSize, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache); @@ -318,26 +349,26 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingDevic auto allocation2 = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); ASSERT_NE(allocation2, nullptr); EXPECT_EQ(0u, svmManager->usmDeviceAllocationsCache->allocations.size()); - EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAlloc(allocation); EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAlloc(allocation2); EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize()); auto recycledAllocation = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); EXPECT_EQ(recycledAllocation, allocation); EXPECT_EQ(svmManager->usmDeviceAllocationsCache->allocations.size(), 0u); - EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAlloc(recycledAllocation); svmManager->trimUSMDeviceAllocCache(); EXPECT_EQ(svmManager->usmDeviceAllocationsCache->allocations.size(), 0u); - EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize()); } { auto allocation = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); @@ -345,26 +376,26 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingDevic auto allocation2 = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); ASSERT_NE(allocation2, nullptr); EXPECT_EQ(0u, svmManager->usmDeviceAllocationsCache->allocations.size()); - EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAllocDefer(allocation); EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAllocDefer(allocation2); EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize()); auto recycledAllocation = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); EXPECT_EQ(recycledAllocation, allocation); EXPECT_EQ(svmManager->usmDeviceAllocationsCache->allocations.size(), 0u); - EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAllocDefer(recycledAllocation); svmManager->trimUSMDeviceAllocCache(); EXPECT_EQ(svmManager->usmDeviceAllocationsCache->allocations.size(), 0u); - EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize()); } } @@ -378,7 +409,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledAndMultipleSVMMa auto svmManager = std::make_unique(device->getMemoryManager(), false); auto secondSvmManager = std::make_unique(device->getMemoryManager(), false); constexpr auto allocationSize = MemoryConstants::pageSize64k; - device->maxAllocationsSavedForReuseSize = allocationSize; + device->usmReuseInfo.init(allocationSize, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); secondSvmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache); @@ -394,22 +425,22 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledAndMultipleSVMMa svmManager->freeSVMAlloc(allocation); EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize()); secondSvmManager->freeSVMAlloc(allocation2); EXPECT_EQ(0u, secondSvmManager->usmDeviceAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize()); auto recycledAllocation = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); EXPECT_EQ(recycledAllocation, allocation); EXPECT_EQ(0u, svmManager->usmDeviceAllocationsCache->allocations.size()); - EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAlloc(recycledAllocation); svmManager->trimUSMDeviceAllocCache(); EXPECT_EQ(svmManager->usmDeviceAllocationsCache->allocations.size(), 0u); - EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize()); } { auto allocation = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); @@ -419,22 +450,22 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledAndMultipleSVMMa secondSvmManager->freeSVMAlloc(allocation2); EXPECT_EQ(1u, secondSvmManager->usmDeviceAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAlloc(allocation); EXPECT_EQ(0u, svmManager->usmDeviceAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize()); auto recycledAllocation = secondSvmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); EXPECT_EQ(recycledAllocation, allocation2); EXPECT_EQ(0u, secondSvmManager->usmDeviceAllocationsCache->allocations.size()); - EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize()); secondSvmManager->freeSVMAlloc(recycledAllocation); secondSvmManager->trimUSMDeviceAllocCache(); EXPECT_EQ(secondSvmManager->usmDeviceAllocationsCache->allocations.size(), 0u); - EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize()); } } @@ -446,7 +477,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsWithDifferentSizesWhenAlloc debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1); auto device = deviceFactory->rootDevices[0]; auto svmManager = std::make_unique(device->getMemoryManager(), false); - device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache); @@ -495,7 +526,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationWithDifferentSizeWhenAllocat debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1); auto device = deviceFactory->rootDevices[0]; auto svmManager = std::make_unique(device->getMemoryManager(), false); - device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache); @@ -540,7 +571,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsWithDifferentSizesWhenAlloc debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1); auto device = deviceFactory->rootDevices[0]; auto svmManager = std::make_unique(device->getMemoryManager(), false); - device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache); @@ -580,7 +611,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationOverSizeLimitWhenAllocatingA debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1); auto device = deviceFactory->rootDevices[0]; auto svmManager = std::make_unique(device->getMemoryManager(), false); - device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache); const auto notAcceptedAllocSize = SVMAllocsManager::SvmAllocationCache::maxServicedSize + 1; @@ -610,7 +641,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenMultipleAllocationsWhenAllocatingAfter debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1); auto device = deviceFactory->rootDevices[0]; auto svmManager = std::make_unique(device->getMemoryManager(), false); - device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache); @@ -688,9 +719,9 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsWithDifferentFlagsWhenAlloc auto secondRootDevice = deviceFactory->rootDevices[1]; auto subDevice1 = reinterpret_cast(deviceFactory->subDevices[0]); auto svmManager = std::make_unique(rootDevice->getMemoryManager(), false); - rootDevice->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; - secondRootDevice->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; - subDevice1->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + rootDevice->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); + secondRootDevice->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); + subDevice1->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*rootDevice); ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache); @@ -763,7 +794,8 @@ TEST_F(SvmDeviceAllocationCacheTest, givenDeviceOutOfMemoryWhenAllocatingThenCac device->injectMemoryManager(new MockMemoryManagerWithCapacity(*device->getExecutionEnvironment())); MockMemoryManagerWithCapacity *memoryManager = static_cast(device->getMemoryManager()); auto svmManager = std::make_unique(memoryManager, false); - device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); + memoryManager->usmReuseInfo.init(0u, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache); @@ -801,7 +833,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationWithIsInternalAllocationSetW debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1); auto device = deviceFactory->rootDevices[0]; auto svmManager = std::make_unique(device->getMemoryManager(), false); - device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache); @@ -833,7 +865,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationInUsageWhenAllocatingAfterFr debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1); auto device = deviceFactory->rootDevices[0]; auto svmManager = std::make_unique(device->getMemoryManager(), false); - device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache); @@ -869,7 +901,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenUsmReuseCleanerWhenTrimOldInCachesCall device->executionEnvironment->unifiedMemoryReuseCleaner.reset(new MockUnifiedMemoryReuseCleaner); auto mockUnifiedMemoryReuseCleaner = reinterpret_cast(device->executionEnvironment->unifiedMemoryReuseCleaner.get()); EXPECT_EQ(0u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size()); - device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache); EXPECT_EQ(1u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size()); @@ -910,6 +942,49 @@ TEST_F(SvmDeviceAllocationCacheTest, givenUsmReuseCleanerWhenTrimOldInCachesCall EXPECT_EQ(0u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size()); } +TEST_F(SvmDeviceAllocationCacheTest, givenUsmReuseCleanerWhenTrimOldInCachesCalledAndShouldLimitUsmReuseThenAllOldAllocationsAreRemovedEvenIfDeferredDeleterHasWork) { + std::unique_ptr deviceFactory(new UltDeviceFactory(1, 1)); + RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex}; + std::map deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}}; + DebugManagerStateRestore restore; + debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1); + debugManager.flags.ExperimentalEnableHostAllocationCache.set(0); + auto device = deviceFactory->rootDevices[0]; + auto memoryManager = reinterpret_cast(device->getMemoryManager()); + auto svmManager = std::make_unique(memoryManager, false); + device->executionEnvironment->unifiedMemoryReuseCleaner.reset(new MockUnifiedMemoryReuseCleaner); + auto mockUnifiedMemoryReuseCleaner = reinterpret_cast(device->executionEnvironment->unifiedMemoryReuseCleaner.get()); + EXPECT_EQ(0u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size()); + device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); + svmManager->initUsmAllocationsCaches(*device); + EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache); + EXPECT_EQ(1u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size()); + EXPECT_EQ(svmManager->usmDeviceAllocationsCache.get(), mockUnifiedMemoryReuseCleaner->svmAllocationCaches[0]); + + SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::deviceUnifiedMemory, 1, rootDeviceIndices, deviceBitfields); + unifiedMemoryProperties.device = device; + auto allocation = svmManager->createUnifiedMemoryAllocation(10u, unifiedMemoryProperties); + auto allocation2 = svmManager->createUnifiedMemoryAllocation(10u, unifiedMemoryProperties); + EXPECT_NE(allocation, nullptr); + EXPECT_NE(allocation2, nullptr); + svmManager->freeSVMAlloc(allocation); + svmManager->freeSVMAlloc(allocation2); + EXPECT_EQ(svmManager->usmDeviceAllocationsCache->allocations.size(), 2u); + + const auto baseTimePoint = std::chrono::high_resolution_clock::now(); + const auto oldTimePoint = baseTimePoint - UnifiedMemoryReuseCleaner::limitedHoldTime; + + svmManager->usmDeviceAllocationsCache->allocations[0].saveTime = oldTimePoint; + svmManager->usmDeviceAllocationsCache->allocations[1].saveTime = oldTimePoint; + + memoryManager->setDeferredDeleter(new MockDeferredDeleter); + memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, alwaysLimited); + mockUnifiedMemoryReuseCleaner->trimOldInCaches(); + EXPECT_EQ(0u, svmManager->usmDeviceAllocationsCache->allocations.size()); + + svmManager->cleanupUSMAllocCaches(); +} + TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCalledThenTrimAllocationsSavedBeforeTimePoint) { std::unique_ptr deviceFactory(new UltDeviceFactory(1, 1)); RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex}; @@ -918,7 +993,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCal debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1); auto device = deviceFactory->rootDevices[0]; auto svmManager = std::make_unique(device->getMemoryManager(), false); - device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache); @@ -942,13 +1017,13 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCal svmManager->usmDeviceAllocationsCache->allocations[1].saveTime = baseTimePoint + timeDiff; svmManager->usmDeviceAllocationsCache->allocations[2].saveTime = baseTimePoint + timeDiff * 2; - svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff); + svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false); EXPECT_EQ(2u, svmManager->usmDeviceAllocationsCache->allocations.size()); - svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff); + svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false); EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size()); - svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff); + svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false); EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size()); EXPECT_EQ(baseTimePoint + timeDiff * 2, svmManager->usmDeviceAllocationsCache->allocations[0].saveTime); @@ -987,7 +1062,7 @@ HWTEST_F(SvmHostAllocationCacheTest, givenOclApiSpecificConfigWhenCheckingIfEnab auto device = deviceFactory->rootDevices[0]; RAIIProductHelperFactory raii(*device->getExecutionEnvironment()->rootDeviceEnvironments[0]); const auto expectedMaxSize = static_cast(0.02 * device->getMemoryManager()->getSystemSharedMemory(0u)); - EXPECT_EQ(expectedMaxSize, device->getMemoryManager()->getMaxAllocationsSavedForReuseSize()); + EXPECT_EQ(expectedMaxSize, device->getMemoryManager()->usmReuseInfo.getMaxAllocationsSavedForReuseSize()); { raii.mockProductHelper->isHostUsmAllocationReuseSupportedResult = false; @@ -1005,6 +1080,37 @@ HWTEST_F(SvmHostAllocationCacheTest, givenOclApiSpecificConfigWhenCheckingIfEnab } } +TEST_F(SvmHostAllocationCacheTest, givenReuseLimitFlagWhenInitUsmReuseLimitCalledThenLimitThresholdSetCorrectly) { + std::unique_ptr deviceFactory(new UltDeviceFactory(1, 1)); + RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex}; + std::map deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}}; + auto device = deviceFactory->rootDevices[0]; + auto memoryManager = reinterpret_cast(device->getMemoryManager()); + + { + DebugManagerStateRestore restore; + debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.set(0); + memoryManager->initUsmReuseLimits(); + const auto expectedLimitThreshold = std::numeric_limits::max(); + EXPECT_EQ(expectedLimitThreshold, memoryManager->usmReuseInfo.getLimitAllocationsReuseThreshold()); + } + { + DebugManagerStateRestore restore; + debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.set(80); + memoryManager->initUsmReuseLimits(); + const auto systemSharedMemory = memoryManager->getSystemSharedMemory(device->getRootDeviceIndex()); + const auto expectedLimitThreshold = static_cast(0.8 * systemSharedMemory); + EXPECT_EQ(expectedLimitThreshold, memoryManager->usmReuseInfo.getLimitAllocationsReuseThreshold()); + } + { + DebugManagerStateRestore restore; + debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.set(-1); + memoryManager->initUsmReuseLimits(); + const auto expectedLimitThreshold = std::numeric_limits::max(); + EXPECT_EQ(expectedLimitThreshold, memoryManager->usmReuseInfo.getLimitAllocationsReuseThreshold()); + } +} + struct SvmHostAllocationCacheSimpleTestDataType { size_t allocationSize; void *allocation; @@ -1019,7 +1125,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingHostAll auto device = deviceFactory->rootDevices[0]; auto memoryManager = reinterpret_cast(device->getMemoryManager()); auto svmManager = std::make_unique(memoryManager, false); - memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache); @@ -1068,7 +1174,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledWhenInitializedThe ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache); auto expectedMaxSize = static_cast(svmManager->memoryManager->getSystemSharedMemory(mockRootDeviceIndex) * 0.02); - EXPECT_EQ(expectedMaxSize, device->getMemoryManager()->getMaxAllocationsSavedForReuseSize()); + EXPECT_EQ(expectedMaxSize, device->getMemoryManager()->usmReuseInfo.getMaxAllocationsSavedForReuseSize()); } TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingHostAllocationThenItIsPutIntoCacheOnlyIfMaxSizeWillNotBeExceeded) { @@ -1081,7 +1187,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingHostAll auto memoryManager = reinterpret_cast(device->getMemoryManager()); auto svmManager = std::make_unique(memoryManager, false); constexpr auto allocationSize = MemoryConstants::pageSize64k; - memoryManager->maxAllocationsSavedForReuseSize = allocationSize; + memoryManager->usmReuseInfo.init(allocationSize, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache); @@ -1092,26 +1198,26 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingHostAll auto allocation2 = svmManager->createHostUnifiedMemoryAllocation(1u, unifiedMemoryProperties); ASSERT_NE(allocation2, nullptr); EXPECT_EQ(0u, svmManager->usmHostAllocationsCache->allocations.size()); - EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAlloc(allocation); EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAlloc(allocation2); EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); auto recycledAllocation = svmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); EXPECT_EQ(recycledAllocation, allocation); EXPECT_EQ(svmManager->usmHostAllocationsCache->allocations.size(), 0u); - EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAlloc(recycledAllocation); svmManager->trimUSMHostAllocCache(); EXPECT_EQ(svmManager->usmHostAllocationsCache->allocations.size(), 0u); - EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); } { auto allocation = svmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); @@ -1119,29 +1225,62 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingHostAll auto allocation2 = svmManager->createHostUnifiedMemoryAllocation(1u, unifiedMemoryProperties); ASSERT_NE(allocation2, nullptr); EXPECT_EQ(0u, svmManager->usmHostAllocationsCache->allocations.size()); - EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAllocDefer(allocation); EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAllocDefer(allocation2); EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); auto recycledAllocation = svmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); EXPECT_EQ(recycledAllocation, allocation); EXPECT_EQ(svmManager->usmHostAllocationsCache->allocations.size(), 0u); - EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAllocDefer(recycledAllocation); svmManager->trimUSMHostAllocCache(); EXPECT_EQ(svmManager->usmHostAllocationsCache->allocations.size(), 0u); - EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); } } +TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingHostAllocationAndShouldLimitUsmReuseThenItIsNotPutIntoCache) { + std::unique_ptr deviceFactory(new UltDeviceFactory(1, 1)); + RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex}; + std::map deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}}; + DebugManagerStateRestore restore; + debugManager.flags.ExperimentalEnableHostAllocationCache.set(1); + auto device = deviceFactory->rootDevices[0]; + auto memoryManager = reinterpret_cast(device->getMemoryManager()); + auto svmManager = std::make_unique(memoryManager, false); + constexpr auto allocationSize = MemoryConstants::pageSize64k; + svmManager->initUsmAllocationsCaches(*device); + ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache); + + SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::hostUnifiedMemory, 1, rootDeviceIndices, deviceBitfields); + auto allocation = svmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); + ASSERT_NE(allocation, nullptr); + auto allocation2 = svmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); + ASSERT_NE(allocation2, nullptr); + EXPECT_EQ(0u, svmManager->usmHostAllocationsCache->allocations.size()); + EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); + + svmManager->freeSVMAlloc(allocation); + EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size()); + EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); + + memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, alwaysLimited); + svmManager->freeSVMAlloc(allocation2); + EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size()); + EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); + + svmManager->cleanupUSMAllocCaches(); +} + TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledAndMultipleSVMManagersWhenFreeingHostAllocationThenItIsPutIntoCacheOnlyIfMaxSizeWillNotBeExceeded) { std::unique_ptr deviceFactory(new UltDeviceFactory(1, 1)); RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex}; @@ -1153,7 +1292,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledAndMultipleSVMMana auto svmManager = std::make_unique(memoryManager, false); auto secondSvmManager = std::make_unique(memoryManager, false); constexpr auto allocationSize = MemoryConstants::pageSize64k; - memoryManager->maxAllocationsSavedForReuseSize = allocationSize; + memoryManager->usmReuseInfo.init(allocationSize, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); secondSvmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache); @@ -1167,26 +1306,26 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledAndMultipleSVMMana ASSERT_NE(allocation2, nullptr); EXPECT_EQ(0u, svmManager->usmHostAllocationsCache->allocations.size()); EXPECT_EQ(0u, secondSvmManager->usmHostAllocationsCache->allocations.size()); - EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAlloc(allocation); EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); secondSvmManager->freeSVMAlloc(allocation2); EXPECT_EQ(0u, secondSvmManager->usmHostAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); auto recycledAllocation = svmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); EXPECT_EQ(recycledAllocation, allocation); EXPECT_EQ(svmManager->usmHostAllocationsCache->allocations.size(), 0u); - EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAlloc(recycledAllocation); svmManager->trimUSMHostAllocCache(); EXPECT_EQ(svmManager->usmHostAllocationsCache->allocations.size(), 0u); - EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); } { auto allocation = svmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); @@ -1195,26 +1334,26 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledAndMultipleSVMMana ASSERT_NE(allocation2, nullptr); EXPECT_EQ(0u, svmManager->usmHostAllocationsCache->allocations.size()); EXPECT_EQ(0u, secondSvmManager->usmHostAllocationsCache->allocations.size()); - EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); secondSvmManager->freeSVMAlloc(allocation2); EXPECT_EQ(1u, secondSvmManager->usmHostAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); svmManager->freeSVMAlloc(allocation); EXPECT_EQ(0u, svmManager->usmHostAllocationsCache->allocations.size()); - EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); auto recycledAllocation = secondSvmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties); EXPECT_EQ(recycledAllocation, allocation2); EXPECT_EQ(secondSvmManager->usmHostAllocationsCache->allocations.size(), 0u); - EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); secondSvmManager->freeSVMAlloc(recycledAllocation); secondSvmManager->trimUSMHostAllocCache(); EXPECT_EQ(secondSvmManager->usmHostAllocationsCache->allocations.size(), 0u); - EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize()); + EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize()); } } @@ -1227,7 +1366,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationsWithDifferentSizesWhenAllocat auto device = deviceFactory->rootDevices[0]; auto memoryManager = reinterpret_cast(device->getMemoryManager()); auto svmManager = std::make_unique(memoryManager, false); - memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache); @@ -1279,7 +1418,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationsWithDifferentSizesWhenAllocat auto device = deviceFactory->rootDevices[0]; auto memoryManager = reinterpret_cast(device->getMemoryManager()); auto svmManager = std::make_unique(memoryManager, false); - memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache); @@ -1319,7 +1458,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationOverSizeLimitWhenAllocatingAft auto device = deviceFactory->rootDevices[0]; auto memoryManager = reinterpret_cast(device->getMemoryManager()); auto svmManager = std::make_unique(memoryManager, false); - memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache); const auto notAcceptedAllocSize = SVMAllocsManager::SvmAllocationCache::maxServicedSize + 1; @@ -1348,7 +1487,7 @@ TEST_F(SvmHostAllocationCacheTest, givenMultipleAllocationsWhenAllocatingAfterFr auto device = deviceFactory->rootDevices[0]; auto memoryManager = reinterpret_cast(device->getMemoryManager()); auto svmManager = std::make_unique(memoryManager, false); - memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache); @@ -1424,7 +1563,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationsWithDifferentFlagsWhenAllocat auto rootDevice = deviceFactory->rootDevices[0]; auto memoryManager = reinterpret_cast(rootDevice->getMemoryManager()); auto svmManager = std::make_unique(memoryManager, false); - memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*rootDevice); ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache); @@ -1491,7 +1630,7 @@ TEST_F(SvmHostAllocationCacheTest, givenHostOutOfMemoryWhenAllocatingThenCacheIs device->injectMemoryManager(new MockMemoryManagerWithCapacity(*device->getExecutionEnvironment())); MockMemoryManagerWithCapacity *memoryManager = static_cast(device->getMemoryManager()); auto svmManager = std::make_unique(memoryManager, false); - memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache); @@ -1529,7 +1668,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationInUsageWhenAllocatingAfterFree auto device = deviceFactory->rootDevices[0]; auto memoryManager = reinterpret_cast(device->getMemoryManager()); auto svmManager = std::make_unique(memoryManager, false); - memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); EXPECT_NE(nullptr, svmManager->usmHostAllocationsCache); @@ -1560,7 +1699,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCalle auto device = deviceFactory->rootDevices[0]; auto memoryManager = reinterpret_cast(device->getMemoryManager()); auto svmManager = std::make_unique(memoryManager, false); - memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte; + memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited); svmManager->initUsmAllocationsCaches(*device); EXPECT_NE(nullptr, svmManager->usmHostAllocationsCache); @@ -1583,13 +1722,13 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCalle svmManager->usmHostAllocationsCache->allocations[1].saveTime = baseTimePoint + timeDiff; svmManager->usmHostAllocationsCache->allocations[2].saveTime = baseTimePoint + timeDiff * 2; - svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff); + svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false); EXPECT_EQ(2u, svmManager->usmHostAllocationsCache->allocations.size()); - svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff); + svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false); EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size()); - svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff); + svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false); EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size()); EXPECT_EQ(baseTimePoint + timeDiff * 2, svmManager->usmHostAllocationsCache->allocations[0].saveTime);