Media
/
compute-runtime
mirror of https://github.com/intel/compute-runtime.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: flag to limit usm reuse based on memory usage 

Host usm and device usm for igfx checks system memory usage.
Device usm for dgfx checks local memory usage.

If used memory is above limit threshold:
- no new allocations will be saved for reuse
- cleaner will use shorter hold time of 2 seconds
- cleaner will free all eligible allocations, regardless of async
deleter thread having work

Motivation: in case of gfx memory being full, making resident new
allocations will require evictions which leads to massive slowdown on
enqueue calls.
This change aims to minimize cases where extra memory usage from usm
reuse mechanism leads to above situation.

Related-To: NEO-6893, NEO-14160

Signed-off-by: Dominik Dabek <dominik.dabek@intel.com>
This commit is contained in:
Dominik Dabek 2025-03-26 14:09:38 +00:00 committed by Compute-Runtime-Automation
parent 5684eb0dca
commit 915d657420
18 changed files with 373 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
shared/source/debug_settings/debug_variables_base.inl
View File

@ -592,6 +592,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalEnableDeviceAllocationCache, -1, "Ex
DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalEnableHostAllocationCache, -1, "Experimentally enable host usm allocation cache. Use X% of shared system memory.")
DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalUSMAllocationReuseVersion, -1, "Version of mechanism to use for usm allocation reuse.")
DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalUSMAllocationReuseCleaner, -1, "Enable usm allocation reuse cleaner. -1: default, 0: disable, 1:enable")
DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalUSMAllocationReuseLimitThreshold, -1, "Threshold of used memory to limit usm reuse. -1: default, 0: disable, >0:X% of shared/device memory")
DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalH2DCpuCopyThreshold, -1, "Override default threshold (in bytes) for H2D CPU copy.")
DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalD2HCpuCopyThreshold, -1, "Override default threshold (in bytes) for D2H CPU copy.")
DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalCopyThroughLock, -1, "Experimentally copy memory through locked ptr. -1: default 0: disable 1: enable ")

26
shared/source/device/device.cpp
View File

@ -210,10 +210,9 @@ void Device::initializeCommonResources() {
deviceBitfields.emplace(getRootDeviceIndex(), getDeviceBitfield());
deviceUsmMemAllocPoolsManager.reset(new UsmMemAllocPoolsManager(getMemoryManager(), rootDeviceIndices, deviceBitfields, this, InternalMemoryType::deviceUnifiedMemory));
}
initUsmReuseMaxSize();
}
void Device::initUsmReuseMaxSize() {
void Device::initUsmReuseLimits() {
const bool usmDeviceAllocationsCacheEnabled = NEO::ApiSpecificConfig::isDeviceAllocationCacheEnabled() && this->getProductHelper().isDeviceUsmAllocationReuseSupported();
auto ailConfiguration = this->getAilConfigurationHelper();
const bool limitDeviceMemoryForReuse = ailConfiguration && ailConfiguration->limitAmountOfDeviceMemoryForRecycling();
@ -222,7 +221,27 @@ void Device::initUsmReuseMaxSize() {
fractionOfTotalMemoryForRecycling = 0.01 * std::min(100, debugManager.flags.ExperimentalEnableDeviceAllocationCache.get());
}
const auto totalDeviceMemory = this->getGlobalMemorySize(static_cast<uint32_t>(this->getDeviceBitfield().to_ulong()));
this->maxAllocationsSavedForReuseSize = static_cast<uint64_t>(fractionOfTotalMemoryForRecycling * totalDeviceMemory);
auto maxAllocationsSavedForReuseSize = static_cast<uint64_t>(fractionOfTotalMemoryForRecycling * totalDeviceMemory);
auto limitAllocationsReuseThreshold = UsmReuseInfo::notLimited;
const auto limitFlagValue = debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.get();
if (limitFlagValue != -1) {
if (limitFlagValue == 0) {
limitAllocationsReuseThreshold = UsmReuseInfo::notLimited;
} else {
const auto fractionOfTotalMemoryToLimitReuse = limitFlagValue / 100.0;
limitAllocationsReuseThreshold = static_cast<uint64_t>(fractionOfTotalMemoryToLimitReuse * totalDeviceMemory);
}
}
this->usmReuseInfo.init(maxAllocationsSavedForReuseSize, limitAllocationsReuseThreshold);
}
bool Device::shouldLimitAllocationsReuse() const {
const bool isIntegratedDevice = getHardwareInfo().capabilityTable.isIntegratedDevice;
if (isIntegratedDevice) {
return getMemoryManager()->shouldLimitAllocationsReuse();
}
return getMemoryManager()->getUsedLocalMemorySize(getRootDeviceIndex()) >= this->usmReuseInfo.getLimitAllocationsReuseThreshold();
}
bool Device::initDeviceFully() {
@ -269,6 +288,7 @@ bool Device::initDeviceFully() {
createBindlessHeapsHelper();
uuid.isValid = false;
initUsmReuseLimits();
if (getRootDeviceEnvironment().osInterface == nullptr) {
return true;

27
shared/source/device/device.h
View File

@ -13,6 +13,7 @@
#include "shared/source/helpers/engine_node_helper.h"
#include "shared/source/helpers/non_copyable_or_moveable.h"
#include "shared/source/helpers/options.h"
#include "shared/source/memory_manager/unified_memory_reuse.h"
#include "shared/source/os_interface/performance_counters.h"
#include "shared/source/os_interface/product_helper.h"
#include "shared/source/utilities/isa_pool_allocator.h"
@ -222,25 +223,8 @@ class Device : public ReferenceTrackedObject<Device>, NEO::NonCopyableAndNonMova
void stopDirectSubmissionForCopyEngine();
uint64_t getMaxAllocationsSavedForReuseSize() const {
return maxAllocationsSavedForReuseSize;
}
bool shouldLimitAllocationsReuse() const;
std::unique_lock<std::mutex> obtainAllocationsReuseLock() const {
return std::unique_lock<std::mutex>(allocationsReuseMtx);
}
void recordAllocationSaveForReuse(size_t size) {
allocationsSavedForReuseSize += size;
}
void recordAllocationGetFromReuse(size_t size) {
allocationsSavedForReuseSize -= size;
}
uint64_t getAllocationsSavedForReuseSize() const {
return allocationsSavedForReuseSize;
}
uint32_t getMicrosecondResolution() const {
return microsecondResolution;
}
@ -262,6 +246,8 @@ class Device : public ReferenceTrackedObject<Device>, NEO::NonCopyableAndNonMova
bufferPoolCount -= size;
}
UsmReuseInfo usmReuseInfo;
protected:
Device() = delete;
Device(ExecutionEnvironment *executionEnvironment, const uint32_t rootDeviceIndex);
@ -281,7 +267,7 @@ class Device : public ReferenceTrackedObject<Device>, NEO::NonCopyableAndNonMova
bool initDeviceWithEngines();
void initializeCommonResources();
bool initDeviceFully();
void initUsmReuseMaxSize();
void initUsmReuseLimits();
virtual bool createEngines();
void addEngineToEngineGroup(EngineControl &engine);
@ -339,11 +325,8 @@ class Device : public ReferenceTrackedObject<Device>, NEO::NonCopyableAndNonMova
ISAPoolAllocator isaPoolAllocator;
std::unique_ptr<UsmMemAllocPoolsManager> deviceUsmMemAllocPoolsManager;
uint64_t allocationsSavedForReuseSize = 0u;
uint64_t maxAllocationsSavedForReuseSize = 0u;
std::atomic_uint32_t bufferPoolCount = 0u;
uint32_t maxBufferPoolCount = 0u;
mutable std::mutex allocationsReuseMtx;
uint32_t microsecondResolution = 1000u;
struct {

2
shared/source/execution_environment/execution_environment.cpp
View File

@ -88,7 +88,7 @@ bool ExecutionEnvironment::initializeMemoryManager() {
} break;
}
memoryManager->initUsmReuseMaxSize();
memoryManager->initUsmReuseLimits();
return memoryManager->isInitialized();
}

1
shared/source/memory_manager/CMakeLists.txt
View File

@ -59,6 +59,7 @@ set(NEO_CORE_MEMORY_MANAGER
${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_manager.h
${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_pooling.cpp
${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_pooling.h
${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_reuse.h
${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_reuse_cleaner.cpp
${CMAKE_CURRENT_SOURCE_DIR}/unified_memory_reuse_cleaner.h
${CMAKE_CURRENT_SOURCE_DIR}/page_table.cpp

22
shared/source/memory_manager/memory_manager.cpp
View File

@ -389,13 +389,25 @@ uint32_t MemoryManager::getFirstContextIdForRootDevice(uint32_t rootDeviceIndex)
return 0;
}
void MemoryManager::initUsmReuseMaxSize() {
const auto totalSystemMemory = this->getSystemSharedMemory(0u);
auto fractionOfTotalMemoryForRecycling = 0.02;
void MemoryManager::initUsmReuseLimits() {
const auto systemSharedMemorySize = this->getSystemSharedMemory(0u);
auto fractionOfTotalMemoryForReuse = 0.02;
if (debugManager.flags.ExperimentalEnableHostAllocationCache.get() != -1) {
fractionOfTotalMemoryForRecycling = 0.01 * std::min(100, debugManager.flags.ExperimentalEnableHostAllocationCache.get());
fractionOfTotalMemoryForReuse = 0.01 * std::min(100, debugManager.flags.ExperimentalEnableHostAllocationCache.get());
}
this->maxAllocationsSavedForReuseSize = static_cast<size_t>(fractionOfTotalMemoryForRecycling * totalSystemMemory);
auto maxAllocationsSavedForReuseSize = static_cast<uint64_t>(fractionOfTotalMemoryForReuse * systemSharedMemorySize);
auto limitAllocationsReuseThreshold = std::numeric_limits<uint64_t>::max();
const auto limitFlagValue = debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.get();
if (limitFlagValue != -1) {
if (limitFlagValue == 0) {
limitAllocationsReuseThreshold = std::numeric_limits<uint64_t>::max();
} else {
const auto fractionOfTotalMemoryToLimitReuse = limitFlagValue / 100.0;
limitAllocationsReuseThreshold = static_cast<uint64_t>(fractionOfTotalMemoryToLimitReuse * systemSharedMemorySize);
}
}
this->usmReuseInfo.init(maxAllocationsSavedForReuseSize, limitAllocationsReuseThreshold);
}
OsContext *MemoryManager::createAndRegisterOsContext(CommandStreamReceiver *commandStreamReceiver,

27
shared/source/memory_manager/memory_manager.h
View File

@ -12,6 +12,7 @@
#include "shared/source/memory_manager/alignment_selector.h"
#include "shared/source/memory_manager/graphics_allocation.h"
#include "shared/source/memory_manager/memadvise_flags.h"
#include "shared/source/memory_manager/unified_memory_reuse.h"
#include "shared/source/os_interface/os_memory.h"
#include "shared/source/utilities/stackvec.h"
@ -338,26 +339,11 @@ class MemoryManager {
virtual void getExtraDeviceProperties(uint32_t rootDeviceIndex, uint32_t *moduleId, uint16_t *serverType) { return; }
std::unique_lock<std::mutex> obtainHostAllocationsReuseLock() const {
return std::unique_lock<std::mutex>(hostAllocationsReuseMtx);
}
void initUsmReuseLimits();
UsmReuseInfo usmReuseInfo;
void initUsmReuseMaxSize();
uint64_t getMaxAllocationsSavedForReuseSize() const {
return maxAllocationsSavedForReuseSize;
}
void recordHostAllocationSaveForReuse(uint64_t size) {
hostAllocationsSavedForReuseSize += size;
}
void recordHostAllocationGetFromReuse(uint64_t size) {
hostAllocationsSavedForReuseSize -= size;
}
uint64_t getHostAllocationsSavedForReuseSize() const {
return hostAllocationsSavedForReuseSize;
bool shouldLimitAllocationsReuse() const {
return getUsedSystemMemorySize() >= usmReuseInfo.getLimitAllocationsReuseThreshold();
}
void addCustomHeapAllocatorConfig(AllocationType allocationType, bool isFrontWindowPool, const CustomHeapAllocatorConfig &config);
@ -436,9 +422,6 @@ class MemoryManager {
std::mutex physicalMemoryAllocationMapMutex;
std::unique_ptr<std::atomic<size_t>[]> localMemAllocsSize;
std::atomic<size_t> sysMemAllocsSize;
uint64_t maxAllocationsSavedForReuseSize = 0u;
uint64_t hostAllocationsSavedForReuseSize = 0u;
mutable std::mutex hostAllocationsReuseMtx;
std::map<std::pair<AllocationType, bool>, CustomHeapAllocatorConfig> customHeapAllocators;
};

53
shared/source/memory_manager/unified_memory_manager.cpp
View File

@ -64,21 +64,24 @@ bool SVMAllocsManager::SvmAllocationCache::insert(size_t size, void *ptr, SvmAll
if (false == sizeAllowed(size)) {
return false;
}
if (svmData->device ? svmData->device->shouldLimitAllocationsReuse() : memoryManager->shouldLimitAllocationsReuse()) {
return false;
}
std::lock_guard<std::mutex> lock(this->mtx);
bool isSuccess = true;
if (auto device = svmData->device) {
auto lock = device->obtainAllocationsReuseLock();
if (size + device->getAllocationsSavedForReuseSize() > device->getMaxAllocationsSavedForReuseSize()) {
auto lock = device->usmReuseInfo.obtainAllocationsReuseLock();
if (size + device->usmReuseInfo.getAllocationsSavedForReuseSize() > device->usmReuseInfo.getMaxAllocationsSavedForReuseSize()) {
isSuccess = false;
} else {
device->recordAllocationSaveForReuse(size);
device->usmReuseInfo.recordAllocationSaveForReuse(size);
}
} else {
auto lock = memoryManager->obtainHostAllocationsReuseLock();
if (size + memoryManager->getHostAllocationsSavedForReuseSize() > memoryManager->getMaxAllocationsSavedForReuseSize()) {
auto lock = memoryManager->usmReuseInfo.obtainAllocationsReuseLock();
if (size + memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize() > memoryManager->usmReuseInfo.getMaxAllocationsSavedForReuseSize()) {
isSuccess = false;
} else {
memoryManager->recordHostAllocationSaveForReuse(size);
memoryManager->usmReuseInfo.recordAllocationSaveForReuse(size);
}
}
if (isSuccess) {
@ -133,11 +136,11 @@ void *SVMAllocsManager::SvmAllocationCache::get(size_t size, const UnifiedMemory
svmData->allocationFlagsProperty.allAllocFlags == unifiedMemoryProperties.allocationFlags.allAllocFlags &&
false == isInUse(svmData)) {
if (svmData->device) {
auto lock = svmData->device->obtainAllocationsReuseLock();
svmData->device->recordAllocationGetFromReuse(allocationIter->allocationSize);
auto lock = svmData->device->usmReuseInfo.obtainAllocationsReuseLock();
svmData->device->usmReuseInfo.recordAllocationGetFromReuse(allocationIter->allocationSize);
} else {
auto lock = memoryManager->obtainHostAllocationsReuseLock();
memoryManager->recordHostAllocationGetFromReuse(allocationIter->allocationSize);
auto lock = memoryManager->usmReuseInfo.obtainAllocationsReuseLock();
memoryManager->usmReuseInfo.recordAllocationGetFromReuse(allocationIter->allocationSize);
}
if (enablePerformanceLogging) {
logCacheOperation({.allocationSize = allocationIter->allocationSize,
@ -167,11 +170,11 @@ void SVMAllocsManager::SvmAllocationCache::trim() {
SvmAllocationData *svmData = svmAllocsManager->getSVMAlloc(cachedAllocationInfo.allocation);
UNRECOVERABLE_IF(nullptr == svmData);
if (svmData->device) {
auto lock = svmData->device->obtainAllocationsReuseLock();
svmData->device->recordAllocationGetFromReuse(cachedAllocationInfo.allocationSize);
auto lock = svmData->device->usmReuseInfo.obtainAllocationsReuseLock();
svmData->device->usmReuseInfo.recordAllocationGetFromReuse(cachedAllocationInfo.allocationSize);
} else {
auto lock = memoryManager->obtainHostAllocationsReuseLock();
memoryManager->recordHostAllocationGetFromReuse(cachedAllocationInfo.allocationSize);
auto lock = memoryManager->usmReuseInfo.obtainAllocationsReuseLock();
memoryManager->usmReuseInfo.recordAllocationGetFromReuse(cachedAllocationInfo.allocationSize);
}
if (enablePerformanceLogging) {
logCacheOperation({.allocationSize = cachedAllocationInfo.allocationSize,
@ -234,7 +237,7 @@ void SVMAllocsManager::SvmAllocationCache::logCacheOperation(const SvmAllocation
isSuccessString);
}
void SVMAllocsManager::SvmAllocationCache::trimOldAllocs(std::chrono::high_resolution_clock::time_point trimTimePoint) {
void SVMAllocsManager::SvmAllocationCache::trimOldAllocs(std::chrono::high_resolution_clock::time_point trimTimePoint, bool shouldLimitReuse) {
if (this->allocations.empty()) {
return;
}
@ -248,11 +251,11 @@ void SVMAllocsManager::SvmAllocationCache::trimOldAllocs(std::chrono::high_resol
SvmAllocationData *svmData = svmAllocsManager->getSVMAlloc(allocationPtr);
UNRECOVERABLE_IF(nullptr == svmData);
if (svmData->device) {
auto lock = svmData->device->obtainAllocationsReuseLock();
svmData->device->recordAllocationGetFromReuse(allocCleanCandidate->allocationSize);
auto lock = svmData->device->usmReuseInfo.obtainAllocationsReuseLock();
svmData->device->usmReuseInfo.recordAllocationGetFromReuse(allocCleanCandidate->allocationSize);
} else {
auto lock = memoryManager->obtainHostAllocationsReuseLock();
memoryManager->recordHostAllocationGetFromReuse(allocCleanCandidate->allocationSize);
auto lock = memoryManager->usmReuseInfo.obtainAllocationsReuseLock();
memoryManager->usmReuseInfo.recordAllocationGetFromReuse(allocCleanCandidate->allocationSize);
}
if (enablePerformanceLogging) {
logCacheOperation({.allocationSize = allocCleanCandidate->allocationSize,
@ -262,8 +265,12 @@ void SVMAllocsManager::SvmAllocationCache::trimOldAllocs(std::chrono::high_resol
.isSuccess = true});
}
svmAllocsManager->freeSVMAllocImpl(allocCleanCandidate->allocation, FreePolicyType::defer, svmData);
allocations.erase(allocCleanCandidate);
break;
if (shouldLimitReuse) {
allocCleanCandidate = allocations.erase(allocCleanCandidate);
} else {
allocations.erase(allocCleanCandidate);
break;
}
}
}
@ -901,7 +908,7 @@ void SVMAllocsManager::freeZeroCopySvmAllocation(SvmAllocationData *svmData) {
void SVMAllocsManager::initUsmDeviceAllocationsCache(Device &device) {
this->usmDeviceAllocationsCache.reset(new SvmAllocationCache);
if (device.getMaxAllocationsSavedForReuseSize() > 0u) {
if (device.usmReuseInfo.getMaxAllocationsSavedForReuseSize() > 0u) {
this->usmDeviceAllocationsCache->allocations.reserve(128u);
this->usmDeviceAllocationsCache->svmAllocsManager = this;
this->usmDeviceAllocationsCache->memoryManager = memoryManager;
@ -913,7 +920,7 @@ void SVMAllocsManager::initUsmDeviceAllocationsCache(Device &device) {
void SVMAllocsManager::initUsmHostAllocationsCache() {
this->usmHostAllocationsCache.reset(new SvmAllocationCache);
if (memoryManager->getMaxAllocationsSavedForReuseSize() > 0u) {
if (memoryManager->usmReuseInfo.getMaxAllocationsSavedForReuseSize() > 0u) {
this->usmHostAllocationsCache->allocations.reserve(128u);
this->usmHostAllocationsCache->svmAllocsManager = this;
this->usmHostAllocationsCache->memoryManager = memoryManager;

2
shared/source/memory_manager/unified_memory_manager.h
View File

@ -194,7 +194,7 @@ class SVMAllocsManager {
bool isInUse(SvmAllocationData *svmData);
void *get(size_t size, const UnifiedMemoryProperties &unifiedMemoryProperties);
void trim();
void trimOldAllocs(std::chrono::high_resolution_clock::time_point trimTimePoint);
void trimOldAllocs(std::chrono::high_resolution_clock::time_point trimTimePoint, bool shouldLimitReuse);
void cleanup();
void logCacheOperation(const SvmAllocationCachePerfInfo &cachePerfEvent) const;

52
shared/source/memory_manager/unified_memory_reuse.h Normal file
View File

@ -0,0 +1,52 @@
/*
* Copyright (C) 2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include <cstdint>
#include <mutex>
namespace NEO {
struct UsmReuseInfo {
uint64_t getAllocationsSavedForReuseSize() const {
return allocationsSavedForReuseSize;
}
uint64_t getMaxAllocationsSavedForReuseSize() const {
return maxAllocationsSavedForReuseSize;
}
uint64_t getLimitAllocationsReuseThreshold() const {
return limitAllocationsReuseThreshold;
}
std::unique_lock<std::mutex> obtainAllocationsReuseLock() const {
return std::unique_lock<std::mutex>(allocationsReuseMtx);
}
void recordAllocationSaveForReuse(size_t size) {
allocationsSavedForReuseSize += size;
}
void recordAllocationGetFromReuse(size_t size) {
allocationsSavedForReuseSize -= size;
}
void init(uint64_t maxAllocationsSavedForReuseSize, uint64_t limitAllocationsReuseThreshold) {
this->maxAllocationsSavedForReuseSize = maxAllocationsSavedForReuseSize;
this->limitAllocationsReuseThreshold = limitAllocationsReuseThreshold;
}
static constexpr uint64_t notLimited = std::numeric_limits<uint64_t>::max();
protected:
uint64_t maxAllocationsSavedForReuseSize = 0u;
uint64_t limitAllocationsReuseThreshold = 0u;
uint64_t allocationsSavedForReuseSize = 0u;
mutable std::mutex allocationsReuseMtx;
};
} // namespace NEO

16
shared/source/memory_manager/unified_memory_reuse_cleaner.cpp
View File

@ -61,15 +61,21 @@ void UnifiedMemoryReuseCleaner::unregisterSvmAllocationCache(SvmAllocationCache
}
void UnifiedMemoryReuseCleaner::trimOldInCaches() {
const std::chrono::high_resolution_clock::time_point trimTimePoint = std::chrono::high_resolution_clock::now() - maxHoldTime;
bool shouldLimitReuse = false;
auto trimTimePoint = std::chrono::high_resolution_clock::now() - maxHoldTime;
std::lock_guard<std::mutex> lockSvmAllocationCaches(this->svmAllocationCachesMutex);
for (auto svmAllocCache : this->svmAllocationCaches) {
if (auto deferredDeleter = svmAllocCache->memoryManager->getDeferredDeleter()) {
if (false == deferredDeleter->areElementsReleased(false)) {
continue;
shouldLimitReuse |= svmAllocCache->memoryManager->shouldLimitAllocationsReuse();
if (shouldLimitReuse) {
trimTimePoint = std::chrono::high_resolution_clock::now() - limitedHoldTime;
} else {
if (auto deferredDeleter = svmAllocCache->memoryManager->getDeferredDeleter()) {
if (false == deferredDeleter->areElementsReleased(false)) {
continue;
}
}
}
svmAllocCache->trimOldAllocs(trimTimePoint);
svmAllocCache->trimOldAllocs(trimTimePoint, shouldLimitReuse);
}
}

1
shared/source/memory_manager/unified_memory_reuse_cleaner.h
View File

@ -22,6 +22,7 @@ class UnifiedMemoryReuseCleaner : NEO::NonCopyableAndNonMovableClass {
public:
static constexpr auto sleepTime = std::chrono::milliseconds(15u);
static constexpr auto maxHoldTime = std::chrono::seconds(10u);
static constexpr auto limitedHoldTime = std::chrono::seconds(2u);
UnifiedMemoryReuseCleaner();
virtual ~UnifiedMemoryReuseCleaner();

4
shared/test/common/mocks/mock_device.h
View File

@ -33,7 +33,6 @@ extern CommandStreamReceiver *createCommandStream(ExecutionEnvironment &executio
struct MockSubDevice : public SubDevice {
using Device::allEngines;
using Device::createEngines;
using Device::maxAllocationsSavedForReuseSize;
using SubDevice::getDeviceBitfield;
using SubDevice::getGlobalMemorySize;
using SubDevice::SubDevice;
@ -67,8 +66,7 @@ class MockDevice : public RootDevice {
using Device::generateUuidFromPciBusInfo;
using Device::getGlobalMemorySize;
using Device::initializeCaps;
using Device::initUsmReuseMaxSize;
using Device::maxAllocationsSavedForReuseSize;
using Device::initUsmReuseLimits;
using Device::maxBufferPoolCount;
using Device::microsecondResolution;
using Device::preemptionMode;

1
shared/test/common/mocks/mock_memory_manager.h
View File

@ -62,7 +62,6 @@ class MockMemoryManager : public MemoryManagerCreate<OsAgnosticMemoryManager> {
using MemoryManager::latestContextId;
using MemoryManager::localMemAllocsSize;
using MemoryManager::localMemorySupported;
using MemoryManager::maxAllocationsSavedForReuseSize;
using MemoryManager::reservedMemory;
using MemoryManager::secondaryEngines;

2
shared/test/common/mocks/ult_device_factory.cpp
View File

@ -94,7 +94,7 @@ bool UltDeviceFactory::initializeMemoryManager(ExecutionEnvironment &executionEn
bool enableLocalMemory = gfxCoreHelper.getEnableLocalMemory(*defaultHwInfo);
bool aubUsage = (testMode == TestMode::aubTests) || (testMode == TestMode::aubTestsWithTbx);
executionEnvironment.memoryManager.reset(new MockMemoryManager(false, enableLocalMemory, aubUsage, executionEnvironment));
executionEnvironment.memoryManager->initUsmReuseMaxSize();
executionEnvironment.memoryManager->initUsmReuseLimits();
}
return true;
}

1
shared/test/common/test_files/igdrcl.config
View File

@ -668,4 +668,5 @@ EnableDeferBacking = 0
SetMaxBVHLevels = -1
GetSipBinaryFromExternalLib = -1
LogUsmReuse = 0
ExperimentalUSMAllocationReuseLimitThreshold = -1
# Please don't edit below this line

2
shared/test/unit_test/execution_environment/execution_environment_tests.cpp
View File

@ -398,7 +398,7 @@ TEST(ExecutionEnvironment, givenExecutionEnvironmentWhenInitializeMemoryManagerI
executionEnvironment.initializeMemoryManager();
ASSERT_NE(nullptr, executionEnvironment.memoryManager);
EXPECT_TRUE(executionEnvironment.memoryManager->isInitialized());
EXPECT_NE(0u, executionEnvironment.memoryManager->getMaxAllocationsSavedForReuseSize());
EXPECT_NE(0u, executionEnvironment.memoryManager->usmReuseInfo.getMaxAllocationsSavedForReuseSize());
}
static_assert(sizeof(ExecutionEnvironment) == sizeof(std::unique_ptr<MemoryManager>) +

303
shared/test/unit_test/memory_manager/unified_memory_manager_cache_tests.cpp
View File

@ -148,6 +148,7 @@ struct SvmAllocationCacheTestFixture {
void tearDown() {
}
static constexpr size_t allocationSizeBasis = MemoryConstants::pageSize64k;
static constexpr uint64_t alwaysLimited = 0u;
MockExecutionEnvironment executionEnvironment;
};
@ -166,11 +167,11 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheDisabledWhenCheckingIfE
TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledAndMaxSizeZeroWhenCallingCleanupThenNoError) {
DebugManagerStateRestore restorer;
debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1);
std::unique_ptr<UltDeviceFactory> deviceFactory(new UltDeviceFactory(1, 1));
auto device = deviceFactory->rootDevices[0];
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1);
device->maxAllocationsSavedForReuseSize = 0u;
device->usmReuseInfo.init(0, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
svmManager->cleanupUSMAllocCaches();
@ -185,8 +186,8 @@ HWTEST_F(SvmDeviceAllocationCacheTest, givenOclApiSpecificConfigWhenCheckingIfEn
device->mockAilConfigurationHelper = &mockAilConfigurationHelper;
{
raii.mockProductHelper->isDeviceUsmAllocationReuseSupportedResult = false;
device->initUsmReuseMaxSize();
EXPECT_EQ(0u, device->getMaxAllocationsSavedForReuseSize());
device->initUsmReuseLimits();
EXPECT_EQ(0u, device->usmReuseInfo.getMaxAllocationsSavedForReuseSize());
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
EXPECT_EQ(nullptr, svmManager->usmDeviceAllocationsCache);
svmManager->initUsmAllocationsCaches(*device);
@ -194,23 +195,23 @@ HWTEST_F(SvmDeviceAllocationCacheTest, givenOclApiSpecificConfigWhenCheckingIfEn
}
{
raii.mockProductHelper->isDeviceUsmAllocationReuseSupportedResult = true;
device->initUsmReuseMaxSize();
device->initUsmReuseLimits();
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
EXPECT_EQ(nullptr, svmManager->usmDeviceAllocationsCache);
svmManager->initUsmAllocationsCaches(*device);
EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
const auto expectedMaxSize = static_cast<size_t>(0.08 * device->getGlobalMemorySize(static_cast<uint32_t>(device->getDeviceBitfield().to_ullong())));
EXPECT_EQ(expectedMaxSize, device->getMaxAllocationsSavedForReuseSize());
EXPECT_EQ(expectedMaxSize, device->usmReuseInfo.getMaxAllocationsSavedForReuseSize());
}
{
raii.mockProductHelper->isDeviceUsmAllocationReuseSupportedResult = true;
mockAilConfigurationHelper.limitAmountOfDeviceMemoryForRecyclingReturn = true;
device->initUsmReuseMaxSize();
device->initUsmReuseLimits();
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
EXPECT_EQ(nullptr, svmManager->usmDeviceAllocationsCache);
svmManager->initUsmAllocationsCaches(*device);
EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
EXPECT_EQ(0u, device->getMaxAllocationsSavedForReuseSize());
EXPECT_EQ(0u, device->usmReuseInfo.getMaxAllocationsSavedForReuseSize());
}
}
@ -231,6 +232,36 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledWhenDirectSubmis
EXPECT_FALSE(device->getExecutionEnvironment()->unifiedMemoryReuseCleaner.get());
}
TEST_F(SvmDeviceAllocationCacheTest, givenReuseLimitFlagWhenInitUsmReuseLimitCalledThenLimitThresholdSetCorrectly) {
std::unique_ptr<UltDeviceFactory> deviceFactory(new UltDeviceFactory(1, 1));
RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex};
std::map<uint32_t, DeviceBitfield> deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}};
auto device = deviceFactory->rootDevices[0];
{
DebugManagerStateRestore restore;
debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.set(0);
device->initUsmReuseLimits();
const auto expectedLimitThreshold = std::numeric_limits<uint64_t>::max();
EXPECT_EQ(expectedLimitThreshold, device->usmReuseInfo.getLimitAllocationsReuseThreshold());
}
{
DebugManagerStateRestore restore;
debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.set(80);
device->initUsmReuseLimits();
const auto totalDeviceMemory = device->getGlobalMemorySize(static_cast<uint32_t>(device->getDeviceBitfield().to_ulong()));
const auto expectedLimitThreshold = static_cast<uint64_t>(0.8 * totalDeviceMemory);
EXPECT_EQ(expectedLimitThreshold, device->usmReuseInfo.getLimitAllocationsReuseThreshold());
}
{
DebugManagerStateRestore restore;
debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.set(-1);
device->initUsmReuseLimits();
const auto expectedLimitThreshold = std::numeric_limits<uint64_t>::max();
EXPECT_EQ(expectedLimitThreshold, device->usmReuseInfo.getLimitAllocationsReuseThreshold());
}
}
struct SvmDeviceAllocationCacheSimpleTestDataType {
size_t allocationSize;
void *allocation;
@ -244,7 +275,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingDevic
debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1);
auto device = deviceFactory->rootDevices[0];
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
@ -294,7 +325,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledWhenInitializedT
ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
auto expectedMaxSize = static_cast<uint64_t>(device->getGlobalMemorySize(static_cast<uint32_t>(mockDeviceBitfield.to_ulong())) * 0.02);
EXPECT_EQ(expectedMaxSize, device->getMaxAllocationsSavedForReuseSize());
EXPECT_EQ(expectedMaxSize, device->usmReuseInfo.getMaxAllocationsSavedForReuseSize());
}
TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingDeviceAllocationThenItIsPutIntoCacheOnlyIfMaxSizeWillNotBeExceeded) {
@ -306,7 +337,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingDevic
auto device = deviceFactory->rootDevices[0];
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
constexpr auto allocationSize = MemoryConstants::pageSize64k;
device->maxAllocationsSavedForReuseSize = allocationSize;
device->usmReuseInfo.init(allocationSize, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
@ -318,26 +349,26 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingDevic
auto allocation2 = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
ASSERT_NE(allocation2, nullptr);
EXPECT_EQ(0u, svmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAlloc(allocation);
EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAlloc(allocation2);
EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize());
auto recycledAllocation = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
EXPECT_EQ(recycledAllocation, allocation);
EXPECT_EQ(svmManager->usmDeviceAllocationsCache->allocations.size(), 0u);
EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAlloc(recycledAllocation);
svmManager->trimUSMDeviceAllocCache();
EXPECT_EQ(svmManager->usmDeviceAllocationsCache->allocations.size(), 0u);
EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize());
}
{
auto allocation = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
@ -345,26 +376,26 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingDevic
auto allocation2 = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
ASSERT_NE(allocation2, nullptr);
EXPECT_EQ(0u, svmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAllocDefer(allocation);
EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAllocDefer(allocation2);
EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize());
auto recycledAllocation = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
EXPECT_EQ(recycledAllocation, allocation);
EXPECT_EQ(svmManager->usmDeviceAllocationsCache->allocations.size(), 0u);
EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAllocDefer(recycledAllocation);
svmManager->trimUSMDeviceAllocCache();
EXPECT_EQ(svmManager->usmDeviceAllocationsCache->allocations.size(), 0u);
EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize());
}
}
@ -378,7 +409,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledAndMultipleSVMMa
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
auto secondSvmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
constexpr auto allocationSize = MemoryConstants::pageSize64k;
device->maxAllocationsSavedForReuseSize = allocationSize;
device->usmReuseInfo.init(allocationSize, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
secondSvmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
@ -394,22 +425,22 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledAndMultipleSVMMa
svmManager->freeSVMAlloc(allocation);
EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize());
secondSvmManager->freeSVMAlloc(allocation2);
EXPECT_EQ(0u, secondSvmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize());
auto recycledAllocation = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
EXPECT_EQ(recycledAllocation, allocation);
EXPECT_EQ(0u, svmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAlloc(recycledAllocation);
svmManager->trimUSMDeviceAllocCache();
EXPECT_EQ(svmManager->usmDeviceAllocationsCache->allocations.size(), 0u);
EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize());
}
{
auto allocation = svmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
@ -419,22 +450,22 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationCacheEnabledAndMultipleSVMMa
secondSvmManager->freeSVMAlloc(allocation2);
EXPECT_EQ(1u, secondSvmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAlloc(allocation);
EXPECT_EQ(0u, svmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, device->usmReuseInfo.getAllocationsSavedForReuseSize());
auto recycledAllocation = secondSvmManager->createUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
EXPECT_EQ(recycledAllocation, allocation2);
EXPECT_EQ(0u, secondSvmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize());
secondSvmManager->freeSVMAlloc(recycledAllocation);
secondSvmManager->trimUSMDeviceAllocCache();
EXPECT_EQ(secondSvmManager->usmDeviceAllocationsCache->allocations.size(), 0u);
EXPECT_EQ(0u, device->getAllocationsSavedForReuseSize());
EXPECT_EQ(0u, device->usmReuseInfo.getAllocationsSavedForReuseSize());
}
}
@ -446,7 +477,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsWithDifferentSizesWhenAlloc
debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1);
auto device = deviceFactory->rootDevices[0];
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
@ -495,7 +526,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationWithDifferentSizeWhenAllocat
debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1);
auto device = deviceFactory->rootDevices[0];
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
@ -540,7 +571,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsWithDifferentSizesWhenAlloc
debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1);
auto device = deviceFactory->rootDevices[0];
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
@ -580,7 +611,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationOverSizeLimitWhenAllocatingA
debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1);
auto device = deviceFactory->rootDevices[0];
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
const auto notAcceptedAllocSize = SVMAllocsManager::SvmAllocationCache::maxServicedSize + 1;
@ -610,7 +641,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenMultipleAllocationsWhenAllocatingAfter
debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1);
auto device = deviceFactory->rootDevices[0];
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
@ -688,9 +719,9 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsWithDifferentFlagsWhenAlloc
auto secondRootDevice = deviceFactory->rootDevices[1];
auto subDevice1 = reinterpret_cast<MockSubDevice *>(deviceFactory->subDevices[0]);
auto svmManager = std::make_unique<MockSVMAllocsManager>(rootDevice->getMemoryManager(), false);
rootDevice->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
secondRootDevice->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
subDevice1->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
rootDevice->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
secondRootDevice->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
subDevice1->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*rootDevice);
ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
@ -763,7 +794,8 @@ TEST_F(SvmDeviceAllocationCacheTest, givenDeviceOutOfMemoryWhenAllocatingThenCac
device->injectMemoryManager(new MockMemoryManagerWithCapacity(*device->getExecutionEnvironment()));
MockMemoryManagerWithCapacity *memoryManager = static_cast<MockMemoryManagerWithCapacity *>(device->getMemoryManager());
auto svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
memoryManager->usmReuseInfo.init(0u, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
@ -801,7 +833,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationWithIsInternalAllocationSetW
debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1);
auto device = deviceFactory->rootDevices[0];
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
@ -833,7 +865,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationInUsageWhenAllocatingAfterFr
debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1);
auto device = deviceFactory->rootDevices[0];
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
@ -869,7 +901,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenUsmReuseCleanerWhenTrimOldInCachesCall
device->executionEnvironment->unifiedMemoryReuseCleaner.reset(new MockUnifiedMemoryReuseCleaner);
auto mockUnifiedMemoryReuseCleaner = reinterpret_cast<MockUnifiedMemoryReuseCleaner *>(device->executionEnvironment->unifiedMemoryReuseCleaner.get());
EXPECT_EQ(0u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size());
device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
EXPECT_EQ(1u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size());
@ -910,6 +942,49 @@ TEST_F(SvmDeviceAllocationCacheTest, givenUsmReuseCleanerWhenTrimOldInCachesCall
EXPECT_EQ(0u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size());
}
TEST_F(SvmDeviceAllocationCacheTest, givenUsmReuseCleanerWhenTrimOldInCachesCalledAndShouldLimitUsmReuseThenAllOldAllocationsAreRemovedEvenIfDeferredDeleterHasWork) {
std::unique_ptr<UltDeviceFactory> deviceFactory(new UltDeviceFactory(1, 1));
RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex};
std::map<uint32_t, DeviceBitfield> deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}};
DebugManagerStateRestore restore;
debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1);
debugManager.flags.ExperimentalEnableHostAllocationCache.set(0);
auto device = deviceFactory->rootDevices[0];
auto memoryManager = reinterpret_cast<MockMemoryManager *>(device->getMemoryManager());
auto svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
device->executionEnvironment->unifiedMemoryReuseCleaner.reset(new MockUnifiedMemoryReuseCleaner);
auto mockUnifiedMemoryReuseCleaner = reinterpret_cast<MockUnifiedMemoryReuseCleaner *>(device->executionEnvironment->unifiedMemoryReuseCleaner.get());
EXPECT_EQ(0u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size());
device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
EXPECT_EQ(1u, mockUnifiedMemoryReuseCleaner->svmAllocationCaches.size());
EXPECT_EQ(svmManager->usmDeviceAllocationsCache.get(), mockUnifiedMemoryReuseCleaner->svmAllocationCaches[0]);
SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::deviceUnifiedMemory, 1, rootDeviceIndices, deviceBitfields);
unifiedMemoryProperties.device = device;
auto allocation = svmManager->createUnifiedMemoryAllocation(10u, unifiedMemoryProperties);
auto allocation2 = svmManager->createUnifiedMemoryAllocation(10u, unifiedMemoryProperties);
EXPECT_NE(allocation, nullptr);
EXPECT_NE(allocation2, nullptr);
svmManager->freeSVMAlloc(allocation);
svmManager->freeSVMAlloc(allocation2);
EXPECT_EQ(svmManager->usmDeviceAllocationsCache->allocations.size(), 2u);
const auto baseTimePoint = std::chrono::high_resolution_clock::now();
const auto oldTimePoint = baseTimePoint - UnifiedMemoryReuseCleaner::limitedHoldTime;
svmManager->usmDeviceAllocationsCache->allocations[0].saveTime = oldTimePoint;
svmManager->usmDeviceAllocationsCache->allocations[1].saveTime = oldTimePoint;
memoryManager->setDeferredDeleter(new MockDeferredDeleter);
memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, alwaysLimited);
mockUnifiedMemoryReuseCleaner->trimOldInCaches();
EXPECT_EQ(0u, svmManager->usmDeviceAllocationsCache->allocations.size());
svmManager->cleanupUSMAllocCaches();
}
TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCalledThenTrimAllocationsSavedBeforeTimePoint) {
std::unique_ptr<UltDeviceFactory> deviceFactory(new UltDeviceFactory(1, 1));
RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex};
@ -918,7 +993,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCal
debugManager.flags.ExperimentalEnableDeviceAllocationCache.set(1);
auto device = deviceFactory->rootDevices[0];
auto svmManager = std::make_unique<MockSVMAllocsManager>(device->getMemoryManager(), false);
device->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
device->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
EXPECT_NE(nullptr, svmManager->usmDeviceAllocationsCache);
@ -942,13 +1017,13 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCal
svmManager->usmDeviceAllocationsCache->allocations[1].saveTime = baseTimePoint + timeDiff;
svmManager->usmDeviceAllocationsCache->allocations[2].saveTime = baseTimePoint + timeDiff * 2;
svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff);
svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false);
EXPECT_EQ(2u, svmManager->usmDeviceAllocationsCache->allocations.size());
svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff);
svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false);
EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size());
svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff);
svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false);
EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(baseTimePoint + timeDiff * 2, svmManager->usmDeviceAllocationsCache->allocations[0].saveTime);
@ -987,7 +1062,7 @@ HWTEST_F(SvmHostAllocationCacheTest, givenOclApiSpecificConfigWhenCheckingIfEnab
auto device = deviceFactory->rootDevices[0];
RAIIProductHelperFactory<MockProductHelper> raii(*device->getExecutionEnvironment()->rootDeviceEnvironments[0]);
const auto expectedMaxSize = static_cast<size_t>(0.02 * device->getMemoryManager()->getSystemSharedMemory(0u));
EXPECT_EQ(expectedMaxSize, device->getMemoryManager()->getMaxAllocationsSavedForReuseSize());
EXPECT_EQ(expectedMaxSize, device->getMemoryManager()->usmReuseInfo.getMaxAllocationsSavedForReuseSize());
{
raii.mockProductHelper->isHostUsmAllocationReuseSupportedResult = false;
@ -1005,6 +1080,37 @@ HWTEST_F(SvmHostAllocationCacheTest, givenOclApiSpecificConfigWhenCheckingIfEnab
}
}
TEST_F(SvmHostAllocationCacheTest, givenReuseLimitFlagWhenInitUsmReuseLimitCalledThenLimitThresholdSetCorrectly) {
std::unique_ptr<UltDeviceFactory> deviceFactory(new UltDeviceFactory(1, 1));
RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex};
std::map<uint32_t, DeviceBitfield> deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}};
auto device = deviceFactory->rootDevices[0];
auto memoryManager = reinterpret_cast<MockMemoryManager *>(device->getMemoryManager());
{
DebugManagerStateRestore restore;
debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.set(0);
memoryManager->initUsmReuseLimits();
const auto expectedLimitThreshold = std::numeric_limits<uint64_t>::max();
EXPECT_EQ(expectedLimitThreshold, memoryManager->usmReuseInfo.getLimitAllocationsReuseThreshold());
}
{
DebugManagerStateRestore restore;
debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.set(80);
memoryManager->initUsmReuseLimits();
const auto systemSharedMemory = memoryManager->getSystemSharedMemory(device->getRootDeviceIndex());
const auto expectedLimitThreshold = static_cast<uint64_t>(0.8 * systemSharedMemory);
EXPECT_EQ(expectedLimitThreshold, memoryManager->usmReuseInfo.getLimitAllocationsReuseThreshold());
}
{
DebugManagerStateRestore restore;
debugManager.flags.ExperimentalUSMAllocationReuseLimitThreshold.set(-1);
memoryManager->initUsmReuseLimits();
const auto expectedLimitThreshold = std::numeric_limits<uint64_t>::max();
EXPECT_EQ(expectedLimitThreshold, memoryManager->usmReuseInfo.getLimitAllocationsReuseThreshold());
}
}
struct SvmHostAllocationCacheSimpleTestDataType {
size_t allocationSize;
void *allocation;
@ -1019,7 +1125,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingHostAll
auto device = deviceFactory->rootDevices[0];
auto memoryManager = reinterpret_cast<MockMemoryManager *>(device->getMemoryManager());
auto svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache);
@ -1068,7 +1174,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledWhenInitializedThe
ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache);
auto expectedMaxSize = static_cast<size_t>(svmManager->memoryManager->getSystemSharedMemory(mockRootDeviceIndex) * 0.02);
EXPECT_EQ(expectedMaxSize, device->getMemoryManager()->getMaxAllocationsSavedForReuseSize());
EXPECT_EQ(expectedMaxSize, device->getMemoryManager()->usmReuseInfo.getMaxAllocationsSavedForReuseSize());
}
TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingHostAllocationThenItIsPutIntoCacheOnlyIfMaxSizeWillNotBeExceeded) {
@ -1081,7 +1187,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingHostAll
auto memoryManager = reinterpret_cast<MockMemoryManager *>(device->getMemoryManager());
auto svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
constexpr auto allocationSize = MemoryConstants::pageSize64k;
memoryManager->maxAllocationsSavedForReuseSize = allocationSize;
memoryManager->usmReuseInfo.init(allocationSize, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache);
@ -1092,26 +1198,26 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingHostAll
auto allocation2 = svmManager->createHostUnifiedMemoryAllocation(1u, unifiedMemoryProperties);
ASSERT_NE(allocation2, nullptr);
EXPECT_EQ(0u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAlloc(allocation);
EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAlloc(allocation2);
EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
auto recycledAllocation = svmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
EXPECT_EQ(recycledAllocation, allocation);
EXPECT_EQ(svmManager->usmHostAllocationsCache->allocations.size(), 0u);
EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAlloc(recycledAllocation);
svmManager->trimUSMHostAllocCache();
EXPECT_EQ(svmManager->usmHostAllocationsCache->allocations.size(), 0u);
EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
}
{
auto allocation = svmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
@ -1119,29 +1225,62 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingHostAll
auto allocation2 = svmManager->createHostUnifiedMemoryAllocation(1u, unifiedMemoryProperties);
ASSERT_NE(allocation2, nullptr);
EXPECT_EQ(0u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAllocDefer(allocation);
EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAllocDefer(allocation2);
EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
auto recycledAllocation = svmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
EXPECT_EQ(recycledAllocation, allocation);
EXPECT_EQ(svmManager->usmHostAllocationsCache->allocations.size(), 0u);
EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAllocDefer(recycledAllocation);
svmManager->trimUSMHostAllocCache();
EXPECT_EQ(svmManager->usmHostAllocationsCache->allocations.size(), 0u);
EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
}
}
TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledWhenFreeingHostAllocationAndShouldLimitUsmReuseThenItIsNotPutIntoCache) {
std::unique_ptr<UltDeviceFactory> deviceFactory(new UltDeviceFactory(1, 1));
RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex};
std::map<uint32_t, DeviceBitfield> deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}};
DebugManagerStateRestore restore;
debugManager.flags.ExperimentalEnableHostAllocationCache.set(1);
auto device = deviceFactory->rootDevices[0];
auto memoryManager = reinterpret_cast<MockMemoryManager *>(device->getMemoryManager());
auto svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
constexpr auto allocationSize = MemoryConstants::pageSize64k;
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache);
SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::hostUnifiedMemory, 1, rootDeviceIndices, deviceBitfields);
auto allocation = svmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
ASSERT_NE(allocation, nullptr);
auto allocation2 = svmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
ASSERT_NE(allocation2, nullptr);
EXPECT_EQ(0u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAlloc(allocation);
EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, alwaysLimited);
svmManager->freeSVMAlloc(allocation2);
EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->cleanupUSMAllocCaches();
}
TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledAndMultipleSVMManagersWhenFreeingHostAllocationThenItIsPutIntoCacheOnlyIfMaxSizeWillNotBeExceeded) {
std::unique_ptr<UltDeviceFactory> deviceFactory(new UltDeviceFactory(1, 1));
RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex};
@ -1153,7 +1292,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledAndMultipleSVMMana
auto svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
auto secondSvmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
constexpr auto allocationSize = MemoryConstants::pageSize64k;
memoryManager->maxAllocationsSavedForReuseSize = allocationSize;
memoryManager->usmReuseInfo.init(allocationSize, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
secondSvmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache);
@ -1167,26 +1306,26 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledAndMultipleSVMMana
ASSERT_NE(allocation2, nullptr);
EXPECT_EQ(0u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(0u, secondSvmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAlloc(allocation);
EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
secondSvmManager->freeSVMAlloc(allocation2);
EXPECT_EQ(0u, secondSvmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
auto recycledAllocation = svmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
EXPECT_EQ(recycledAllocation, allocation);
EXPECT_EQ(svmManager->usmHostAllocationsCache->allocations.size(), 0u);
EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAlloc(recycledAllocation);
svmManager->trimUSMHostAllocCache();
EXPECT_EQ(svmManager->usmHostAllocationsCache->allocations.size(), 0u);
EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
}
{
auto allocation = svmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
@ -1195,26 +1334,26 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationCacheEnabledAndMultipleSVMMana
ASSERT_NE(allocation2, nullptr);
EXPECT_EQ(0u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(0u, secondSvmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
secondSvmManager->freeSVMAlloc(allocation2);
EXPECT_EQ(1u, secondSvmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
svmManager->freeSVMAlloc(allocation);
EXPECT_EQ(0u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(allocationSize, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(allocationSize, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
auto recycledAllocation = secondSvmManager->createHostUnifiedMemoryAllocation(allocationSize, unifiedMemoryProperties);
EXPECT_EQ(recycledAllocation, allocation2);
EXPECT_EQ(secondSvmManager->usmHostAllocationsCache->allocations.size(), 0u);
EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
secondSvmManager->freeSVMAlloc(recycledAllocation);
secondSvmManager->trimUSMHostAllocCache();
EXPECT_EQ(secondSvmManager->usmHostAllocationsCache->allocations.size(), 0u);
EXPECT_EQ(0u, memoryManager->getHostAllocationsSavedForReuseSize());
EXPECT_EQ(0u, memoryManager->usmReuseInfo.getAllocationsSavedForReuseSize());
}
}
@ -1227,7 +1366,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationsWithDifferentSizesWhenAllocat
auto device = deviceFactory->rootDevices[0];
auto memoryManager = reinterpret_cast<MockMemoryManager *>(device->getMemoryManager());
auto svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache);
@ -1279,7 +1418,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationsWithDifferentSizesWhenAllocat
auto device = deviceFactory->rootDevices[0];
auto memoryManager = reinterpret_cast<MockMemoryManager *>(device->getMemoryManager());
auto svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache);
@ -1319,7 +1458,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationOverSizeLimitWhenAllocatingAft
auto device = deviceFactory->rootDevices[0];
auto memoryManager = reinterpret_cast<MockMemoryManager *>(device->getMemoryManager());
auto svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache);
const auto notAcceptedAllocSize = SVMAllocsManager::SvmAllocationCache::maxServicedSize + 1;
@ -1348,7 +1487,7 @@ TEST_F(SvmHostAllocationCacheTest, givenMultipleAllocationsWhenAllocatingAfterFr
auto device = deviceFactory->rootDevices[0];
auto memoryManager = reinterpret_cast<MockMemoryManager *>(device->getMemoryManager());
auto svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache);
@ -1424,7 +1563,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationsWithDifferentFlagsWhenAllocat
auto rootDevice = deviceFactory->rootDevices[0];
auto memoryManager = reinterpret_cast<MockMemoryManager *>(rootDevice->getMemoryManager());
auto svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*rootDevice);
ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache);
@ -1491,7 +1630,7 @@ TEST_F(SvmHostAllocationCacheTest, givenHostOutOfMemoryWhenAllocatingThenCacheIs
device->injectMemoryManager(new MockMemoryManagerWithCapacity(*device->getExecutionEnvironment()));
MockMemoryManagerWithCapacity *memoryManager = static_cast<MockMemoryManagerWithCapacity *>(device->getMemoryManager());
auto svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
ASSERT_NE(nullptr, svmManager->usmHostAllocationsCache);
@ -1529,7 +1668,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationInUsageWhenAllocatingAfterFree
auto device = deviceFactory->rootDevices[0];
auto memoryManager = reinterpret_cast<MockMemoryManager *>(device->getMemoryManager());
auto svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
EXPECT_NE(nullptr, svmManager->usmHostAllocationsCache);
@ -1560,7 +1699,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCalle
auto device = deviceFactory->rootDevices[0];
auto memoryManager = reinterpret_cast<MockMemoryManager *>(device->getMemoryManager());
auto svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager, false);
memoryManager->maxAllocationsSavedForReuseSize = 1 * MemoryConstants::gigaByte;
memoryManager->usmReuseInfo.init(1 * MemoryConstants::gigaByte, UsmReuseInfo::notLimited);
svmManager->initUsmAllocationsCaches(*device);
EXPECT_NE(nullptr, svmManager->usmHostAllocationsCache);
@ -1583,13 +1722,13 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCalle
svmManager->usmHostAllocationsCache->allocations[1].saveTime = baseTimePoint + timeDiff;
svmManager->usmHostAllocationsCache->allocations[2].saveTime = baseTimePoint + timeDiff * 2;
svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff);
svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false);
EXPECT_EQ(2u, svmManager->usmHostAllocationsCache->allocations.size());
svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff);
svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false);
EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size());
svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff);
svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false);
EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(baseTimePoint + timeDiff * 2, svmManager->usmHostAllocationsCache->allocations[0].saveTime);