intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-12-24 12:23:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: usm reuse, clean from largest

Browse Source 
When trimming old allocations in usm reuse start from largest
allocations.
This will reduce memory usage more quickly once max hold time is hit.

Related-To: NEO-6893, NEO-14429

Signed-off-by: Dominik Dabek <dominik.dabek@intel.com>
This commit is contained in:
Dominik Dabek
2025-04-04 10:05:48 +00:00
committed by Compute-Runtime-Automation
parent cc308719d8
commit bd516b3552
3 changed files with 58 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
shared/source/memory_manager/unified_memory_manager.cpp
View File

@@ -235,39 +235,43 @@ void SVMAllocsManager::SvmAllocationCache::logCacheOperation(const SvmAllocation
isSuccessString); isSuccessString);
} }
void SVMAllocsManager::SvmAllocationCache::trimOldAllocs(std::chrono::high_resolution_clock::time_point trimTimePoint, bool shouldLimitReuse) { void SVMAllocsManager::SvmAllocationCache::trimOldAllocs(std::chrono::high_resolution_clock::time_point trimTimePoint, bool trimAll) {
if (this->allocations.empty()) { if (this->allocations.empty()) {
return; return;
} }
std::lock_guard<std::mutex> lock(this->mtx); std::lock_guard<std::mutex> lock(this->mtx);
for (auto allocCleanCandidate = allocations.begin(); allocCleanCandidate != allocations.end();) { auto allocCleanCandidateIndex = allocations.size();
if (allocCleanCandidate->saveTime > trimTimePoint) { while (0u != allocCleanCandidateIndex) {
++allocCleanCandidate; auto &allocCleanCandidate = allocations[--allocCleanCandidateIndex];
if (allocCleanCandidate.saveTime > trimTimePoint) {
continue; continue;
} }
DEBUG_BREAK_IF(nullptr == allocCleanCandidate->svmData); DEBUG_BREAK_IF(nullptr == allocCleanCandidate.svmData);
if (allocCleanCandidate->svmData->device) { if (allocCleanCandidate.svmData->device) {
auto lock = allocCleanCandidate->svmData->device->usmReuseInfo.obtainAllocationsReuseLock(); auto lock = allocCleanCandidate.svmData->device->usmReuseInfo.obtainAllocationsReuseLock();
allocCleanCandidate->svmData->device->usmReuseInfo.recordAllocationGetFromReuse(allocCleanCandidate->allocationSize); allocCleanCandidate.svmData->device->usmReuseInfo.recordAllocationGetFromReuse(allocCleanCandidate.allocationSize);
} else { } else {
auto lock = memoryManager->usmReuseInfo.obtainAllocationsReuseLock(); auto lock = memoryManager->usmReuseInfo.obtainAllocationsReuseLock();
memoryManager->usmReuseInfo.recordAllocationGetFromReuse(allocCleanCandidate->allocationSize); memoryManager->usmReuseInfo.recordAllocationGetFromReuse(allocCleanCandidate.allocationSize);
} }
if (enablePerformanceLogging) { if (enablePerformanceLogging) {
logCacheOperation({.allocationSize = allocCleanCandidate->allocationSize, logCacheOperation({.allocationSize = allocCleanCandidate.allocationSize,
.timePoint = std::chrono::high_resolution_clock::now(), .timePoint = std::chrono::high_resolution_clock::now(),
.allocationType = allocCleanCandidate->svmData->memoryType, .allocationType = allocCleanCandidate.svmData->memoryType,
.operationType = CacheOperationType::trimOld, .operationType = CacheOperationType::trimOld,
.isSuccess = true}); .isSuccess = true});
} }
svmAllocsManager->freeSVMAllocImpl(allocCleanCandidate->allocation, FreePolicyType::defer, allocCleanCandidate->svmData); svmAllocsManager->freeSVMAllocImpl(allocCleanCandidate.allocation, FreePolicyType::defer, allocCleanCandidate.svmData);
if (shouldLimitReuse) { if (trimAll) {
allocCleanCandidate = allocations.erase(allocCleanCandidate); allocCleanCandidate.markForDelete();
} else { } else {
allocations.erase(allocCleanCandidate); allocations.erase(allocations.begin() + allocCleanCandidateIndex);
break; break;
} }
} }
if (trimAll) {
std::erase_if(allocations, SvmCacheAllocationInfo::isMarkedForDelete);
}
} }
SvmAllocationData *SVMAllocsManager::MapBasedAllocationTracker::get(const void *ptr) { SvmAllocationData *SVMAllocsManager::MapBasedAllocationTracker::get(const void *ptr) {

8
shared/source/memory_manager/unified_memory_manager.h
View File

@@ -165,6 +165,12 @@ class SVMAllocsManager {
bool operator<(size_t const &size) const { bool operator<(size_t const &size) const {
return allocationSize < size; return allocationSize < size;
} }
void markForDelete() {
allocationSize = 0u;
}
static bool isMarkedForDelete(SvmCacheAllocationInfo const &info) {
return 0 == info.allocationSize;
}
}; };
struct SvmAllocationCache { struct SvmAllocationCache {
@@ -195,7 +201,7 @@ class SVMAllocsManager {
bool isInUse(SvmAllocationData *svmData); bool isInUse(SvmAllocationData *svmData);
void *get(size_t size, const UnifiedMemoryProperties &unifiedMemoryProperties); void *get(size_t size, const UnifiedMemoryProperties &unifiedMemoryProperties);
void trim(); void trim();
void trimOldAllocs(std::chrono::high_resolution_clock::time_point trimTimePoint, bool shouldLimitReuse); void trimOldAllocs(std::chrono::high_resolution_clock::time_point trimTimePoint, bool trimAll);
void cleanup(); void cleanup();
void logCacheOperation(const SvmAllocationCachePerfInfo &cachePerfEvent) const; void logCacheOperation(const SvmAllocationCachePerfInfo &cachePerfEvent) const;

44
shared/test/unit_test/memory_manager/unified_memory_manager_cache_tests.cpp
View File

@@ -108,6 +108,14 @@ TEST(SvmAllocationCacheSimpleTest, givenDifferentSizesWhenCheckingIfSizeAllowsTh
EXPECT_FALSE(SVMAllocsManager::SvmAllocationCache::sizeAllowed(256 * MemoryConstants::megaByte + 1)); EXPECT_FALSE(SVMAllocsManager::SvmAllocationCache::sizeAllowed(256 * MemoryConstants::megaByte + 1));
} }
TEST(SvmAllocationCacheSimpleTest, givenSvmAllocationCacheInfoWhenMarkedForDeleteThenSetSizeToZero) {
SVMAllocsManager::SvmCacheAllocationInfo info(MemoryConstants::pageSize64k, nullptr, nullptr);
EXPECT_FALSE(SVMAllocsManager::SvmCacheAllocationInfo::isMarkedForDelete(info));
info.markForDelete();
EXPECT_EQ(0u, info.allocationSize);
EXPECT_TRUE(SVMAllocsManager::SvmCacheAllocationInfo::isMarkedForDelete(info));
}
TEST(SvmAllocationCacheSimpleTest, givenAllocationsWhenCheckingIsInUseThenReturnCorrectValue) { TEST(SvmAllocationCacheSimpleTest, givenAllocationsWhenCheckingIsInUseThenReturnCorrectValue) {
SVMAllocsManager::SvmAllocationCache allocationCache; SVMAllocsManager::SvmAllocationCache allocationCache;
MockMemoryManager memoryManager; MockMemoryManager memoryManager;
@@ -990,7 +998,7 @@ TEST_F(SvmDeviceAllocationCacheTest, givenUsmReuseCleanerWhenTrimOldInCachesCall
svmManager->cleanupUSMAllocCaches(); svmManager->cleanupUSMAllocCaches();
} }
TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCalledThenTrimAllocationsSavedBeforeTimePoint) { TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCalledThenTrimAllocationsSavedBeforeTimePointLargestFirst) {
std::unique_ptr<UltDeviceFactory> deviceFactory(new UltDeviceFactory(1, 1)); std::unique_ptr<UltDeviceFactory> deviceFactory(new UltDeviceFactory(1, 1));
RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex}; RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex};
std::map<uint32_t, DeviceBitfield> deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}}; std::map<uint32_t, DeviceBitfield> deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}};
@@ -1004,9 +1012,9 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCal
SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::deviceUnifiedMemory, 1, rootDeviceIndices, deviceBitfields); SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::deviceUnifiedMemory, 1, rootDeviceIndices, deviceBitfields);
unifiedMemoryProperties.device = device; unifiedMemoryProperties.device = device;
auto allocation = svmManager->createUnifiedMemoryAllocation(10u, unifiedMemoryProperties); auto allocation = svmManager->createUnifiedMemoryAllocation(1 * MemoryConstants::pageSize64k, unifiedMemoryProperties);
auto allocation2 = svmManager->createUnifiedMemoryAllocation(10u, unifiedMemoryProperties); auto allocation2 = svmManager->createUnifiedMemoryAllocation(2 * MemoryConstants::pageSize64k, unifiedMemoryProperties);
auto allocation3 = svmManager->createUnifiedMemoryAllocation(10u, unifiedMemoryProperties); auto allocation3 = svmManager->createUnifiedMemoryAllocation(3 * MemoryConstants::pageSize64k, unifiedMemoryProperties);
EXPECT_NE(allocation, nullptr); EXPECT_NE(allocation, nullptr);
EXPECT_NE(allocation2, nullptr); EXPECT_NE(allocation2, nullptr);
EXPECT_NE(allocation3, nullptr); EXPECT_NE(allocation3, nullptr);
@@ -1014,19 +1022,25 @@ TEST_F(SvmDeviceAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCal
svmManager->freeSVMAlloc(allocation2); svmManager->freeSVMAlloc(allocation2);
svmManager->freeSVMAlloc(allocation3); svmManager->freeSVMAlloc(allocation3);
EXPECT_EQ(3u, svmManager->usmDeviceAllocationsCache->allocations.size()); EXPECT_EQ(3u, svmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(1 * MemoryConstants::pageSize64k, svmManager->usmDeviceAllocationsCache->allocations[0].allocationSize);
EXPECT_EQ(2 * MemoryConstants::pageSize64k, svmManager->usmDeviceAllocationsCache->allocations[1].allocationSize);
EXPECT_EQ(3 * MemoryConstants::pageSize64k, svmManager->usmDeviceAllocationsCache->allocations[2].allocationSize);
const auto baseTimePoint = std::chrono::high_resolution_clock::now(); const auto baseTimePoint = std::chrono::high_resolution_clock::now();
const auto timeDiff = std::chrono::microseconds(1); const auto timeDiff = std::chrono::microseconds(1);
svmManager->usmDeviceAllocationsCache->allocations[0].saveTime = baseTimePoint; svmManager->usmDeviceAllocationsCache->allocations[0].saveTime = baseTimePoint;
svmManager->usmDeviceAllocationsCache->allocations[1].saveTime = baseTimePoint + timeDiff; svmManager->usmDeviceAllocationsCache->allocations[1].saveTime = baseTimePoint + timeDiff * 2;
svmManager->usmDeviceAllocationsCache->allocations[2].saveTime = baseTimePoint + timeDiff * 2; svmManager->usmDeviceAllocationsCache->allocations[2].saveTime = baseTimePoint + timeDiff;
svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false); svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false);
EXPECT_EQ(2u, svmManager->usmDeviceAllocationsCache->allocations.size()); EXPECT_EQ(2u, svmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(1 * MemoryConstants::pageSize64k, svmManager->usmDeviceAllocationsCache->allocations[0].allocationSize);
EXPECT_EQ(2 * MemoryConstants::pageSize64k, svmManager->usmDeviceAllocationsCache->allocations[1].allocationSize);
svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false); svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false);
EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size()); EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size());
EXPECT_EQ(2 * MemoryConstants::pageSize64k, svmManager->usmDeviceAllocationsCache->allocations[0].allocationSize);
svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false); svmManager->usmDeviceAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false);
EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size()); EXPECT_EQ(1u, svmManager->usmDeviceAllocationsCache->allocations.size());
@@ -1700,7 +1714,7 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationInUsageWhenAllocatingAfterFree
svmManager->cleanupUSMAllocCaches(); svmManager->cleanupUSMAllocCaches();
} }
TEST_F(SvmHostAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCalledThenTrimAllocationsSavedBeforeTimePoint) { TEST_F(SvmHostAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCalledThenTrimAllocationsSavedBeforeTimePointLargestFirst) {
std::unique_ptr<UltDeviceFactory> deviceFactory(new UltDeviceFactory(1, 1)); std::unique_ptr<UltDeviceFactory> deviceFactory(new UltDeviceFactory(1, 1));
RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex}; RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex};
std::map<uint32_t, DeviceBitfield> deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}}; std::map<uint32_t, DeviceBitfield> deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}};
@@ -1714,9 +1728,9 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCalle
EXPECT_NE(nullptr, svmManager->usmHostAllocationsCache); EXPECT_NE(nullptr, svmManager->usmHostAllocationsCache);
SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::hostUnifiedMemory, 1, rootDeviceIndices, deviceBitfields); SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::hostUnifiedMemory, 1, rootDeviceIndices, deviceBitfields);
auto allocation = svmManager->createUnifiedMemoryAllocation(10u, unifiedMemoryProperties); auto allocation = svmManager->createUnifiedMemoryAllocation(1 * MemoryConstants::pageSize64k, unifiedMemoryProperties);
auto allocation2 = svmManager->createUnifiedMemoryAllocation(10u, unifiedMemoryProperties); auto allocation2 = svmManager->createUnifiedMemoryAllocation(2 * MemoryConstants::pageSize64k, unifiedMemoryProperties);
auto allocation3 = svmManager->createUnifiedMemoryAllocation(10u, unifiedMemoryProperties); auto allocation3 = svmManager->createUnifiedMemoryAllocation(3 * MemoryConstants::pageSize64k, unifiedMemoryProperties);
EXPECT_NE(allocation, nullptr); EXPECT_NE(allocation, nullptr);
EXPECT_NE(allocation2, nullptr); EXPECT_NE(allocation2, nullptr);
EXPECT_NE(allocation3, nullptr); EXPECT_NE(allocation3, nullptr);
@@ -1724,19 +1738,25 @@ TEST_F(SvmHostAllocationCacheTest, givenAllocationsInReuseWhenTrimOldAllocsCalle
svmManager->freeSVMAlloc(allocation2); svmManager->freeSVMAlloc(allocation2);
svmManager->freeSVMAlloc(allocation3); svmManager->freeSVMAlloc(allocation3);
EXPECT_EQ(3u, svmManager->usmHostAllocationsCache->allocations.size()); EXPECT_EQ(3u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(1 * MemoryConstants::pageSize64k, svmManager->usmHostAllocationsCache->allocations[0].allocationSize);
EXPECT_EQ(2 * MemoryConstants::pageSize64k, svmManager->usmHostAllocationsCache->allocations[1].allocationSize);
EXPECT_EQ(3 * MemoryConstants::pageSize64k, svmManager->usmHostAllocationsCache->allocations[2].allocationSize);
auto baseTimePoint = std::chrono::high_resolution_clock::now(); auto baseTimePoint = std::chrono::high_resolution_clock::now();
auto timeDiff = std::chrono::microseconds(1); auto timeDiff = std::chrono::microseconds(1);
svmManager->usmHostAllocationsCache->allocations[0].saveTime = baseTimePoint; svmManager->usmHostAllocationsCache->allocations[0].saveTime = baseTimePoint;
svmManager->usmHostAllocationsCache->allocations[1].saveTime = baseTimePoint + timeDiff; svmManager->usmHostAllocationsCache->allocations[1].saveTime = baseTimePoint + timeDiff * 2;
svmManager->usmHostAllocationsCache->allocations[2].saveTime = baseTimePoint + timeDiff * 2; svmManager->usmHostAllocationsCache->allocations[2].saveTime = baseTimePoint + timeDiff;
svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false); svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false);
EXPECT_EQ(2u, svmManager->usmHostAllocationsCache->allocations.size()); EXPECT_EQ(2u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(1 * MemoryConstants::pageSize64k, svmManager->usmHostAllocationsCache->allocations[0].allocationSize);
EXPECT_EQ(2 * MemoryConstants::pageSize64k, svmManager->usmHostAllocationsCache->allocations[1].allocationSize);
svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false); svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false);
EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size()); EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size());
EXPECT_EQ(2 * MemoryConstants::pageSize64k, svmManager->usmHostAllocationsCache->allocations[0].allocationSize);
svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false); svmManager->usmHostAllocationsCache->trimOldAllocs(baseTimePoint + timeDiff, false);
EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size()); EXPECT_EQ(1u, svmManager->usmHostAllocationsCache->allocations.size());