From f54e3fda41812a5abc25ed658d3b7a28c4a49a19 Mon Sep 17 00:00:00 2001 From: Dominik Dabek Date: Wed, 24 Sep 2025 09:30:47 +0000 Subject: [PATCH] feature: adapt usm pool manager Change behavior to continue allocating usm pools as needed. Intended to replace singular usm pools. Related-To: NEO-16084 Signed-off-by: Dominik Dabek --- .../core/source/context/context_imp.cpp | 14 +- .../core/source/driver/driver_handle_imp.cpp | 35 +-- .../sources/memory/test_memory_pooling.cpp | 160 ++---------- .../test/unit_test/context/context_tests.cpp | 6 +- .../mem_obj/usm_memory_pool_tests.cpp | 14 +- .../debug_settings/debug_variables_base.inl | 2 +- shared/source/device/device.cpp | 23 +- shared/source/device/device.h | 1 + .../memory_manager/unified_memory_pooling.cpp | 171 ++++++------- .../memory_manager/unified_memory_pooling.h | 88 +++---- .../test/common/mocks/mock_usm_memory_pool.h | 16 +- shared/test/common/test_files/igdrcl.config | 2 +- .../unit_test/device/neo_device_tests.cpp | 79 +----- .../unified_memory_pooling_tests.cpp | 231 ++++++------------ 14 files changed, 271 insertions(+), 571 deletions(-) diff --git a/level_zero/core/source/context/context_imp.cpp b/level_zero/core/source/context/context_imp.cpp index 94812c977c..9190d40c62 100644 --- a/level_zero/core/source/context/context_imp.cpp +++ b/level_zero/core/source/context/context_imp.cpp @@ -342,9 +342,6 @@ ze_result_t ContextImp::allocDeviceMem(ze_device_handle_t hDevice, void *usmPtr = this->driverHandle->svmAllocsManager->createUnifiedMemoryAllocation(size, unifiedMemoryProperties); if (usmPtr == nullptr) { - if (neoDevice->getUsmMemAllocPoolsManager()) { - neoDevice->getUsmMemAllocPoolsManager()->trim(); - } if (driverHandle->svmAllocsManager->getNumDeferFreeAllocs() > 0) { this->driverHandle->svmAllocsManager->freeSVMAllocDeferImpl(); usmPtr = @@ -515,18 +512,13 @@ NEO::UsmMemAllocPool *ContextImp::getUsmPoolOwningPtr(const void *ptr, NEO::SvmA } bool ContextImp::tryFreeViaPooling(const void *ptr, NEO::SvmAllocationData *svmData, NEO::UsmMemAllocPool *usmPool) { + if (svmData->device && svmData->device->getUsmMemAllocPoolsManager()) { + return svmData->device->getUsmMemAllocPoolsManager()->freeSVMAlloc(ptr, false); + } if (usmPool) { [[maybe_unused]] auto status = usmPool->freeSVMAlloc(ptr, false); DEBUG_BREAK_IF(false == status); return true; - } else if (InternalMemoryType::deviceUnifiedMemory == svmData->memoryType) { - if (auto deviceUsmPoolsManager = svmData->device->getUsmMemAllocPoolsManager()) { - DEBUG_BREAK_IF(false == deviceUsmPoolsManager->isInitialized()); - if (deviceUsmPoolsManager->recycleSVMAlloc(const_cast(ptr), - false)) { - return true; - } - } } return false; } diff --git a/level_zero/core/source/driver/driver_handle_imp.cpp b/level_zero/core/source/driver/driver_handle_imp.cpp index 587bf453d1..564b7c20aa 100644 --- a/level_zero/core/source/driver/driver_handle_imp.cpp +++ b/level_zero/core/source/driver/driver_handle_imp.cpp @@ -336,9 +336,6 @@ ze_result_t DriverHandleImp::initialize(std::vector this->initHostUsmAllocPool(); for (auto &device : this->devices) { this->initDeviceUsmAllocPool(*device->getNEODevice(), this->numDevices > 1); - if (auto deviceUsmAllocPoolsManager = device->getNEODevice()->getUsmMemAllocPoolsManager()) { - deviceUsmAllocPoolsManager->ensureInitialized(this->svmAllocsManager); - } } uuidTimestamp = static_cast(std::chrono::system_clock::now().time_since_epoch().count()); @@ -403,6 +400,20 @@ void DriverHandleImp::initHostUsmAllocPool() { } void DriverHandleImp::initDeviceUsmAllocPool(NEO::Device &device, bool multiDevice) { + bool useUsmPoolManager = false; + if (NEO::debugManager.flags.EnableUsmAllocationPoolManager.get() != -1) { + useUsmPoolManager = !!NEO::debugManager.flags.EnableUsmAllocationPoolManager.get(); + } + auto &hwInfo = device.getHardwareInfo(); + auto &l0GfxCoreHelper = device.getRootDeviceEnvironment().getHelper(); + const bool compressionEnabledByDefault = l0GfxCoreHelper.usmCompressionSupported(hwInfo) && l0GfxCoreHelper.forceDefaultUsmCompressionSupport(); + NEO::SVMAllocsManager::UnifiedMemoryProperties poolMemoryProperties(InternalMemoryType::deviceUnifiedMemory, + MemoryConstants::pageSize2M, + rootDeviceIndices, + deviceBitfields); + poolMemoryProperties.device = &device; + poolMemoryProperties.allocationFlags.flags.compressedHint = compressionEnabledByDefault; + bool enabled = NEO::ApiSpecificConfig::isDeviceUsmPoolingEnabled() && device.getProductHelper().isDeviceUsmPoolAllocatorSupported() && nullptr == device.getL0Debugger() && @@ -415,17 +426,13 @@ void DriverHandleImp::initDeviceUsmAllocPool(NEO::Device &device, bool multiDevi } if (enabled) { - device.resetUsmAllocationPool(new NEO::UsmMemAllocPool); - auto &hwInfo = device.getHardwareInfo(); - auto &l0GfxCoreHelper = device.getRootDeviceEnvironment().getHelper(); - const bool compressionEnabledByDefault = l0GfxCoreHelper.usmCompressionSupported(hwInfo) && l0GfxCoreHelper.forceDefaultUsmCompressionSupport(); - NEO::SVMAllocsManager::UnifiedMemoryProperties poolMemoryProperties(InternalMemoryType::deviceUnifiedMemory, - MemoryConstants::pageSize2M, - rootDeviceIndices, - deviceBitfields); - poolMemoryProperties.device = &device; - poolMemoryProperties.allocationFlags.flags.compressedHint = compressionEnabledByDefault; - device.getUsmMemAllocPool()->initialize(this->svmAllocsManager, poolMemoryProperties, poolParams.poolSize, poolParams.minServicedSize, poolParams.maxServicedSize); + if (useUsmPoolManager) { + device.resetUsmAllocationPoolManager(new NEO::UsmMemAllocPoolsManager(InternalMemoryType::deviceUnifiedMemory, rootDeviceIndices, deviceBitfields, &device)); + device.getUsmMemAllocPoolsManager()->initialize(this->svmAllocsManager); + } else { + device.resetUsmAllocationPool(new NEO::UsmMemAllocPool); + device.getUsmMemAllocPool()->initialize(this->svmAllocsManager, poolMemoryProperties, poolParams.poolSize, poolParams.minServicedSize, poolParams.maxServicedSize); + } } } diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp index 05e7f98c7c..67d4ae944f 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp @@ -24,12 +24,12 @@ #include "level_zero/core/test/unit_tests/mocks/mock_driver_handle.h" namespace L0 { namespace ult { -template +template struct AllocUsmPoolMemoryTest : public ::testing::Test { void SetUp() override { NEO::debugManager.flags.EnableHostUsmAllocationPool.set(hostUsmPoolFlag); NEO::debugManager.flags.EnableDeviceUsmAllocationPool.set(deviceUsmPoolFlag); - NEO::debugManager.flags.ExperimentalUSMAllocationReuseVersion.set(poolingVersionFlag); + NEO::debugManager.flags.EnableUsmAllocationPoolManager.set(usmPoolManagerFlag); executionEnvironment = new NEO::ExecutionEnvironment(); executionEnvironment->prepareRootDeviceEnvironments(numRootDevices); @@ -663,7 +663,7 @@ TEST_F(AllocUsmDeviceEnabledMemoryTest, givenMultiplePooledAllocationsWhenOpenin EXPECT_EQ(0u, ipcHandleMap.size()); } -using AllocUsmDeviceEnabledMemoryNewVersionTest = AllocUsmPoolMemoryTest<-1, 1, 2>; +using AllocUsmDeviceEnabledMemoryNewVersionTest = AllocUsmPoolMemoryTest<-1, 1, 1>; TEST_F(AllocUsmDeviceEnabledMemoryNewVersionTest, givenContextWhenAllocatingAndFreeingDeviceUsmThenPoolingIsUsed) { executionEnvironment->rootDeviceEnvironments[0]->osInterface.reset(new NEO::OSInterface()); @@ -697,138 +697,30 @@ TEST_F(AllocUsmDeviceEnabledMemoryNewVersionTest, givenContextWhenAllocatingAndF result = context->freeMem(allocation); EXPECT_EQ(ZE_RESULT_SUCCESS, result); - void *allocation2MB = nullptr; - result = context->allocDeviceMem(deviceHandle, &deviceAllocDesc, 2 * MemoryConstants::megaByte, 0u, &allocation2MB); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_NE(nullptr, allocation2MB); - EXPECT_EQ(allocation2MB, usmMemAllocPoolsManager->getPooledAllocationBasePtr(allocation2MB)); - EXPECT_EQ(2 * MemoryConstants::megaByte, usmMemAllocPoolsManager->getPooledAllocationSize(allocation2MB)); + { + void *allocation2MB = nullptr; + result = context->allocDeviceMem(deviceHandle, &deviceAllocDesc, 2 * MemoryConstants::megaByte, 0u, &allocation2MB); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, allocation2MB); + EXPECT_EQ(allocation2MB, usmMemAllocPoolsManager->getPooledAllocationBasePtr(allocation2MB)); + auto pool = usmMemAllocPoolsManager->getPoolContainingAlloc(allocation2MB); + EXPECT_NE(nullptr, pool); + const auto poolInfo = pool->getPoolInfo(); + EXPECT_GE(2 * MemoryConstants::megaByte, poolInfo.minServicedSize); + EXPECT_LE(2 * MemoryConstants::megaByte, poolInfo.maxServicedSize); + EXPECT_EQ(2 * MemoryConstants::megaByte, usmMemAllocPoolsManager->getPooledAllocationSize(allocation2MB)); + } - void *allocation4MB = nullptr; - result = context->allocDeviceMem(deviceHandle, &deviceAllocDesc, 4 * MemoryConstants::megaByte, 0u, &allocation4MB); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_NE(nullptr, allocation4MB); - EXPECT_EQ(nullptr, usmMemAllocPoolsManager->getPooledAllocationBasePtr(allocation4MB)); - EXPECT_EQ(0u, usmMemAllocPoolsManager->getPooledAllocationSize(allocation4MB)); - - result = context->freeMem(allocation4MB); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - void *allocation4MBRecycled = nullptr; - result = context->allocDeviceMem(deviceHandle, &deviceAllocDesc, 4 * MemoryConstants::megaByte, 0u, &allocation4MBRecycled); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_NE(nullptr, allocation4MBRecycled); - EXPECT_EQ(allocation4MBRecycled, usmMemAllocPoolsManager->getPooledAllocationBasePtr(allocation4MBRecycled)); - EXPECT_EQ(4 * MemoryConstants::megaByte, usmMemAllocPoolsManager->getPooledAllocationSize(allocation4MBRecycled)); - EXPECT_EQ(allocation4MBRecycled, allocation4MB); - - result = context->freeMem(allocation4MBRecycled); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - void *allocation3MBRecycled = nullptr; - result = context->allocDeviceMem(deviceHandle, &deviceAllocDesc, 3 * MemoryConstants::megaByte, 0u, &allocation3MBRecycled); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_NE(nullptr, allocation3MBRecycled); - EXPECT_EQ(allocation3MBRecycled, usmMemAllocPoolsManager->getPooledAllocationBasePtr(allocation3MBRecycled)); - EXPECT_EQ(3 * MemoryConstants::megaByte, usmMemAllocPoolsManager->getPooledAllocationSize(allocation3MBRecycled)); - auto address4MB = castToUint64(allocation4MB); - auto address3MB = castToUint64(allocation3MBRecycled); - EXPECT_GE(address3MB, address4MB); - EXPECT_LT(address3MB, address4MB + MemoryConstants::megaByte); - - void *allocation2MB1B = nullptr; - result = context->allocDeviceMem(deviceHandle, &deviceAllocDesc, 2 * MemoryConstants::megaByte + 1, 0u, &allocation2MB1B); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_NE(nullptr, allocation2MB1B); - this->setMaxMemoryUsed(*driverHandle->devices[0]->getNEODevice()); - result = context->freeMem(allocation2MB1B); // should not be recycled, because too much device memory is used - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - void *allocation2MB1BNotRecycled = nullptr; - result = context->allocDeviceMem(deviceHandle, &deviceAllocDesc, 2 * MemoryConstants::megaByte + 1, 0u, &allocation2MB1BNotRecycled); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_NE(nullptr, allocation2MB1BNotRecycled); - EXPECT_EQ(nullptr, usmMemAllocPoolsManager->getPooledAllocationBasePtr(allocation2MB1BNotRecycled)); - EXPECT_EQ(0u, usmMemAllocPoolsManager->getPooledAllocationSize(allocation2MB1BNotRecycled)); - - result = context->getIpcMemHandle(allocation2MB1BNotRecycled, &ipcHandle); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - ipcData = *reinterpret_cast(ipcHandle.data); - pooledAllocationOffset = usmMemAllocPoolsManager->getOffsetInPool(allocation2MB1BNotRecycled); - EXPECT_EQ(0u, pooledAllocationOffset); - EXPECT_EQ(0u, ipcData.poolOffset); - - ipcPointer = nullptr; - result = context->openIpcMemHandle(driverHandle->devices[0]->toHandle(), ipcHandle, ipcFlags, &ipcPointer); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_EQ(addrToPtr(0x1u), ipcPointer); - result = context->closeIpcMemHandle(addrToPtr(0x1u)); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - - result = context->freeMem(allocation2MB1BNotRecycled); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); + { + auto maxPooledSize = UsmMemAllocPoolsManager::poolInfos.back().maxServicedSize; + void *allocationOverLimit = nullptr; + result = context->allocDeviceMem(deviceHandle, &deviceAllocDesc, maxPooledSize + 1, 0u, &allocationOverLimit); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, allocationOverLimit); + auto pool = usmMemAllocPoolsManager->getPoolContainingAlloc(allocationOverLimit); + EXPECT_EQ(nullptr, pool); + context->freeMem(allocationOverLimit); + } } - -TEST_F(AllocUsmDeviceEnabledMemoryNewVersionTest, givenContextWhenCallingFreeMemExtThenPoolingIsUsedCorrectly) { - auto usmMemAllocPoolsManager = driverHandle->devices[0]->getNEODevice()->getUsmMemAllocPoolsManager(); - ASSERT_NE(nullptr, usmMemAllocPoolsManager); - auto mockUsmMemAllocPoolsManager = reinterpret_cast(usmMemAllocPoolsManager); - auto deviceHandle = driverHandle->devices[0]->toHandle(); - ze_device_mem_alloc_desc_t deviceAllocDesc{}; - - const size_t allocationSize = 4 * MemoryConstants::megaByte; - void *allocation = nullptr; - ze_result_t result = context->allocDeviceMem(deviceHandle, &deviceAllocDesc, allocationSize, 0u, &allocation); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_NE(nullptr, allocation); - - ze_memory_free_ext_desc_t memFreeDesc = {}; - memFreeDesc.freePolicy = ZE_DRIVER_MEMORY_FREE_POLICY_EXT_FLAG_DEFER_FREE; - result = context->freeMemExt(&memFreeDesc, allocation); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - const auto &poolInfo = usmMemAllocPoolsManager->poolInfos[UsmMemAllocPoolsManager::firstNonPreallocatedIndex]; - auto &poolsList = mockUsmMemAllocPoolsManager->pools[poolInfo]; - ASSERT_EQ(1u, poolsList.size()); - auto usmPool = reinterpret_cast(poolsList[0].get()); - EXPECT_EQ(allocation, usmPool->pool); - EXPECT_EQ(allocationSize, usmPool->poolSize); - - usmMemAllocPoolsManager->trim(); - EXPECT_EQ(0u, poolsList.size()); - - result = context->allocDeviceMem(deviceHandle, &deviceAllocDesc, allocationSize, 0u, &allocation); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_NE(nullptr, allocation); - - this->setMaxMemoryUsed(*driverHandle->devices[0]->getNEODevice()); - result = context->freeMemExt(&memFreeDesc, allocation); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_EQ(0u, poolsList.size()); // not recycled -} - -TEST_F(AllocUsmDeviceEnabledMemoryNewVersionTest, givenContextWhenNormalAllocFailsThenPoolsAreTrimmed) { - auto usmMemAllocPoolsManager = driverHandle->devices[0]->getNEODevice()->getUsmMemAllocPoolsManager(); - ASSERT_NE(nullptr, usmMemAllocPoolsManager); - auto mockUsmMemAllocPoolsManager = reinterpret_cast(usmMemAllocPoolsManager); - auto deviceHandle = driverHandle->devices[0]->toHandle(); - ze_device_mem_alloc_desc_t deviceAllocDesc{}; - void *allocation = nullptr; - const auto startingPoolSize = 20 * MemoryConstants::megaByte; - ze_result_t result = context->allocDeviceMem(deviceHandle, &deviceAllocDesc, 2 * MemoryConstants::megaByte + 1, 0u, &allocation); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_NE(nullptr, allocation); - EXPECT_TRUE(usmMemAllocPoolsManager->isInitialized()); - result = context->freeMem(allocation); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - - EXPECT_EQ(2 * MemoryConstants::megaByte + 1 + startingPoolSize, mockUsmMemAllocPoolsManager->totalSize); - - auto mockMemoryManager = reinterpret_cast(driverHandle->getMemoryManager()); - mockMemoryManager->failInDevicePoolWithError = true; - - void *failAllocation = nullptr; - result = context->allocDeviceMem(deviceHandle, &deviceAllocDesc, 2 * MemoryConstants::megaByte + 1, 0u, &failAllocation); - EXPECT_EQ(nullptr, failAllocation); - EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY, result); - EXPECT_EQ(startingPoolSize, mockUsmMemAllocPoolsManager->totalSize); -} - } // namespace ult } // namespace L0 diff --git a/opencl/test/unit_test/context/context_tests.cpp b/opencl/test/unit_test/context/context_tests.cpp index f61973475c..59672f5179 100644 --- a/opencl/test/unit_test/context/context_tests.cpp +++ b/opencl/test/unit_test/context/context_tests.cpp @@ -830,9 +830,9 @@ TEST_F(ContextUsmPoolParamsTest, GivenUsmPoolAllocatorSupportedWhenInitializingU { auto mockHostUsmMemAllocPool = static_cast(&platform->getHostMemAllocPool()); const UsmPoolParams givenUsmHostPoolParams{ - .poolSize = mockHostUsmMemAllocPool->poolSize, - .minServicedSize = mockHostUsmMemAllocPool->minServicedSize, - .maxServicedSize = mockHostUsmMemAllocPool->maxServicedSize}; + .poolSize = mockHostUsmMemAllocPool->poolInfo.poolSize, + .minServicedSize = mockHostUsmMemAllocPool->poolInfo.minServicedSize, + .maxServicedSize = mockHostUsmMemAllocPool->poolInfo.maxServicedSize}; const UsmPoolParams expectedUsmHostPoolParams = UsmPoolParams::getUsmPoolParams(context->getDevice(0)->getGfxCoreHelper()); EXPECT_TRUE(compareUsmPoolParams(expectedUsmHostPoolParams, givenUsmHostPoolParams)); diff --git a/opencl/test/unit_test/mem_obj/usm_memory_pool_tests.cpp b/opencl/test/unit_test/mem_obj/usm_memory_pool_tests.cpp index 9822c69653..53d6e895ab 100644 --- a/opencl/test/unit_test/mem_obj/usm_memory_pool_tests.cpp +++ b/opencl/test/unit_test/mem_obj/usm_memory_pool_tests.cpp @@ -45,11 +45,11 @@ struct UsmPoolTest : public ::testing::Test { TEST_F(UsmPoolTest, givenCreatedContextWhenCheckingUsmPoolsThenPoolsAreNotInitialized) { EXPECT_FALSE(mockDeviceUsmMemAllocPool->isInitialized()); - EXPECT_EQ(0u, mockDeviceUsmMemAllocPool->poolSize); + EXPECT_EQ(0u, mockDeviceUsmMemAllocPool->poolInfo.poolSize); EXPECT_EQ(nullptr, mockDeviceUsmMemAllocPool->pool); EXPECT_FALSE(mockHostUsmMemAllocPool->isInitialized()); - EXPECT_EQ(0u, mockHostUsmMemAllocPool->poolSize); + EXPECT_EQ(0u, mockHostUsmMemAllocPool->poolInfo.poolSize); EXPECT_EQ(nullptr, mockHostUsmMemAllocPool->pool); } @@ -75,12 +75,12 @@ TEST_F(UsmPoolTest, givenEnabledDebugFlagsAndUsmPoolsNotSupportedWhenCreatingAll clMemFreeINTEL(mockContext.get(), pooledHostAlloc); EXPECT_TRUE(mockDeviceUsmMemAllocPool->isInitialized()); - EXPECT_EQ(1 * MemoryConstants::megaByte, mockDeviceUsmMemAllocPool->poolSize); + EXPECT_EQ(1 * MemoryConstants::megaByte, mockDeviceUsmMemAllocPool->poolInfo.poolSize); EXPECT_NE(nullptr, mockDeviceUsmMemAllocPool->pool); EXPECT_EQ(InternalMemoryType::deviceUnifiedMemory, mockDeviceUsmMemAllocPool->poolMemoryType); EXPECT_TRUE(mockHostUsmMemAllocPool->isInitialized()); - EXPECT_EQ(3 * MemoryConstants::megaByte, mockHostUsmMemAllocPool->poolSize); + EXPECT_EQ(3 * MemoryConstants::megaByte, mockHostUsmMemAllocPool->poolInfo.poolSize); EXPECT_NE(nullptr, mockHostUsmMemAllocPool->pool); EXPECT_EQ(InternalMemoryType::hostUnifiedMemory, mockHostUsmMemAllocPool->poolMemoryType); } @@ -103,8 +103,8 @@ TEST_F(UsmPoolTest, givenUsmPoolsSupportedWhenCreatingAllocationsThenPoolsAreIni EXPECT_NE(nullptr, pooledHostAlloc); clMemFreeINTEL(mockContext.get(), pooledHostAlloc); - EXPECT_EQ(UsmPoolParams::getUsmPoolSize(deviceFactory->rootDevices[0]->getGfxCoreHelper()), mockDeviceUsmMemAllocPool->poolSize); - EXPECT_EQ(UsmPoolParams::getUsmPoolSize(deviceFactory->rootDevices[0]->getGfxCoreHelper()), mockHostUsmMemAllocPool->poolSize); + EXPECT_EQ(UsmPoolParams::getUsmPoolSize(deviceFactory->rootDevices[0]->getGfxCoreHelper()), mockDeviceUsmMemAllocPool->poolInfo.poolSize); + EXPECT_EQ(UsmPoolParams::getUsmPoolSize(deviceFactory->rootDevices[0]->getGfxCoreHelper()), mockHostUsmMemAllocPool->poolInfo.poolSize); EXPECT_TRUE(mockDeviceUsmMemAllocPool->isInitialized()); EXPECT_TRUE(mockHostUsmMemAllocPool->isInitialized()); } @@ -167,7 +167,7 @@ TEST_F(UsmPoolTest, givenTwoContextsWhenHostAllocationIsFreedInFirstContextThenI EXPECT_NE(nullptr, pooledHostAlloc1); EXPECT_TRUE(mockHostUsmMemAllocPool->isInitialized()); - EXPECT_EQ(3 * MemoryConstants::megaByte, mockHostUsmMemAllocPool->poolSize); + EXPECT_EQ(3 * MemoryConstants::megaByte, mockHostUsmMemAllocPool->poolInfo.poolSize); clMemFreeINTEL(mockContext.get(), pooledHostAlloc1); diff --git a/shared/source/debug_settings/debug_variables_base.inl b/shared/source/debug_settings/debug_variables_base.inl index 36be112dd3..1fac13829e 100644 --- a/shared/source/debug_settings/debug_variables_base.inl +++ b/shared/source/debug_settings/debug_variables_base.inl @@ -433,6 +433,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, SkipDcFlushOnBarrierWithoutEvents, -1, "-1: defa DECLARE_DEBUG_VARIABLE(int32_t, EnableL3FlushAfterPostSync, -1, "-1: default, 0: disabled, 1: enabled. If enabled flush L3 after post sync operation") DECLARE_DEBUG_VARIABLE(int32_t, EnableDeviceUsmAllocationPool, -1, "-1: default (enabled, 2MB), 0: disabled, >=1: enabled, size in MB") DECLARE_DEBUG_VARIABLE(int32_t, EnableHostUsmAllocationPool, -1, "-1: default (enabled, 2MB), 0: disabled, >=1: enabled, size in MB") +DECLARE_DEBUG_VARIABLE(int32_t, EnableUsmAllocationPoolManager, -1, "-1: default, 0: disabled, 1: enabled, use growing pools") DECLARE_DEBUG_VARIABLE(int32_t, UseLocalPreferredForCacheableBuffers, -1, "Use localPreferred for cacheable buffers") DECLARE_DEBUG_VARIABLE(int32_t, EnableCopyWithStagingBuffers, -1, "Enable copy with non-usm memory through staging buffers. -1: default, 0: disabled, 1: enabled") DECLARE_DEBUG_VARIABLE(int32_t, StagingBufferSize, -1, "Size of single staging buffer. -1: default (2MB), >0: size in KB") @@ -587,7 +588,6 @@ DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalSetWalkerPartitionType, -1, "Experim DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalEnableCustomLocalMemoryAlignment, 0, "Align local memory allocations to a given value. Works only with allocations at least as big as the value. 0: no effect, 2097152: 2 megabytes, 1073741824: 1 gigabyte") DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalEnableDeviceAllocationCache, -1, "Experimentally enable device usm allocation cache. Use X% of device memory.") DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalEnableHostAllocationCache, -1, "Experimentally enable host usm allocation cache. Use X% of shared system memory.") -DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalUSMAllocationReuseVersion, -1, "Version of mechanism to use for usm allocation reuse.") DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalUSMAllocationReuseCleaner, -1, "Enable usm allocation reuse cleaner. -1: default, 0: disable, 1:enable") DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalUSMAllocationReuseLimitThreshold, -1, "Threshold of used memory to limit usm reuse. -1: default, 0: disable, >0:X% of shared/device memory") DECLARE_DEBUG_VARIABLE(int32_t, ExperimentalH2DCpuCopyThreshold, -1, "Override default threshold (in bytes) for H2D CPU copy.") diff --git a/shared/source/device/device.cpp b/shared/source/device/device.cpp index 03388eefea..236da3b147 100644 --- a/shared/source/device/device.cpp +++ b/shared/source/device/device.cpp @@ -214,22 +214,6 @@ bool Device::initializeCommonResources() { } } - bool usmPoolManagerEnabled = ApiSpecificConfig::isDeviceUsmPoolingEnabled() && - getProductHelper().isDeviceUsmPoolAllocatorSupported(); - - if (NEO::debugManager.flags.EnableDeviceUsmAllocationPool.get() != -1) { - usmPoolManagerEnabled = NEO::debugManager.flags.EnableDeviceUsmAllocationPool.get() > 0; - } - - if (usmPoolManagerEnabled && NEO::debugManager.flags.ExperimentalUSMAllocationReuseVersion.get() == 2) { - - RootDeviceIndicesContainer rootDeviceIndices; - rootDeviceIndices.pushUnique(getRootDeviceIndex()); - std::map deviceBitfields; - deviceBitfields.emplace(getRootDeviceIndex(), getDeviceBitfield()); - deviceUsmMemAllocPoolsManager.reset(new UsmMemAllocPoolsManager(getMemoryManager(), rootDeviceIndices, deviceBitfields, this, InternalMemoryType::deviceUnifiedMemory)); - } - this->resetUsmConstantSurfaceAllocPool(new UsmMemAllocPool); this->resetUsmGlobalSurfaceAllocPool(new UsmMemAllocPool); @@ -272,10 +256,17 @@ void Device::resetUsmAllocationPool(UsmMemAllocPool *usmMemAllocPool) { this->usmMemAllocPool.reset(usmMemAllocPool); } +void Device::resetUsmAllocationPoolManager(UsmMemAllocPoolsManager *usmMemAllocPoolManager) { + this->deviceUsmMemAllocPoolsManager.reset(usmMemAllocPoolManager); +} + void Device::cleanupUsmAllocationPool() { if (usmMemAllocPool) { usmMemAllocPool->cleanup(); } + if (deviceUsmMemAllocPoolsManager) { + deviceUsmMemAllocPoolsManager->cleanup(); + } } void Device::resetUsmConstantSurfaceAllocPool(UsmMemAllocPool *usmMemAllocPool) { diff --git a/shared/source/device/device.h b/shared/source/device/device.h index b6653ae8d7..bf03d8c85f 100644 --- a/shared/source/device/device.h +++ b/shared/source/device/device.h @@ -266,6 +266,7 @@ class Device : public ReferenceTrackedObject, NEO::NonCopyableAndNonMova UsmReuseInfo usmReuseInfo; void resetUsmAllocationPool(UsmMemAllocPool *usmMemAllocPool); + void resetUsmAllocationPoolManager(UsmMemAllocPoolsManager *usmMemAllocPoolManager); void cleanupUsmAllocationPool(); void resetUsmConstantSurfaceAllocPool(UsmMemAllocPool *usmMemAllocPool); diff --git a/shared/source/memory_manager/unified_memory_pooling.cpp b/shared/source/memory_manager/unified_memory_pooling.cpp index fdfe0e119b..bbebb5b641 100644 --- a/shared/source/memory_manager/unified_memory_pooling.cpp +++ b/shared/source/memory_manager/unified_memory_pooling.cpp @@ -35,10 +35,10 @@ bool UsmMemAllocPool::initialize(SVMAllocsManager *svmMemoryManager, void *ptr, svmData->size, chunkAlignment, maxServicedSize / 2)); - this->poolSize = svmData->size; this->poolMemoryType = svmData->memoryType; - this->minServicedSize = minServicedSize; - this->maxServicedSize = maxServicedSize; + this->poolInfo.minServicedSize = minServicedSize; + this->poolInfo.maxServicedSize = maxServicedSize; + this->poolInfo.poolSize = svmData->size; return true; } @@ -47,7 +47,7 @@ bool UsmMemAllocPool::isInitialized() const { } size_t UsmMemAllocPool::getPoolSize() const { - return this->poolSize; + return this->poolInfo.poolSize; } void UsmMemAllocPool::cleanup() { @@ -57,7 +57,7 @@ void UsmMemAllocPool::cleanup() { this->svmMemoryManager = nullptr; this->pool = nullptr; this->poolEnd = nullptr; - this->poolSize = 0u; + this->poolInfo.poolSize = 0u; this->poolMemoryType = InternalMemoryType::notSpecified; } } @@ -67,7 +67,7 @@ bool UsmMemAllocPool::alignmentIsAllowed(size_t alignment) { } bool UsmMemAllocPool::sizeIsAllowed(size_t size) { - return size >= minServicedSize && size <= maxServicedSize; + return size >= poolInfo.minServicedSize && size <= poolInfo.maxServicedSize; } bool UsmMemAllocPool::flagsAreAllowed(const UnifiedMemoryProperties &memoryProperties) { @@ -121,7 +121,7 @@ bool UsmMemAllocPool::isInPool(const void *ptr) const { return ptr >= this->pool && ptr < this->poolEnd; } -bool UsmMemAllocPool::isEmpty() { +bool UsmMemAllocPool::isEmpty() const { return 0u == this->allocations.getNumAllocs(); } @@ -171,38 +171,23 @@ uint64_t UsmMemAllocPool::getPoolAddress() const { return castToUint64(this->pool); } -bool UsmMemAllocPoolsManager::PoolInfo::isPreallocated() const { - return 0u != preallocateSize; +UsmMemAllocPool::PoolInfo UsmMemAllocPool::getPoolInfo() const { + return poolInfo; } -bool UsmMemAllocPoolsManager::ensureInitialized(SVMAllocsManager *svmMemoryManager) { +bool UsmMemAllocPoolsManager::initialize(SVMAllocsManager *svmMemoryManager) { DEBUG_BREAK_IF(poolMemoryType != InternalMemoryType::deviceUnifiedMemory && poolMemoryType != InternalMemoryType::hostUnifiedMemory); - if (isInitialized()) { - return true; - } - std::unique_lock lock(mtx); - if (isInitialized()) { - return true; - } - bool allPoolAllocationsSucceeded = true; - this->totalSize = 0u; - SVMAllocsManager::UnifiedMemoryProperties poolsMemoryProperties(poolMemoryType, MemoryConstants::pageSize2M, rootDeviceIndices, deviceBitFields); - poolsMemoryProperties.device = device; + DEBUG_BREAK_IF(device == nullptr && poolMemoryType == InternalMemoryType::deviceUnifiedMemory); + this->svmMemoryManager = svmMemoryManager; for (const auto &poolInfo : this->poolInfos) { this->pools[poolInfo] = std::vector>(); - if (poolInfo.isPreallocated()) { - auto pool = std::make_unique(); - allPoolAllocationsSucceeded &= pool->initialize(svmMemoryManager, poolsMemoryProperties, poolInfo.preallocateSize, poolInfo.minServicedSize, poolInfo.maxServicedSize); - this->pools[poolInfo].push_back(std::move(pool)); - this->totalSize += poolInfo.preallocateSize; + auto pool = tryAddPool(poolInfo); + if (nullptr == pool) { + cleanup(); + return false; } } - if (false == allPoolAllocationsSucceeded) { - cleanup(); - return false; - } - this->svmMemoryManager = svmMemoryManager; return true; } @@ -210,34 +195,13 @@ bool UsmMemAllocPoolsManager::isInitialized() const { return nullptr != this->svmMemoryManager; } -void UsmMemAllocPoolsManager::trim() { - std::unique_lock lock(mtx); - for (const auto &poolInfo : this->poolInfos) { - if (false == poolInfo.isPreallocated()) { - trim(this->pools[poolInfo]); - } - } -} - -void UsmMemAllocPoolsManager::trim(std::vector> &poolVector) { - auto poolIterator = poolVector.begin(); - while (poolIterator != poolVector.end()) { - if ((*poolIterator)->isEmpty()) { - this->totalSize -= (*poolIterator)->getPoolSize(); - (*poolIterator)->cleanup(); - poolIterator = poolVector.erase(poolIterator); - } else { - ++poolIterator; - } - } -} - void UsmMemAllocPoolsManager::cleanup() { - for (const auto &poolInfo : this->poolInfos) { - for (const auto &pool : this->pools[poolInfo]) { + for (auto &[_, bucket] : this->pools) { + for (const auto &pool : bucket) { pool->cleanup(); } } + this->pools.clear(); this->svmMemoryManager = nullptr; } @@ -247,83 +211,90 @@ void *UsmMemAllocPoolsManager::createUnifiedMemoryAllocation(size_t size, const return nullptr; } std::unique_lock lock(mtx); + void *ptr = nullptr; for (const auto &poolInfo : this->poolInfos) { if (size <= poolInfo.maxServicedSize) { for (auto &pool : this->pools[poolInfo]) { - if (void *ptr = pool->createUnifiedMemoryAllocation(size, memoryProperties)) { - return ptr; + if (nullptr != (ptr = pool->createUnifiedMemoryAllocation(size, memoryProperties))) { + break; + } + } + if (nullptr == ptr) { + if (auto pool = tryAddPool(poolInfo)) { + ptr = pool->createUnifiedMemoryAllocation(size, memoryProperties); + DEBUG_BREAK_IF(nullptr == ptr); } } break; } } - return nullptr; + return ptr; +} + +UsmMemAllocPool *UsmMemAllocPoolsManager::tryAddPool(PoolInfo poolInfo) { + UsmMemAllocPool *poolPtr = nullptr; + if (canAddPool(poolInfo)) { + auto pool = std::make_unique(); + if (pool->initialize(svmMemoryManager, poolMemoryProperties, poolInfo.poolSize, poolInfo.minServicedSize, poolInfo.maxServicedSize)) { + poolPtr = pool.get(); + this->totalSize += pool->getPoolSize(); + this->pools[poolInfo].push_back(std::move(pool)); + } + } + return poolPtr; +} +bool UsmMemAllocPoolsManager::canAddPool(PoolInfo poolInfo) { + return true; +} + +void UsmMemAllocPoolsManager::trimEmptyPools(PoolInfo poolInfo) { + std::lock_guard lock(mtx); + auto &bucket = pools[poolInfo]; + auto firstEmptyPoolIt = std::partition(bucket.begin(), bucket.end(), [](std::unique_ptr &pool) { + return !pool->isEmpty(); + }); + const auto emptyPoolsCount = static_cast(std::distance(firstEmptyPoolIt, bucket.end())); + if (emptyPoolsCount > maxEmptyPoolsPerBucket) { + std::advance(firstEmptyPoolIt, maxEmptyPoolsPerBucket); + for (auto it = firstEmptyPoolIt; it != bucket.end(); ++it) { + (*it)->cleanup(); + } + bucket.erase(firstEmptyPoolIt, bucket.end()); + } } bool UsmMemAllocPoolsManager::freeSVMAlloc(const void *ptr, bool blocking) { - if (UsmMemAllocPool *pool = this->getPoolContainingAlloc(ptr)) { - return pool->freeSVMAlloc(ptr, blocking); + bool allocFreed = false; + if (auto pool = this->getPoolContainingAlloc(ptr); pool) { + allocFreed = pool->freeSVMAlloc(ptr, blocking); + if (allocFreed && pool->isEmpty()) { + trimEmptyPools(pool->getPoolInfo()); + } } - return false; + return allocFreed; } size_t UsmMemAllocPoolsManager::getPooledAllocationSize(const void *ptr) { - if (UsmMemAllocPool *pool = this->getPoolContainingAlloc(ptr)) { + if (auto pool = this->getPoolContainingAlloc(ptr); pool) { return pool->getPooledAllocationSize(ptr); } return 0u; } void *UsmMemAllocPoolsManager::getPooledAllocationBasePtr(const void *ptr) { - if (UsmMemAllocPool *pool = this->getPoolContainingAlloc(ptr)) { + if (auto pool = this->getPoolContainingAlloc(ptr); pool) { return pool->getPooledAllocationBasePtr(ptr); } return nullptr; } size_t UsmMemAllocPoolsManager::getOffsetInPool(const void *ptr) { - if (UsmMemAllocPool *pool = this->getPoolContainingAlloc(ptr)) { + if (auto pool = this->getPoolContainingAlloc(ptr); pool) { return pool->getOffsetInPool(ptr); } return 0u; } -uint64_t UsmMemAllocPoolsManager::getFreeMemory() { - const auto isIntegrated = device->getHardwareInfo().capabilityTable.isIntegratedDevice; - const uint64_t deviceMemory = isIntegrated ? device->getDeviceInfo().globalMemSize : device->getDeviceInfo().localMemSize; - const uint64_t usedMemory = memoryManager->getUsedLocalMemorySize(device->getRootDeviceIndex()); - DEBUG_BREAK_IF(usedMemory > deviceMemory); - const uint64_t freeMemory = deviceMemory - usedMemory; - return freeMemory; -} - -bool UsmMemAllocPoolsManager::recycleSVMAlloc(void *ptr, bool blocking) { - if (false == isInitialized()) { - return false; - } - auto svmData = this->svmMemoryManager->getSVMAlloc(ptr); - DEBUG_BREAK_IF(svmData->memoryType != this->poolMemoryType); - if (svmData->size > maxPoolableSize || belongsInPreallocatedPool(svmData->size)) { - return false; - } - std::unique_lock lock(mtx); - if (this->totalSize + svmData->size > getFreeMemory() * UsmMemAllocPool::getPercentOfFreeMemoryForRecycling(svmData->memoryType)) { - return false; - } - for (auto poolInfoIndex = firstNonPreallocatedIndex; poolInfoIndex < this->poolInfos.size(); ++poolInfoIndex) { - const auto &poolInfo = this->poolInfos[poolInfoIndex]; - if (svmData->size <= poolInfo.maxServicedSize) { - auto pool = std::make_unique(); - pool->initialize(this->svmMemoryManager, ptr, svmData, poolInfo.minServicedSize, svmData->size); - this->pools[poolInfo].push_back(std::move(pool)); - this->totalSize += svmData->size; - return true; - } - } - DEBUG_BREAK_IF(true); - return false; -} - UsmMemAllocPool *UsmMemAllocPoolsManager::getPoolContainingAlloc(const void *ptr) { std::unique_lock lock(mtx); for (const auto &poolInfo : this->poolInfos) { diff --git a/shared/source/memory_manager/unified_memory_pooling.h b/shared/source/memory_manager/unified_memory_pooling.h index 89ddf10936..c41aa676f4 100644 --- a/shared/source/memory_manager/unified_memory_pooling.h +++ b/shared/source/memory_manager/unified_memory_pooling.h @@ -15,8 +15,16 @@ #include namespace NEO { -class UsmMemAllocPool { +class UsmMemAllocPool : NEO::NonCopyableAndNonMovableClass { public: + struct PoolInfo { + size_t minServicedSize; + size_t maxServicedSize; + size_t poolSize; + bool operator<(const PoolInfo &rhs) const { + return this->minServicedSize < rhs.minServicedSize; + } + }; using UnifiedMemoryProperties = SVMAllocsManager::UnifiedMemoryProperties; struct AllocationInfo { uint64_t address; @@ -26,7 +34,7 @@ class UsmMemAllocPool { using AllocationsInfoStorage = BaseSortedPointerWithValueVector; UsmMemAllocPool() = default; - virtual ~UsmMemAllocPool() = default; + MOCKABLE_VIRTUAL ~UsmMemAllocPool() = default; MOCKABLE_VIRTUAL bool initialize(SVMAllocsManager *svmMemoryManager, const UnifiedMemoryProperties &memoryProperties, size_t poolSize, size_t minServicedSize, size_t maxServicedSize); bool initialize(SVMAllocsManager *svmMemoryManager, void *ptr, SvmAllocationData *svmData, size_t minServicedSize, size_t maxServicedSize); bool isInitialized() const; @@ -39,12 +47,13 @@ class UsmMemAllocPool { bool canBePooled(size_t size, const UnifiedMemoryProperties &memoryProperties); MOCKABLE_VIRTUAL void *createUnifiedMemoryAllocation(size_t size, const UnifiedMemoryProperties &memoryProperties); bool isInPool(const void *ptr) const; - bool isEmpty(); + bool isEmpty() const; MOCKABLE_VIRTUAL bool freeSVMAlloc(const void *ptr, bool blocking); size_t getPooledAllocationSize(const void *ptr); void *getPooledAllocationBasePtr(const void *ptr); size_t getOffsetInPool(const void *ptr) const; uint64_t getPoolAddress() const; + PoolInfo getPoolInfo() const; std::mutex &getMutex() noexcept { return mtx; } static constexpr auto chunkAlignment = 512u; @@ -59,54 +68,41 @@ class UsmMemAllocPool { std::mutex mtx; RootDeviceIndicesContainer rootDeviceIndices; std::map deviceBitFields; - Device *device; - InternalMemoryType poolMemoryType; - size_t poolSize{}; - size_t minServicedSize; - size_t maxServicedSize; + Device *device{}; + InternalMemoryType poolMemoryType = InternalMemoryType::notSpecified; + PoolInfo poolInfo{}; }; -class UsmMemAllocPoolsManager { +class UsmMemAllocPoolsManager : NEO::NonCopyableAndNonMovableClass { public: - struct PoolInfo { - size_t minServicedSize; - size_t maxServicedSize; - size_t preallocateSize; - bool isPreallocated() const; - bool operator<(const PoolInfo &rhs) const { - return this->minServicedSize < rhs.minServicedSize; - } - }; - // clang-format off - const std::array poolInfos = { - PoolInfo{ 0, 4 * KB, 2 * MB}, - PoolInfo{ 4 * KB+1, 64 * KB, 2 * MB}, - PoolInfo{64 * KB+1, 2 * MB, 16 * MB}, - PoolInfo{ 2 * MB+1, 16 * MB, 0}, - PoolInfo{16 * MB+1, 64 * MB, 0}, - PoolInfo{64 * MB+1, 256 * MB, 0}}; - // clang-format on - static constexpr size_t firstNonPreallocatedIndex = 3u; - - using UnifiedMemoryProperties = SVMAllocsManager::UnifiedMemoryProperties; + using PoolInfo = UsmMemAllocPool::PoolInfo; static constexpr uint64_t KB = MemoryConstants::kiloByte; // NOLINT(readability-identifier-naming) static constexpr uint64_t MB = MemoryConstants::megaByte; // NOLINT(readability-identifier-naming) - static constexpr uint64_t maxPoolableSize = 256 * MB; - UsmMemAllocPoolsManager(MemoryManager *memoryManager, + static constexpr uint64_t maxPoolableSize = 2 * MB; + // clang-format off + static constexpr std::array poolInfos = { + PoolInfo{ 0, 4 * KB, 2 * MB}, + PoolInfo{ 4 * KB+1, 64 * KB, 2 * MB}, + PoolInfo{64 * KB+1, 2 * MB, 16 * MB}}; + // clang-format on + static constexpr size_t maxEmptyPoolsPerBucket = 1u; + + using UnifiedMemoryProperties = SVMAllocsManager::UnifiedMemoryProperties; + UsmMemAllocPoolsManager(InternalMemoryType memoryType, const RootDeviceIndicesContainer &rootDeviceIndices, - const std::map &deviceBitFields, - Device *device, - InternalMemoryType poolMemoryType) : memoryManager(memoryManager), rootDeviceIndices(rootDeviceIndices), deviceBitFields(deviceBitFields), device(device), poolMemoryType(poolMemoryType){}; + const std::map &subdeviceBitfields, + Device *device) : device(device), poolMemoryType(memoryType), poolMemoryProperties(memoryType, UsmMemAllocPool::poolAlignment, rootDeviceIndices, subdeviceBitfields) { + poolMemoryProperties.device = device; + }; MOCKABLE_VIRTUAL ~UsmMemAllocPoolsManager() = default; - bool ensureInitialized(SVMAllocsManager *svmMemoryManager); + bool initialize(SVMAllocsManager *svmMemoryManager); bool isInitialized() const; - void trim(); - void trim(std::vector> &poolVector); void cleanup(); void *createUnifiedMemoryAllocation(size_t size, const UnifiedMemoryProperties &memoryProperties); + UsmMemAllocPool *tryAddPool(PoolInfo poolInfo); + MOCKABLE_VIRTUAL bool canAddPool(PoolInfo poolInfo); + void trimEmptyPools(PoolInfo poolInfo); bool freeSVMAlloc(const void *ptr, bool blocking); - MOCKABLE_VIRTUAL uint64_t getFreeMemory(); - bool recycleSVMAlloc(void *ptr, bool blocking); size_t getPooledAllocationSize(const void *ptr); void *getPooledAllocationBasePtr(const void *ptr); size_t getOffsetInPool(const void *ptr); @@ -118,16 +114,12 @@ class UsmMemAllocPoolsManager { UsmMemAllocPool::alignmentIsAllowed(memoryProperties.alignment) && UsmMemAllocPool::flagsAreAllowed(memoryProperties); } - bool belongsInPreallocatedPool(size_t size) { - return size <= poolInfos[firstNonPreallocatedIndex - 1].maxServicedSize; - } SVMAllocsManager *svmMemoryManager{}; - MemoryManager *memoryManager; - RootDeviceIndicesContainer rootDeviceIndices; - std::map deviceBitFields; - Device *device; - InternalMemoryType poolMemoryType; + MemoryManager *memoryManager{}; + Device *device{nullptr}; + const InternalMemoryType poolMemoryType; + UnifiedMemoryProperties poolMemoryProperties; size_t totalSize{}; std::mutex mtx; std::map>> pools; diff --git a/shared/test/common/mocks/mock_usm_memory_pool.h b/shared/test/common/mocks/mock_usm_memory_pool.h index a6d7e82f10..2a49cc0493 100644 --- a/shared/test/common/mocks/mock_usm_memory_pool.h +++ b/shared/test/common/mocks/mock_usm_memory_pool.h @@ -12,12 +12,10 @@ namespace NEO { class MockUsmMemAllocPool : public UsmMemAllocPool { public: using UsmMemAllocPool::allocations; - using UsmMemAllocPool::maxServicedSize; - using UsmMemAllocPool::minServicedSize; using UsmMemAllocPool::pool; using UsmMemAllocPool::poolEnd; + using UsmMemAllocPool::poolInfo; using UsmMemAllocPool::poolMemoryType; - using UsmMemAllocPool::poolSize; bool initialize(SVMAllocsManager *svmMemoryManager, const UnifiedMemoryProperties &memoryProperties, size_t poolSize, size_t minServicedSize, size_t maxServicedSize) override { if (callBaseInitialize) { @@ -62,13 +60,13 @@ class MockUsmMemAllocPoolsManager : public UsmMemAllocPoolsManager { using UsmMemAllocPoolsManager::pools; using UsmMemAllocPoolsManager::totalSize; using UsmMemAllocPoolsManager::UsmMemAllocPoolsManager; - uint64_t getFreeMemory() override { - if (callBaseGetFreeMemory) { - return UsmMemAllocPoolsManager::getFreeMemory(); + bool canAddPool(PoolInfo poolInfo) override { + if (canAddPoolCallBase) { + return UsmMemAllocPoolsManager::canAddPool(poolInfo); } - return mockFreeMemory; + return canAddPools; } - uint64_t mockFreeMemory = 0u; - bool callBaseGetFreeMemory = false; + bool canAddPools = true; + bool canAddPoolCallBase = false; }; } // namespace NEO \ No newline at end of file diff --git a/shared/test/common/test_files/igdrcl.config b/shared/test/common/test_files/igdrcl.config index 678d1fb409..5cd9585761 100644 --- a/shared/test/common/test_files/igdrcl.config +++ b/shared/test/common/test_files/igdrcl.config @@ -614,7 +614,6 @@ WaitForPagingFenceInController = -1 DirectSubmissionPrintSemaphoreUsage = -1 ForceNonCoherentModeForTimestamps = -1 SetAssumeNotInUse = 1 -ExperimentalUSMAllocationReuseVersion = -1 ForceNonWalkerSplitMemoryCopy = -1 FinalizerInputType = unk FinalizerLibraryName = unk @@ -670,5 +669,6 @@ Disable2MBSizeAlignment = 0 InOrderCopyMiFlushSync = -1 SplitBcsForCopyOffload = -1 LimitIsaPrefetchSize = -1 +EnableUsmAllocationPoolManager = -1 ForceTotalWMTPDataSize = -1 # Please don't edit below this line diff --git a/shared/test/unit_test/device/neo_device_tests.cpp b/shared/test/unit_test/device/neo_device_tests.cpp index f3d121009e..eccbd5ce14 100644 --- a/shared/test/unit_test/device/neo_device_tests.cpp +++ b/shared/test/unit_test/device/neo_device_tests.cpp @@ -2577,74 +2577,6 @@ TEST_F(DeviceTests, givenDebuggerRequestedByUserWhenDeviceWithSubDevicesCreatedT EXPECT_NE(nullptr, deviceFactory.rootDevices[0]->getL0Debugger()); } -TEST_F(DeviceTests, givenNewUsmPoolingEnabledWhenDeviceInitializedThenUsmMemAllocPoolsManagerIsCreatedButNotInitialized) { - VariableBackup backupIsDeviceUsmPoolingEnabledForUlts(&isDeviceUsmPoolingEnabledForUlts); - isDeviceUsmPoolingEnabledForUlts = true; - { - DebugManagerStateRestore restorer; - debugManager.flags.ExperimentalUSMAllocationReuseVersion.set(2); - auto executionEnvironment = MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u); - auto mockProductHelper = new MockProductHelper; - executionEnvironment->rootDeviceEnvironments[0]->productHelper.reset(mockProductHelper); - mockProductHelper->isDeviceUsmPoolAllocatorSupportedResult = true; - UltDeviceFactory deviceFactory{1, 1, *executionEnvironment}; - auto device = deviceFactory.rootDevices[0]; - auto usmMemAllocPoolsManager = device->getUsmMemAllocPoolsManager(); - ASSERT_NE(nullptr, usmMemAllocPoolsManager); - EXPECT_FALSE(usmMemAllocPoolsManager->isInitialized()); - } - { - DebugManagerStateRestore restorer; - debugManager.flags.ExperimentalUSMAllocationReuseVersion.set(-1); - auto executionEnvironment = MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u); - auto mockProductHelper = new MockProductHelper; - executionEnvironment->rootDeviceEnvironments[0]->productHelper.reset(mockProductHelper); - mockProductHelper->isDeviceUsmPoolAllocatorSupportedResult = true; - UltDeviceFactory deviceFactory{1, 1, *executionEnvironment}; - auto device = deviceFactory.rootDevices[0]; - auto usmMemAllocPoolsManager = device->getUsmMemAllocPoolsManager(); - EXPECT_EQ(nullptr, usmMemAllocPoolsManager); - } - { - DebugManagerStateRestore restorer; - debugManager.flags.ExperimentalUSMAllocationReuseVersion.set(2); - auto executionEnvironment = MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u); - auto mockProductHelper = new MockProductHelper; - executionEnvironment->rootDeviceEnvironments[0]->productHelper.reset(mockProductHelper); - mockProductHelper->isDeviceUsmPoolAllocatorSupportedResult = false; - UltDeviceFactory deviceFactory{1, 1, *executionEnvironment}; - auto device = deviceFactory.rootDevices[0]; - auto usmMemAllocPoolsManager = device->getUsmMemAllocPoolsManager(); - EXPECT_EQ(nullptr, usmMemAllocPoolsManager); - } - isDeviceUsmPoolingEnabledForUlts = false; - { - DebugManagerStateRestore restorer; - debugManager.flags.ExperimentalUSMAllocationReuseVersion.set(2); - auto executionEnvironment = MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u); - auto mockProductHelper = new MockProductHelper; - executionEnvironment->rootDeviceEnvironments[0]->productHelper.reset(mockProductHelper); - mockProductHelper->isDeviceUsmPoolAllocatorSupportedResult = true; - UltDeviceFactory deviceFactory{1, 1, *executionEnvironment}; - auto device = deviceFactory.rootDevices[0]; - auto usmMemAllocPoolsManager = device->getUsmMemAllocPoolsManager(); - EXPECT_EQ(nullptr, usmMemAllocPoolsManager); - } - { - DebugManagerStateRestore restorer; - debugManager.flags.ExperimentalUSMAllocationReuseVersion.set(2); - debugManager.flags.EnableDeviceUsmAllocationPool.set(1); - auto executionEnvironment = MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u); - auto mockProductHelper = new MockProductHelper; - executionEnvironment->rootDeviceEnvironments[0]->productHelper.reset(mockProductHelper); - mockProductHelper->isDeviceUsmPoolAllocatorSupportedResult = true; - UltDeviceFactory deviceFactory{1, 1, *executionEnvironment}; - auto device = deviceFactory.rootDevices[0]; - auto usmMemAllocPoolsManager = device->getUsmMemAllocPoolsManager(); - ASSERT_NE(nullptr, usmMemAllocPoolsManager); - EXPECT_FALSE(usmMemAllocPoolsManager->isInitialized()); - } -} TEST(DeviceWithoutAILTest, givenNoAILWhenCreateDeviceThenDeviceIsCreated) { DebugManagerStateRestore dbgRestorer; debugManager.flags.EnableAIL.set(false); @@ -2697,13 +2629,22 @@ TEST(Device, givenDeviceWhenCallingUsmAllocationPoolMethodsThenCorrectValueRetur EXPECT_EQ(nullptr, device->getUsmMemAllocPool()); device->cleanupUsmAllocationPool(); - MockUsmMemAllocPool *usmAllocPool = new MockUsmMemAllocPool; + auto *usmAllocPool = new MockUsmMemAllocPool; device->resetUsmAllocationPool(usmAllocPool); EXPECT_EQ(usmAllocPool, device->getUsmMemAllocPool()); usmAllocPool->callBaseCleanup = false; EXPECT_EQ(0u, usmAllocPool->cleanupCalled); device->cleanupUsmAllocationPool(); EXPECT_EQ(1u, usmAllocPool->cleanupCalled); + + EXPECT_EQ(nullptr, device->getUsmMemAllocPoolsManager()); + RootDeviceIndicesContainer rootDeviceIndices = {device->getRootDeviceIndex()}; + std::map deviceBitfields{{device->getRootDeviceIndex(), device->getDeviceBitfield()}}; + MockUsmMemAllocPoolsManager *usmAllocPoolManager = new MockUsmMemAllocPoolsManager(InternalMemoryType::deviceUnifiedMemory, rootDeviceIndices, deviceBitfields, device.get()); + device->resetUsmAllocationPoolManager(usmAllocPoolManager); + EXPECT_EQ(usmAllocPoolManager, device->getUsmMemAllocPoolsManager()); + usmAllocPoolManager->canAddPoolCallBase = true; + EXPECT_TRUE(usmAllocPoolManager->canAddPool(UsmMemAllocPoolsManager::poolInfos[0])); } TEST(GroupDevicesTest, whenMultipleDevicesAreCreatedThenGroupDevicesCreatesVectorPerEachProductFamilySortedOverGpuTypeAndProductFamily) { diff --git a/shared/test/unit_test/memory_manager/unified_memory_pooling_tests.cpp b/shared/test/unit_test/memory_manager/unified_memory_pooling_tests.cpp index d1aea7e8da..bd3ee0aed7 100644 --- a/shared/test/unit_test/memory_manager/unified_memory_pooling_tests.cpp +++ b/shared/test/unit_test/memory_manager/unified_memory_pooling_tests.cpp @@ -291,37 +291,27 @@ TEST_F(UnifiedMemoryPoolingManagerStaticTest, givenUsmMemAllocPoolsManagerWhenCa EXPECT_FALSE(MockUsmMemAllocPoolsManager::canBePooled(UsmMemAllocPoolsManager::maxPoolableSize, unifiedMemoryProperties)); } -class UnifiedMemoryPoolingManagerTest : public SVMMemoryAllocatorFixture, public ::testing::TestWithParam> { +class UnifiedMemoryPoolingManagerTest : public SVMMemoryAllocatorFixture, public ::testing::TestWithParam> { public: void SetUp() override { REQUIRE_64BIT_OR_SKIP(); SVMMemoryAllocatorFixture::setUp(); poolMemoryType = std::get<0>(GetParam()); - failAllocation = std::get<1>(GetParam()); deviceFactory = std::unique_ptr(new UltDeviceFactory(1, 1)); device = deviceFactory->rootDevices[0]; - RootDeviceIndicesContainer rootDeviceIndicesPool; - rootDeviceIndicesPool.pushUnique(device->getRootDeviceIndex()); - std::map deviceBitfieldsPool; - deviceBitfieldsPool.emplace(device->getRootDeviceIndex(), device->getDeviceBitfield()); - usmMemAllocPoolsManager.reset(new MockUsmMemAllocPoolsManager(device->getMemoryManager(), - rootDeviceIndicesPool, - deviceBitfieldsPool, - device, - poolMemoryType)); + usmMemAllocPoolsManager.reset(new MockUsmMemAllocPoolsManager(poolMemoryType, + rootDeviceIndices, + deviceBitfields, + poolMemoryType == InternalMemoryType::deviceUnifiedMemory ? device : nullptr)); ASSERT_NE(nullptr, usmMemAllocPoolsManager); EXPECT_FALSE(usmMemAllocPoolsManager->isInitialized()); poolInfo0To4Kb = usmMemAllocPoolsManager->poolInfos[0]; poolInfo4KbTo64Kb = usmMemAllocPoolsManager->poolInfos[1]; poolInfo64KbTo2Mb = usmMemAllocPoolsManager->poolInfos[2]; - poolInfo2MbTo16Mb = usmMemAllocPoolsManager->poolInfos[3]; - poolInfo16MbTo64Mb = usmMemAllocPoolsManager->poolInfos[4]; - poolInfo64MbTo256Mb = usmMemAllocPoolsManager->poolInfos[5]; svmManager = std::make_unique(device->getMemoryManager()); mockMemoryManager = static_cast(device->getMemoryManager()); - mockMemoryManager->failInDevicePoolWithError = failAllocation; if (InternalMemoryType::deviceUnifiedMemory == poolMemoryType) { mockMemoryManager->localMemorySupported[mockRootDeviceIndex] = true; } @@ -364,65 +354,68 @@ class UnifiedMemoryPoolingManagerTest : public SVMMemoryAllocatorFixture, std::unique_ptr poolMemoryProperties; MockMemoryManager *mockMemoryManager; InternalMemoryType poolMemoryType; - bool failAllocation; uint64_t nextMockGraphicsAddress = alignUp(std::numeric_limits::max() - MemoryConstants::teraByte, MemoryConstants::pageSize2M); + UsmMemAllocPoolsManager::PoolInfo poolInfo0To4Kb; UsmMemAllocPoolsManager::PoolInfo poolInfo4KbTo64Kb; UsmMemAllocPoolsManager::PoolInfo poolInfo64KbTo2Mb; - UsmMemAllocPoolsManager::PoolInfo poolInfo2MbTo16Mb; - UsmMemAllocPoolsManager::PoolInfo poolInfo16MbTo64Mb; - UsmMemAllocPoolsManager::PoolInfo poolInfo64MbTo256Mb; }; INSTANTIATE_TEST_SUITE_P( UnifiedMemoryPoolingManagerTestParameterized, UnifiedMemoryPoolingManagerTest, ::testing::Combine( - ::testing::Values(InternalMemoryType::deviceUnifiedMemory, InternalMemoryType::hostUnifiedMemory), - ::testing::Values(false))); + ::testing::Values(InternalMemoryType::deviceUnifiedMemory, InternalMemoryType::hostUnifiedMemory))); -TEST_P(UnifiedMemoryPoolingManagerTest, givenNotInitializedPoolsManagerWhenUsingPoolThenMethodsSucceed) { - void *ptr = reinterpret_cast(0x1u); - const void *constPtr = const_cast(ptr); - EXPECT_FALSE(usmMemAllocPoolsManager->freeSVMAlloc(constPtr, true)); - EXPECT_EQ(nullptr, usmMemAllocPoolsManager->getPooledAllocationBasePtr(constPtr)); - EXPECT_EQ(0u, usmMemAllocPoolsManager->getPooledAllocationSize(constPtr)); - EXPECT_FALSE(usmMemAllocPoolsManager->recycleSVMAlloc(ptr, true)); - EXPECT_EQ(0u, usmMemAllocPoolsManager->getOffsetInPool(constPtr)); - EXPECT_EQ(nullptr, usmMemAllocPoolsManager->getPoolContainingAlloc(constPtr)); - usmMemAllocPoolsManager->trim(); +TEST_P(UnifiedMemoryPoolingManagerTest, givenInitializationFailsForOneOfTheSmallPoolsWhenInitializingPoolsManagerThenPoolsAreCleanedUp) { + mockMemoryManager->maxSuccessAllocatedGraphicsMemoryIndex = mockMemoryManager->successAllocatedGraphicsMemoryIndex + 2; + EXPECT_FALSE(usmMemAllocPoolsManager->initialize(svmManager.get())); + EXPECT_FALSE(usmMemAllocPoolsManager->isInitialized()); + EXPECT_EQ(0u, usmMemAllocPoolsManager->pools[poolInfo0To4Kb].size()); + EXPECT_EQ(0u, usmMemAllocPoolsManager->pools[poolInfo4KbTo64Kb].size()); + EXPECT_EQ(0u, usmMemAllocPoolsManager->pools[poolInfo64KbTo2Mb].size()); +} - auto mockDevice = reinterpret_cast(usmMemAllocPoolsManager->device); - usmMemAllocPoolsManager->callBaseGetFreeMemory = true; - mockDevice->deviceInfo.localMemSize = 4 * MemoryConstants::gigaByte; - mockDevice->deviceInfo.globalMemSize = 8 * MemoryConstants::gigaByte; - mockMemoryManager->localMemAllocsSize[mockDevice->getRootDeviceIndex()].store(1 * MemoryConstants::gigaByte); - auto mutableHwInfo = mockDevice->getRootDeviceEnvironment().getMutableHardwareInfo(); - EXPECT_EQ(mutableHwInfo, &mockDevice->getHardwareInfo()); - mutableHwInfo->capabilityTable.isIntegratedDevice = false; - EXPECT_EQ(3 * MemoryConstants::gigaByte, usmMemAllocPoolsManager->getFreeMemory()); +TEST_P(UnifiedMemoryPoolingManagerTest, givenInitializedPoolsManagerWhenAllocatingGreaterThan2MBOrWrongAlignmentOrWrongFlagsThenDoNotPool) { + ASSERT_TRUE(usmMemAllocPoolsManager->initialize(svmManager.get())); + ASSERT_TRUE(usmMemAllocPoolsManager->isInitialized()); + auto allocOverLimit = usmMemAllocPoolsManager->createUnifiedMemoryAllocation(2 * MemoryConstants::megaByte + 1u, *poolMemoryProperties.get()); + EXPECT_EQ(nullptr, allocOverLimit); - mutableHwInfo->capabilityTable.isIntegratedDevice = true; - EXPECT_EQ(7 * MemoryConstants::gigaByte, usmMemAllocPoolsManager->getFreeMemory()); + poolMemoryProperties->allocationFlags.allFlags = 1u; + auto allocWithExtraFlags = usmMemAllocPoolsManager->createUnifiedMemoryAllocation(1u, *poolMemoryProperties.get()); + EXPECT_EQ(nullptr, allocWithExtraFlags); + + poolMemoryProperties->allocationFlags.allFlags = 0u; + poolMemoryProperties->alignment = 4 * MemoryConstants::megaByte; + auto allocWithWrongAlignment = usmMemAllocPoolsManager->createUnifiedMemoryAllocation(1u, *poolMemoryProperties.get()); + EXPECT_EQ(nullptr, allocWithWrongAlignment); usmMemAllocPoolsManager->cleanup(); } -TEST_P(UnifiedMemoryPoolingManagerTest, givenInitializationFailsForOneOfTheSmallPoolsWhenInitializingPoolsManagerThenPoolsAreCleanedUp) { - mockMemoryManager->maxSuccessAllocatedGraphicsMemoryIndex = mockMemoryManager->successAllocatedGraphicsMemoryIndex + 2; - EXPECT_FALSE(usmMemAllocPoolsManager->ensureInitialized(svmManager.get())); - EXPECT_FALSE(usmMemAllocPoolsManager->isInitialized()); - ASSERT_EQ(1u, usmMemAllocPoolsManager->pools[poolInfo0To4Kb].size()); - EXPECT_FALSE(usmMemAllocPoolsManager->pools[poolInfo0To4Kb][0]->isInitialized()); - ASSERT_EQ(1u, usmMemAllocPoolsManager->pools[poolInfo4KbTo64Kb].size()); - EXPECT_FALSE(usmMemAllocPoolsManager->pools[poolInfo4KbTo64Kb][0]->isInitialized()); - ASSERT_EQ(1u, usmMemAllocPoolsManager->pools[poolInfo64KbTo2Mb].size()); - EXPECT_FALSE(usmMemAllocPoolsManager->pools[poolInfo64KbTo2Mb][0]->isInitialized()); +TEST_P(UnifiedMemoryPoolingManagerTest, givenInitializedPoolsManagerWhenCallingMethodsWithNotAllocatedPointersThenReturnCorrectValues) { + ASSERT_TRUE(usmMemAllocPoolsManager->initialize(svmManager.get())); + ASSERT_TRUE(usmMemAllocPoolsManager->isInitialized()); + + void *ptrOutsidePools = addrToPtr(0x1); + EXPECT_FALSE(usmMemAllocPoolsManager->freeSVMAlloc(ptrOutsidePools, true)); + EXPECT_EQ(0u, usmMemAllocPoolsManager->getPooledAllocationSize(ptrOutsidePools)); + EXPECT_EQ(0u, usmMemAllocPoolsManager->getPooledAllocationBasePtr(ptrOutsidePools)); + EXPECT_EQ(0u, usmMemAllocPoolsManager->getOffsetInPool(ptrOutsidePools)); + + void *notAllocatedPtrInPoolAddressSpace = addrToPtr(usmMemAllocPoolsManager->pools[poolInfo0To4Kb][0]->getPoolAddress()); + EXPECT_FALSE(usmMemAllocPoolsManager->freeSVMAlloc(notAllocatedPtrInPoolAddressSpace, true)); + EXPECT_EQ(0u, usmMemAllocPoolsManager->getPooledAllocationSize(notAllocatedPtrInPoolAddressSpace)); + EXPECT_EQ(0u, usmMemAllocPoolsManager->getPooledAllocationBasePtr(notAllocatedPtrInPoolAddressSpace)); + EXPECT_EQ(0u, usmMemAllocPoolsManager->getOffsetInPool(notAllocatedPtrInPoolAddressSpace)); + + usmMemAllocPoolsManager->cleanup(); } -TEST_P(UnifiedMemoryPoolingManagerTest, givenInitializedPoolsManagerWhenAllocatingNotGreaterThan2MBThenSmallPoolsAreUsed) { - EXPECT_TRUE(usmMemAllocPoolsManager->ensureInitialized(svmManager.get())); - EXPECT_TRUE(usmMemAllocPoolsManager->isInitialized()); +TEST_P(UnifiedMemoryPoolingManagerTest, givenInitializedPoolsManagerWhenAllocatingNotGreaterThan2MBThenPoolsAreUsed) { + ASSERT_TRUE(usmMemAllocPoolsManager->initialize(svmManager.get())); + ASSERT_TRUE(usmMemAllocPoolsManager->isInitialized()); ASSERT_EQ(1u, usmMemAllocPoolsManager->pools[poolInfo0To4Kb].size()); EXPECT_TRUE(usmMemAllocPoolsManager->pools[poolInfo0To4Kb][0]->isInitialized()); ASSERT_EQ(1u, usmMemAllocPoolsManager->pools[poolInfo4KbTo64Kb].size()); @@ -500,116 +493,38 @@ TEST_P(UnifiedMemoryPoolingManagerTest, givenInitializedPoolsManagerWhenAllocati EXPECT_TRUE(usmMemAllocPoolsManager->pools[poolInfo64KbTo2Mb][0]->isInPool(thirdPoolAlloc2MB)); EXPECT_EQ(20 * MemoryConstants::megaByte, usmMemAllocPoolsManager->totalSize); - for (auto i = 0u; i < 7; ++i) { // use all memory in third pool + usmMemAllocPoolsManager->canAddPools = false; + std::vector ptrsToFree; + for (auto i = 0u; i < 9; ++i) { // use all memory in third pool auto ptr = usmMemAllocPoolsManager->createUnifiedMemoryAllocation(2 * MemoryConstants::megaByte, *poolMemoryProperties.get()); if (nullptr == ptr) { break; } + const auto address = castToUint64(ptr); + const auto offset = usmMemAllocPoolsManager->getOffsetInPool(ptr); + const auto pool = usmMemAllocPoolsManager->getPoolContainingAlloc(ptr); + const auto poolAddress = pool->getPoolAddress(); + EXPECT_EQ(ptrOffset(poolAddress, offset), address); + + ptrsToFree.push_back(ptr); } + EXPECT_EQ(1u, usmMemAllocPoolsManager->pools[poolInfo64KbTo2Mb].size()); + usmMemAllocPoolsManager->canAddPools = true; auto thirdPoolAlloc2MBOverCapacity = usmMemAllocPoolsManager->createUnifiedMemoryAllocation(2 * MemoryConstants::megaByte, *poolMemoryProperties.get()); - EXPECT_EQ(nullptr, thirdPoolAlloc2MBOverCapacity); + EXPECT_NE(nullptr, thirdPoolAlloc2MBOverCapacity); + EXPECT_EQ(36 * MemoryConstants::megaByte, usmMemAllocPoolsManager->totalSize); + ASSERT_EQ(2u, usmMemAllocPoolsManager->pools[poolInfo64KbTo2Mb].size()); + auto &newPool = usmMemAllocPoolsManager->pools[poolInfo64KbTo2Mb][1]; + EXPECT_NE(nullptr, newPool->getPooledAllocationBasePtr(thirdPoolAlloc2MBOverCapacity)); + + ptrsToFree.push_back(thirdPoolAlloc2MB); + ptrsToFree.push_back(thirdPoolAlloc2MBOverCapacity); + + for (auto ptr : ptrsToFree) { + EXPECT_TRUE(usmMemAllocPoolsManager->freeSVMAlloc(ptr, true)); + } + EXPECT_EQ(1u, usmMemAllocPoolsManager->pools[poolInfo64KbTo2Mb].size()); usmMemAllocPoolsManager->cleanup(); } - -TEST_P(UnifiedMemoryPoolingManagerTest, givenInitializedPoolsManagerWhenAllocatingGreaterThan2MBAndNotGreaterThan256BMThenBigPoolsAreUsed) { - void *ptr = reinterpret_cast(0x1u); - const void *constPtr = const_cast(ptr); - EXPECT_TRUE(usmMemAllocPoolsManager->ensureInitialized(svmManager.get())); - EXPECT_TRUE(usmMemAllocPoolsManager->isInitialized()); - EXPECT_TRUE(usmMemAllocPoolsManager->ensureInitialized(svmManager.get())); - - EXPECT_EQ(0u, usmMemAllocPoolsManager->pools[poolInfo2MbTo16Mb].size()); - EXPECT_EQ(0u, usmMemAllocPoolsManager->pools[poolInfo16MbTo64Mb].size()); - EXPECT_EQ(0u, usmMemAllocPoolsManager->pools[poolInfo64MbTo256Mb].size()); - poolMemoryProperties->alignment = UsmMemAllocPool::chunkAlignment; - const auto allocSize = 14 * MemoryConstants::megaByte; - auto normalAlloc = createAlloc(allocSize, *poolMemoryProperties.get()); - EXPECT_NE(nullptr, normalAlloc); - - size_t freeMemoryThreshold = static_cast((allocSize + usmMemAllocPoolsManager->totalSize) / UsmMemAllocPool::getPercentOfFreeMemoryForRecycling(poolMemoryType)); - const auto toleranceForFloatingPointArithmetic = static_cast(1 / UsmMemAllocPool::getPercentOfFreeMemoryForRecycling(poolMemoryType)); - usmMemAllocPoolsManager->mockFreeMemory = freeMemoryThreshold - toleranceForFloatingPointArithmetic; - EXPECT_FALSE(usmMemAllocPoolsManager->recycleSVMAlloc(normalAlloc, true)); - EXPECT_EQ(0u, usmMemAllocPoolsManager->pools[poolInfo2MbTo16Mb].size()); - - const auto totalSizeStart = usmMemAllocPoolsManager->totalSize; - usmMemAllocPoolsManager->mockFreeMemory = freeMemoryThreshold + toleranceForFloatingPointArithmetic; - EXPECT_TRUE(usmMemAllocPoolsManager->recycleSVMAlloc(normalAlloc, true)); - EXPECT_EQ(1u, usmMemAllocPoolsManager->pools[poolInfo2MbTo16Mb].size()); - EXPECT_EQ(totalSizeStart + allocSize, usmMemAllocPoolsManager->totalSize); - auto firstPool = reinterpret_cast(usmMemAllocPoolsManager->pools[poolInfo2MbTo16Mb][0].get()); - EXPECT_EQ(2 * MemoryConstants::megaByte + 1, firstPool->minServicedSize); - EXPECT_EQ(allocSize, firstPool->maxServicedSize); - - auto poolAlloc8MB = usmMemAllocPoolsManager->createUnifiedMemoryAllocation(8 * MemoryConstants::megaByte, *poolMemoryProperties.get()); - EXPECT_NE(nullptr, poolAlloc8MB); - EXPECT_TRUE(firstPool->isInPool(poolAlloc8MB)); - EXPECT_EQ(8 * MemoryConstants::megaByte, usmMemAllocPoolsManager->getPooledAllocationSize(poolAlloc8MB)); - EXPECT_EQ(poolAlloc8MB, usmMemAllocPoolsManager->getPooledAllocationBasePtr(poolAlloc8MB)); - EXPECT_EQ(firstPool->getOffsetInPool(poolAlloc8MB), usmMemAllocPoolsManager->getOffsetInPool(poolAlloc8MB)); - - auto allocationNotFitInPool = usmMemAllocPoolsManager->createUnifiedMemoryAllocation(8 * MemoryConstants::megaByte, *poolMemoryProperties.get()); - EXPECT_EQ(nullptr, allocationNotFitInPool); - - auto alloc64MB = createAlloc(64 * MemoryConstants::megaByte, *poolMemoryProperties.get()); - freeMemoryThreshold = static_cast((64 * MemoryConstants::megaByte + usmMemAllocPoolsManager->totalSize) / UsmMemAllocPool::getPercentOfFreeMemoryForRecycling(poolMemoryType)); - usmMemAllocPoolsManager->mockFreeMemory = freeMemoryThreshold + toleranceForFloatingPointArithmetic; - EXPECT_TRUE(usmMemAllocPoolsManager->recycleSVMAlloc(alloc64MB, true)); - auto poolAlloc64MB = usmMemAllocPoolsManager->createUnifiedMemoryAllocation(64 * MemoryConstants::megaByte, *poolMemoryProperties.get()); - EXPECT_NE(nullptr, poolAlloc64MB); - auto secondPool = reinterpret_cast(usmMemAllocPoolsManager->pools[poolInfo16MbTo64Mb][0].get()); - EXPECT_TRUE(secondPool->isInPool(poolAlloc64MB)); - EXPECT_EQ(64 * MemoryConstants::megaByte, usmMemAllocPoolsManager->getPooledAllocationSize(poolAlloc64MB)); - EXPECT_EQ(poolAlloc64MB, usmMemAllocPoolsManager->getPooledAllocationBasePtr(poolAlloc64MB)); - allocationNotFitInPool = usmMemAllocPoolsManager->createUnifiedMemoryAllocation(64 * MemoryConstants::megaByte, *poolMemoryProperties.get()); - EXPECT_EQ(nullptr, allocationNotFitInPool); - EXPECT_EQ(nullptr, usmMemAllocPoolsManager->getPoolContainingAlloc(constPtr)); - - auto alloc256MB = createAlloc(256 * MemoryConstants::megaByte, *poolMemoryProperties.get()); - freeMemoryThreshold = static_cast((256 * MemoryConstants::megaByte + usmMemAllocPoolsManager->totalSize) / UsmMemAllocPool::getPercentOfFreeMemoryForRecycling(poolMemoryType)); - usmMemAllocPoolsManager->mockFreeMemory = freeMemoryThreshold + toleranceForFloatingPointArithmetic; - EXPECT_TRUE(usmMemAllocPoolsManager->recycleSVMAlloc(alloc256MB, true)); - auto poolAlloc256MB = usmMemAllocPoolsManager->createUnifiedMemoryAllocation(256 * MemoryConstants::megaByte, *poolMemoryProperties.get()); - EXPECT_NE(nullptr, poolAlloc256MB); - auto thirdPool = reinterpret_cast(usmMemAllocPoolsManager->pools[poolInfo64MbTo256Mb][0].get()); - EXPECT_TRUE(thirdPool->isInPool(poolAlloc256MB)); - EXPECT_EQ(256 * MemoryConstants::megaByte, usmMemAllocPoolsManager->getPooledAllocationSize(poolAlloc256MB)); - EXPECT_EQ(poolAlloc256MB, usmMemAllocPoolsManager->getPooledAllocationBasePtr(poolAlloc256MB)); - allocationNotFitInPool = usmMemAllocPoolsManager->createUnifiedMemoryAllocation(256 * MemoryConstants::megaByte, *poolMemoryProperties.get()); - EXPECT_EQ(nullptr, allocationNotFitInPool); - - auto alloc256MBForTrim = createAlloc(256 * MemoryConstants::megaByte, *poolMemoryProperties.get()); - freeMemoryThreshold = static_cast((256 * MemoryConstants::megaByte + usmMemAllocPoolsManager->totalSize) / UsmMemAllocPool::getPercentOfFreeMemoryForRecycling(poolMemoryType)); - usmMemAllocPoolsManager->mockFreeMemory = freeMemoryThreshold + toleranceForFloatingPointArithmetic; - EXPECT_TRUE(usmMemAllocPoolsManager->recycleSVMAlloc(alloc256MBForTrim, true)); - EXPECT_EQ(1u, usmMemAllocPoolsManager->pools[poolInfo2MbTo16Mb].size()); - EXPECT_EQ(1u, usmMemAllocPoolsManager->pools[poolInfo16MbTo64Mb].size()); - EXPECT_EQ(2u, usmMemAllocPoolsManager->pools[poolInfo64MbTo256Mb].size()); - usmMemAllocPoolsManager->trim(); - EXPECT_EQ(1u, usmMemAllocPoolsManager->pools[poolInfo2MbTo16Mb].size()); - EXPECT_EQ(1u, usmMemAllocPoolsManager->pools[poolInfo16MbTo64Mb].size()); - EXPECT_EQ(1u, usmMemAllocPoolsManager->pools[poolInfo64MbTo256Mb].size()); - - auto smallAlloc = createAlloc(2 * MemoryConstants::megaByte - 1, *poolMemoryProperties.get()); - EXPECT_NE(nullptr, smallAlloc); - freeMemoryThreshold = static_cast((2 * MemoryConstants::megaByte + usmMemAllocPoolsManager->totalSize) / UsmMemAllocPool::getPercentOfFreeMemoryForRecycling(poolMemoryType)); - usmMemAllocPoolsManager->mockFreeMemory = freeMemoryThreshold + toleranceForFloatingPointArithmetic; - EXPECT_FALSE(usmMemAllocPoolsManager->recycleSVMAlloc(smallAlloc, true)); - - auto bigAlloc = createAlloc(256 * MemoryConstants::megaByte + 1, *poolMemoryProperties.get()); - EXPECT_NE(nullptr, bigAlloc); - freeMemoryThreshold = static_cast((256 * MemoryConstants::megaByte + 1 + usmMemAllocPoolsManager->totalSize) / UsmMemAllocPool::getPercentOfFreeMemoryForRecycling(poolMemoryType)); - usmMemAllocPoolsManager->mockFreeMemory = freeMemoryThreshold + toleranceForFloatingPointArithmetic; - EXPECT_FALSE(usmMemAllocPoolsManager->recycleSVMAlloc(bigAlloc, true)); - - svmManager->freeSVMAlloc(smallAlloc); - svmManager->freeSVMAlloc(bigAlloc); - - auto allocationOverMaxSize = usmMemAllocPoolsManager->createUnifiedMemoryAllocation(256 * MemoryConstants::megaByte + 1, *poolMemoryProperties.get()); - EXPECT_EQ(nullptr, allocationOverMaxSize); - - EXPECT_EQ(nullptr, usmMemAllocPoolsManager->getPoolContainingAlloc(constPtr)); - usmMemAllocPoolsManager->cleanup(); -} \ No newline at end of file