diff --git a/level_zero/core/source/driver/driver_handle_imp.cpp b/level_zero/core/source/driver/driver_handle_imp.cpp index 4f2b7ddd82..16ace49340 100644 --- a/level_zero/core/source/driver/driver_handle_imp.cpp +++ b/level_zero/core/source/driver/driver_handle_imp.cpp @@ -307,9 +307,6 @@ ze_result_t DriverHandleImp::initialize(std::vector this->initHostUsmAllocPool(); for (auto &device : this->devices) { this->initDeviceUsmAllocPool(*device->getNEODevice()); - if (auto deviceUsmAllocPool = device->getNEODevice()->getUsmMemAllocPool()) { - deviceUsmAllocPool->ensureInitialized(this->svmAllocsManager); - } if (auto deviceUsmAllocPoolsManager = device->getNEODevice()->getUsmMemAllocPoolsManager()) { deviceUsmAllocPoolsManager->ensureInitialized(this->svmAllocsManager); } @@ -387,8 +384,17 @@ void DriverHandleImp::initDeviceUsmAllocPool(NEO::Device &device) { } if (enabled) { - device.resetUsmAllocationPool(new NEO::UsmMemAllocPool(rootDeviceIndices, deviceBitfields, &device, InternalMemoryType::deviceUnifiedMemory, - poolSize, minServicedSize, maxServicedSize)); + device.resetUsmAllocationPool(new NEO::UsmMemAllocPool); + auto &hwInfo = device.getHardwareInfo(); + auto &l0GfxCoreHelper = device.getRootDeviceEnvironment().getHelper(); + const bool compressionEnabledByDefault = l0GfxCoreHelper.usmCompressionSupported(hwInfo) && l0GfxCoreHelper.forceDefaultUsmCompressionSupport(); + NEO::SVMAllocsManager::UnifiedMemoryProperties poolMemoryProperties(InternalMemoryType::deviceUnifiedMemory, + MemoryConstants::pageSize2M, + rootDeviceIndices, + deviceBitfields); + poolMemoryProperties.device = &device; + poolMemoryProperties.allocationFlags.flags.compressedHint = compressionEnabledByDefault; + device.getUsmMemAllocPool()->initialize(this->svmAllocsManager, poolMemoryProperties, poolSize, minServicedSize, maxServicedSize); } } diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp index 21d94d7557..8ae293abc4 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory_pooling.cpp @@ -19,9 +19,9 @@ #include "level_zero/core/source/context/context_imp.h" #include "level_zero/core/source/device/device_imp.h" #include "level_zero/core/source/driver/driver_handle_imp.h" +#include "level_zero/core/source/gfx_core_helpers/l0_gfx_core_helper.h" #include "level_zero/core/test/unit_tests/mock.h" #include "level_zero/core/test/unit_tests/mocks/mock_driver_handle.h" - namespace L0 { namespace ult { template @@ -246,70 +246,134 @@ TEST_F(AllocUsmDeviceEnabledMemoryTest, givenDeviceWhenCallingAllocDeviceMemWith auto mockDeviceMemAllocPool = reinterpret_cast(l0Devices[0]->getNEODevice()->getUsmMemAllocPool()); ASSERT_NE(nullptr, mockDeviceMemAllocPool); EXPECT_TRUE(mockDeviceMemAllocPool->isInitialized()); - - void *ptr1Byte = nullptr; - ze_device_mem_alloc_desc_t deviceDesc = {}; - ze_result_t result = context->allocDeviceMem(l0Devices[0], &deviceDesc, 1u, 0u, &ptr1Byte); - EXPECT_TRUE(mockDeviceMemAllocPool->isInitialized()); auto poolAllocationData = driverHandle->svmAllocsManager->getSVMAlloc(mockDeviceMemAllocPool->pool); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_NE(nullptr, ptr1Byte); - EXPECT_TRUE(mockDeviceMemAllocPool->isInPool(ptr1Byte)); - EXPECT_EQ(1u, mockDeviceMemAllocPool->allocations.getNumAllocs()); - EXPECT_EQ(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptr1Byte)); - result = context->freeMem(ptr1Byte); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + ASSERT_NE(nullptr, poolAllocationData); + const bool poolIsCompressed = poolAllocationData->gpuAllocations.getDefaultGraphicsAllocation()->isCompressionEnabled(); - void *ptrThreshold = nullptr; - result = context->allocDeviceMem(l0Devices[0], &deviceDesc, poolAllocationThreshold, 0u, &ptrThreshold); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_NE(nullptr, ptrThreshold); - EXPECT_TRUE(mockDeviceMemAllocPool->isInPool(ptrThreshold)); - EXPECT_EQ(1u, mockDeviceMemAllocPool->allocations.getNumAllocs()); - EXPECT_EQ(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptrThreshold)); - result = context->freeMem(ptrThreshold); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + auto &hwInfo = l0Devices[0]->getHwInfo(); + auto &l0GfxCoreHelper = l0Devices[0]->getNEODevice()->getRootDeviceEnvironment().getHelper(); + if (l0GfxCoreHelper.usmCompressionSupported(hwInfo) && l0GfxCoreHelper.forceDefaultUsmCompressionSupport()) { + EXPECT_TRUE(poolIsCompressed); + } else { + EXPECT_FALSE(poolIsCompressed); + } - void *ptrOverThreshold = nullptr; - result = context->allocDeviceMem(l0Devices[0], &deviceDesc, poolAllocationThreshold + 1u, 0u, &ptrOverThreshold); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_NE(nullptr, ptrOverThreshold); - EXPECT_FALSE(mockDeviceMemAllocPool->isInPool(ptrOverThreshold)); - EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); - EXPECT_NE(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptrOverThreshold)); - result = context->freeMem(ptrOverThreshold); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + { + void *ptr1Byte = nullptr; + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_result_t result = context->allocDeviceMem(l0Devices[0], &deviceDesc, 1u, 0u, &ptr1Byte); + EXPECT_TRUE(mockDeviceMemAllocPool->isInitialized()); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptr1Byte); + EXPECT_TRUE(mockDeviceMemAllocPool->isInPool(ptr1Byte)); + EXPECT_EQ(1u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + EXPECT_EQ(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptr1Byte)); + result = context->freeMem(ptr1Byte); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + } - void *ptrFreeMemExt = nullptr; - result = context->allocDeviceMem(l0Devices[0], &deviceDesc, poolAllocationThreshold, 0u, &ptrFreeMemExt); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_NE(nullptr, ptrFreeMemExt); - EXPECT_TRUE(mockDeviceMemAllocPool->isInPool(ptrFreeMemExt)); - EXPECT_EQ(1u, mockDeviceMemAllocPool->allocations.getNumAllocs()); - EXPECT_EQ(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptrFreeMemExt)); - ze_memory_free_ext_desc_t memFreeDesc = {}; - memFreeDesc.freePolicy = ZE_DRIVER_MEMORY_FREE_POLICY_EXT_FLAG_DEFER_FREE; - result = context->freeMemExt(&memFreeDesc, ptrFreeMemExt); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + { + void *ptrCompressedHint = nullptr; + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_memory_compression_hints_ext_desc_t externalMemoryDesc{}; + externalMemoryDesc.stype = ZE_STRUCTURE_TYPE_MEMORY_COMPRESSION_HINTS_EXT_DESC; + externalMemoryDesc.flags = ZE_MEMORY_COMPRESSION_HINTS_EXT_FLAG_COMPRESSED; + deviceDesc.pNext = &externalMemoryDesc; + ze_result_t result = context->allocDeviceMem(l0Devices[0], &deviceDesc, 1u, 0u, &ptrCompressedHint); + EXPECT_TRUE(mockDeviceMemAllocPool->isInitialized()); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptrCompressedHint); + EXPECT_TRUE(mockDeviceMemAllocPool->isInPool(ptrCompressedHint)); + EXPECT_EQ(1u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + EXPECT_EQ(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptrCompressedHint)); + result = context->freeMem(ptrCompressedHint); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + } - void *ptrExportMemory = nullptr; - ze_external_memory_export_desc_t externalMemoryDesc{}; - externalMemoryDesc.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_DESC; - externalMemoryDesc.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_DMA_BUF; - deviceDesc.pNext = &externalMemoryDesc; - result = context->allocDeviceMem(l0Devices[0], &deviceDesc, poolAllocationThreshold, 0u, &ptrExportMemory); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_NE(nullptr, ptrExportMemory); - EXPECT_FALSE(mockDeviceMemAllocPool->isInPool(ptrExportMemory)); - EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); - EXPECT_NE(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptrExportMemory)); - result = context->freeMemExt(&memFreeDesc, ptrExportMemory); - EXPECT_EQ(ZE_RESULT_SUCCESS, result); - EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + { + void *ptrCompressedHint = nullptr; + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_memory_compression_hints_ext_desc_t externalMemoryDesc{}; + externalMemoryDesc.stype = ZE_STRUCTURE_TYPE_MEMORY_COMPRESSION_HINTS_EXT_DESC; + externalMemoryDesc.flags = ZE_MEMORY_COMPRESSION_HINTS_EXT_FLAG_UNCOMPRESSED; + deviceDesc.pNext = &externalMemoryDesc; + ze_result_t result = context->allocDeviceMem(l0Devices[0], &deviceDesc, 1u, 0u, &ptrCompressedHint); + EXPECT_TRUE(mockDeviceMemAllocPool->isInitialized()); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptrCompressedHint); + EXPECT_TRUE(mockDeviceMemAllocPool->isInPool(ptrCompressedHint)); + EXPECT_EQ(1u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + EXPECT_EQ(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptrCompressedHint)); + result = context->freeMem(ptrCompressedHint); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + } + + { + void *ptrThreshold = nullptr; + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_result_t result = context->allocDeviceMem(l0Devices[0], &deviceDesc, poolAllocationThreshold, 0u, &ptrThreshold); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptrThreshold); + EXPECT_TRUE(mockDeviceMemAllocPool->isInPool(ptrThreshold)); + EXPECT_EQ(1u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + EXPECT_EQ(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptrThreshold)); + result = context->freeMem(ptrThreshold); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + } + + { + void *ptrOverThreshold = nullptr; + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_result_t result = context->allocDeviceMem(l0Devices[0], &deviceDesc, poolAllocationThreshold + 1u, 0u, &ptrOverThreshold); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptrOverThreshold); + EXPECT_FALSE(mockDeviceMemAllocPool->isInPool(ptrOverThreshold)); + EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + EXPECT_NE(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptrOverThreshold)); + result = context->freeMem(ptrOverThreshold); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + } + + { + void *ptrFreeMemExt = nullptr; + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_result_t result = context->allocDeviceMem(l0Devices[0], &deviceDesc, poolAllocationThreshold, 0u, &ptrFreeMemExt); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptrFreeMemExt); + EXPECT_TRUE(mockDeviceMemAllocPool->isInPool(ptrFreeMemExt)); + EXPECT_EQ(1u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + EXPECT_EQ(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptrFreeMemExt)); + ze_memory_free_ext_desc_t memFreeDesc = {}; + memFreeDesc.freePolicy = ZE_DRIVER_MEMORY_FREE_POLICY_EXT_FLAG_DEFER_FREE; + result = context->freeMemExt(&memFreeDesc, ptrFreeMemExt); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + } + { + + void *ptrExportMemory = nullptr; + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_external_memory_export_desc_t externalMemoryDesc{}; + externalMemoryDesc.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_DESC; + externalMemoryDesc.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_DMA_BUF; + deviceDesc.pNext = &externalMemoryDesc; + ze_result_t result = context->allocDeviceMem(l0Devices[0], &deviceDesc, poolAllocationThreshold, 0u, &ptrExportMemory); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptrExportMemory); + EXPECT_FALSE(mockDeviceMemAllocPool->isInPool(ptrExportMemory)); + EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + EXPECT_NE(poolAllocationData, driverHandle->svmAllocsManager->getSVMAlloc(ptrExportMemory)); + ze_memory_free_ext_desc_t memFreeDesc = {}; + memFreeDesc.freePolicy = ZE_DRIVER_MEMORY_FREE_POLICY_EXT_FLAG_DEFER_FREE; + result = context->freeMemExt(&memFreeDesc, ptrExportMemory); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_EQ(0u, mockDeviceMemAllocPool->allocations.getNumAllocs()); + } } TEST_F(AllocUsmDeviceEnabledMemoryTest, givenDrmDriverModelWhenOpeningIpcHandleFromPooledAllocationThenOffsetIsApplied) { diff --git a/shared/source/memory_manager/unified_memory_pooling.cpp b/shared/source/memory_manager/unified_memory_pooling.cpp index 18548096aa..03bb0e1486 100644 --- a/shared/source/memory_manager/unified_memory_pooling.cpp +++ b/shared/source/memory_manager/unified_memory_pooling.cpp @@ -42,20 +42,6 @@ bool UsmMemAllocPool::initialize(SVMAllocsManager *svmMemoryManager, void *ptr, return true; } -bool UsmMemAllocPool::ensureInitialized(SVMAllocsManager *svmMemoryManager) { - if (isInitialized()) { - return true; - } - std::unique_lock lock(mtx); - if (isInitialized()) { - return true; - } - DEBUG_BREAK_IF(0u == poolSize || 0u == maxServicedSize || 0u == rootDeviceIndices.size() || deviceBitFields.empty()); - SVMAllocsManager::UnifiedMemoryProperties poolMemoryProperties(poolMemoryType, poolAlignment, rootDeviceIndices, deviceBitFields); - poolMemoryProperties.device = device; - return initialize(svmMemoryManager, poolMemoryProperties, poolSize, minServicedSize, maxServicedSize); -} - bool UsmMemAllocPool::isInitialized() const { return this->pool; } @@ -85,7 +71,11 @@ bool UsmMemAllocPool::sizeIsAllowed(size_t size) { } bool UsmMemAllocPool::flagsAreAllowed(const UnifiedMemoryProperties &memoryProperties) { - return memoryProperties.allocationFlags.allFlags == 0u && + auto flagsWithoutCompression = memoryProperties.allocationFlags; + flagsWithoutCompression.flags.compressedHint = 0u; + flagsWithoutCompression.flags.uncompressedHint = 0u; + + return flagsWithoutCompression.allFlags == 0u && memoryProperties.allocationFlags.allAllocFlags == 0u; } diff --git a/shared/source/memory_manager/unified_memory_pooling.h b/shared/source/memory_manager/unified_memory_pooling.h index 730c1c0b6e..fc6ce9b6d6 100644 --- a/shared/source/memory_manager/unified_memory_pooling.h +++ b/shared/source/memory_manager/unified_memory_pooling.h @@ -26,23 +26,9 @@ class UsmMemAllocPool { using AllocationsInfoStorage = BaseSortedPointerWithValueVector; UsmMemAllocPool() = default; - UsmMemAllocPool(const RootDeviceIndicesContainer &rootDeviceIndices, - const std::map &deviceBitFields, - Device *device, - InternalMemoryType poolMemoryType, - size_t poolSize, - size_t minServicedSize, - size_t maxServicedSize) : rootDeviceIndices(rootDeviceIndices), - deviceBitFields(deviceBitFields), - device(device), - poolMemoryType(poolMemoryType), - poolSize(poolSize), - minServicedSize(minServicedSize), - maxServicedSize(maxServicedSize){}; virtual ~UsmMemAllocPool() = default; bool initialize(SVMAllocsManager *svmMemoryManager, const UnifiedMemoryProperties &memoryProperties, size_t poolSize, size_t minServicedSize, size_t maxServicedSize); bool initialize(SVMAllocsManager *svmMemoryManager, void *ptr, SvmAllocationData *svmData, size_t minServicedSize, size_t maxServicedSize); - bool ensureInitialized(SVMAllocsManager *svmMemoryManager); bool isInitialized() const; size_t getPoolSize() const; MOCKABLE_VIRTUAL void cleanup(); diff --git a/shared/test/unit_test/memory_manager/unified_memory_pooling_tests.cpp b/shared/test/unit_test/memory_manager/unified_memory_pooling_tests.cpp index 587972508b..e0745cf7dc 100644 --- a/shared/test/unit_test/memory_manager/unified_memory_pooling_tests.cpp +++ b/shared/test/unit_test/memory_manager/unified_memory_pooling_tests.cpp @@ -45,6 +45,7 @@ TEST_F(UnifiedMemoryPoolingStaticTest, givenUsmAllocPoolWhenCallingStaticMethods } using UnifiedMemoryPoolingTest = Test>; + TEST_F(UnifiedMemoryPoolingTest, givenUsmAllocPoolWhenCallingIsInitializedThenReturnCorrectValue) { UsmMemAllocPool usmMemAllocPool; EXPECT_FALSE(usmMemAllocPool.isInitialized()); @@ -63,25 +64,6 @@ TEST_F(UnifiedMemoryPoolingTest, givenUsmAllocPoolWhenCallingIsInitializedThenRe EXPECT_FALSE(usmMemAllocPool.freeSVMAlloc(reinterpret_cast(0x1), true)); } -TEST_F(UnifiedMemoryPoolingTest, givenUsmAllocPoolWhenCallingEnsureInitializedThenReturnCorrectValue) { - std::unique_ptr deviceFactory(new UltDeviceFactory(1, 1)); - auto device = deviceFactory->rootDevices[0]; - auto svmManager = std::make_unique(device->getMemoryManager(), false); - - UsmMemAllocPool usmMemAllocPool(rootDeviceIndices, deviceBitfields, device, InternalMemoryType::deviceUnifiedMemory, 1 * MemoryConstants::megaByte, 0u, 1 * MemoryConstants::megaByte); - EXPECT_FALSE(usmMemAllocPool.isInitialized()); - - EXPECT_TRUE(usmMemAllocPool.ensureInitialized(svmManager.get())); - - EXPECT_TRUE(usmMemAllocPool.isInitialized()); - - EXPECT_TRUE(usmMemAllocPool.ensureInitialized(svmManager.get())); - - usmMemAllocPool.cleanup(); - EXPECT_FALSE(usmMemAllocPool.isInitialized()); - EXPECT_FALSE(usmMemAllocPool.freeSVMAlloc(reinterpret_cast(0x1), true)); -} - template class InitializedUnifiedMemoryPoolingTest : public UnifiedMemoryPoolingTest { public: