fix: prepare for l0 host usm pool

enable only if all devices support hostUsmPooling

split product helper method by api type for pre PTL products

add ult helper function to cleanup and initialize pools -
used when replacing memory managers or normal cleanup during driver
destruction would access already freed resources

adjust ults

Related-To: NEO-6893

Signed-off-by: Dominik Dabek <dominik.dabek@intel.com>
This commit is contained in:
Dominik Dabek
2025-06-13 14:34:19 +00:00
committed by Compute-Runtime-Automation
parent f165e713a5
commit e5477e614e
14 changed files with 126 additions and 11 deletions

View File

@@ -355,6 +355,10 @@ DriverHandle *DriverHandle::create(std::vector<std::unique_ptr<NEO::Device>> dev
void DriverHandleImp::initHostUsmAllocPool() {
auto usmHostAllocPoolingEnabled = NEO::ApiSpecificConfig::isHostUsmPoolingEnabled();
for (auto device : this->devices) {
usmHostAllocPoolingEnabled &= device->getNEODevice()->getProductHelper().isHostUsmPoolAllocatorSupported() &&
nullptr == device->getL0Debugger();
}
auto poolSize = 2 * MemoryConstants::megaByte;
if (NEO::debugManager.flags.EnableHostUsmAllocationPool.get() != -1) {
usmHostAllocPoolingEnabled = NEO::debugManager.flags.EnableHostUsmAllocationPool.get() > 0;

View File

@@ -0,0 +1,31 @@
/*
* Copyright (C) 2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "level_zero/core/source/device/device_imp.h"
#include "level_zero/core/source/driver/driver_handle_imp.h"
namespace L0 {
namespace ult {
struct L0UltHelper {
static void cleanupUsmAllocPools(DriverHandleImp *driverHandle) {
driverHandle->usmHostMemAllocPool.cleanup();
for (auto device : driverHandle->devices) {
device->getNEODevice()->cleanupUsmAllocationPool();
}
}
static void initUsmAllocPools(DriverHandleImp *driverHandle) {
driverHandle->initHostUsmAllocPool();
for (auto device : driverHandle->devices) {
driverHandle->initDeviceUsmAllocPool(*device->getNEODevice());
}
}
};
} // namespace ult
} // namespace L0

View File

@@ -60,6 +60,7 @@ template <typename T>
struct DeviceFixtureWithCustomMemoryManager : public DeviceFixture {
void setUp() {
debugManager.flags.EnableDeviceUsmAllocationPool.set(0);
debugManager.flags.EnableHostUsmAllocationPool.set(0);
auto executionEnvironment = NEO::MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u);
memoryManager = new T(*executionEnvironment);
executionEnvironment->memoryManager.reset(memoryManager);

View File

@@ -22,13 +22,13 @@
#include "shared/test/common/mocks/ult_device_factory.h"
#include "shared/test/common/test_macros/hw_test.h"
#include "level_zero/core/test/common/ult_helpers_l0.h"
#include "level_zero/core/test/unit_tests/fixtures/module_fixture.h"
#include "level_zero/core/test/unit_tests/mocks/mock_cmdlist.h"
#include "level_zero/core/test/unit_tests/mocks/mock_cmdqueue.h"
#include "level_zero/core/test/unit_tests/mocks/mock_cmdqueue_handle_indirect_allocs.h"
#include "level_zero/core/test/unit_tests/mocks/mock_memory_manager.h"
#include "level_zero/core/test/unit_tests/mocks/mock_module.h"
namespace L0 {
namespace ult {
struct MockMemoryManagerCommandQueueSBA : public MemoryManagerMock {
@@ -527,6 +527,7 @@ HWTEST_F(CommandQueueIndirectAllocations, givenDeviceThatSupportsSubmittingIndir
device->getDriverHandle()->getSvmAllocsManager()->freeSVMAlloc(deviceAlloc);
commandQueue->destroy();
L0UltHelper::cleanupUsmAllocPools(driverHandle.get());
}
HWTEST_F(CommandQueueIndirectAllocations, givenDeviceThatSupportsSubmittingIndirectAllocationsAsPackWhenIndirectAccessIsUsedThenWholePackIsMadeResidentWithImmediateCommandListAndFlushTask) {

View File

@@ -26,6 +26,7 @@
#include "level_zero/core/source/context/context_imp.h"
#include "level_zero/core/source/driver/driver_handle_imp.h"
#include "level_zero/core/source/image/image.h"
#include "level_zero/core/test/common/ult_helpers_l0.h"
#include "level_zero/core/test/unit_tests/fixtures/device_fixture.h"
#include "level_zero/core/test/unit_tests/fixtures/host_pointer_manager_fixture.h"
#include "level_zero/core/test/unit_tests/mocks/mock_cmdlist.h"
@@ -393,9 +394,11 @@ struct ContextHostAllocTests : public ::testing::Test {
ze_result_t res = driverHandle->initialize(std::move(devices));
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
L0UltHelper::cleanupUsmAllocPools(driverHandle.get());
prevSvmAllocsManager = driverHandle->svmAllocsManager;
currSvmAllocsManager = new SVMAllocsManagerContextMock(driverHandle->memoryManager);
driverHandle->svmAllocsManager = currSvmAllocsManager;
L0UltHelper::initUsmAllocPools(driverHandle.get());
zeDevices.resize(numberOfDevicesInContext);
driverHandle->getDevice(&numberOfDevicesInContext, zeDevices.data());
@@ -407,6 +410,7 @@ struct ContextHostAllocTests : public ::testing::Test {
}
void TearDown() override {
L0UltHelper::cleanupUsmAllocPools(driverHandle.get());
driverHandle->svmAllocsManager = prevSvmAllocsManager;
delete currSvmAllocsManager;
}

View File

@@ -35,6 +35,7 @@ struct L0DebuggerWindowsFixture {
void setUp() {
debugManager.flags.ForcePreferredAllocationMethod.set(static_cast<int32_t>(GfxMemoryAllocationMethod::useUmdSystemPtr));
debugManager.flags.EnableDeviceUsmAllocationPool.set(0);
debugManager.flags.EnableHostUsmAllocationPool.set(0);
executionEnvironment = new NEO::ExecutionEnvironment;
executionEnvironment->setDebuggingMode(NEO::DebuggingMode::online);
executionEnvironment->prepareRootDeviceEnvironments(1);

View File

@@ -34,6 +34,7 @@
#include "level_zero/core/source/driver/driver_handle_imp.h"
#include "level_zero/core/source/driver/driver_imp.h"
#include "level_zero/core/source/gfx_core_helpers/l0_gfx_core_helper.h"
#include "level_zero/core/test/common/ult_helpers_l0.h"
#include "level_zero/core/test/unit_tests/fixtures/device_fixture.h"
#include "level_zero/core/test/unit_tests/fixtures/host_pointer_manager_fixture.h"
#include "level_zero/core/test/unit_tests/mocks/mock_builtin_functions_lib_impl.h"
@@ -193,6 +194,7 @@ TEST_F(DriverVersionTest, givenExternalAllocatorWhenCallingGetExtensionPropertie
NEO::debugManager.flags.UseBindlessMode.set(1);
NEO::debugManager.flags.UseExternalAllocatorForSshAndDsh.set(1);
NEO::debugManager.flags.EnableDeviceUsmAllocationPool.set(0);
NEO::debugManager.flags.EnableHostUsmAllocationPool.set(0);
auto hwInfo = *NEO::defaultHwInfo;
NEO::MockDevice *neoDevice = NEO::MockDevice::createWithNewExecutionEnvironment<NEO::MockDevice>(&hwInfo);
@@ -411,9 +413,11 @@ HWTEST_F(ImportNTHandleWithMockMemoryManager, givenNTHandleWhenCreatingHostMemor
}
HWTEST_F(ImportNTHandleWithMockMemoryManager, whenCallingCreateGraphicsAllocationFromMultipleSharedHandlesFromOsAgnosticMemoryManagerThenNullptrIsReturned) {
L0UltHelper::cleanupUsmAllocPools(driverHandle.get());
delete driverHandle->svmAllocsManager;
driverHandle->setMemoryManager(execEnv->memoryManager.get());
driverHandle->svmAllocsManager = new NEO::SVMAllocsManager(execEnv->memoryManager.get());
L0UltHelper::initUsmAllocPools(driverHandle.get());
std::vector<osHandle> handles{6, 7};
AllocationProperties properties = {device->getRootDeviceIndex(),

View File

@@ -25,6 +25,7 @@
#include "level_zero/core/source/gfx_core_helpers/l0_gfx_core_helper.h"
#include "level_zero/core/source/image/image_format_desc_helper.h"
#include "level_zero/core/source/image/image_formats.h"
#include "level_zero/core/test/common/ult_helpers_l0.h"
#include "level_zero/core/test/unit_tests/fixtures/device_fixture.h"
#include "level_zero/core/test/unit_tests/mocks/mock_device.h"
#include "level_zero/core/test/unit_tests/mocks/mock_image.h"
@@ -697,9 +698,11 @@ HWTEST_F(ImageCreateExternalMemoryTest, givenNTHandleWhenCreatingImageThenSucces
importNTHandle.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMPORT_WIN32;
desc.pNext = &importNTHandle;
L0UltHelper::cleanupUsmAllocPools(driverHandle.get());
delete driverHandle->svmAllocsManager;
driverHandle->setMemoryManager(execEnv->memoryManager.get());
driverHandle->svmAllocsManager = new NEO::SVMAllocsManager(execEnv->memoryManager.get());
L0UltHelper::initUsmAllocPools(driverHandle.get());
auto imageHW = std::make_unique<WhiteBox<::L0::ImageCoreFamily<FamilyType::gfxCoreFamily>>>();
auto ret = imageHW->initialize(device, &desc);
@@ -716,9 +719,11 @@ HWTEST_F(ImageCreateExternalMemoryTest, givenD3D12HeapHandleWhenCreatingImageThe
importNTHandle.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMPORT_WIN32;
desc.pNext = &importNTHandle;
L0UltHelper::cleanupUsmAllocPools(driverHandle.get());
delete driverHandle->svmAllocsManager;
driverHandle->setMemoryManager(execEnv->memoryManager.get());
driverHandle->svmAllocsManager = new NEO::SVMAllocsManager(execEnv->memoryManager.get());
L0UltHelper::initUsmAllocPools(driverHandle.get());
auto imageHW = std::make_unique<WhiteBox<::L0::ImageCoreFamily<FamilyType::gfxCoreFamily>>>();
auto ret = imageHW->initialize(device, &desc);
@@ -735,9 +740,11 @@ HWTEST_F(ImageCreateExternalMemoryTest, givenD3D12ResourceHandleWhenCreatingImag
importNTHandle.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMPORT_WIN32;
desc.pNext = &importNTHandle;
L0UltHelper::cleanupUsmAllocPools(driverHandle.get());
delete driverHandle->svmAllocsManager;
driverHandle->setMemoryManager(execEnv->memoryManager.get());
driverHandle->svmAllocsManager = new NEO::SVMAllocsManager(execEnv->memoryManager.get());
L0UltHelper::initUsmAllocPools(driverHandle.get());
auto imageHW = std::make_unique<WhiteBox<::L0::ImageCoreFamily<FamilyType::gfxCoreFamily>>>();
auto ret = imageHW->initialize(device, &desc);
@@ -754,9 +761,11 @@ HWTEST_F(ImageCreateExternalMemoryTest, givenD3D11TextureHandleWhenCreatingImage
importNTHandle.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMPORT_WIN32;
desc.pNext = &importNTHandle;
L0UltHelper::cleanupUsmAllocPools(driverHandle.get());
delete driverHandle->svmAllocsManager;
driverHandle->setMemoryManager(execEnv->memoryManager.get());
driverHandle->svmAllocsManager = new NEO::SVMAllocsManager(execEnv->memoryManager.get());
L0UltHelper::initUsmAllocPools(driverHandle.get());
auto imageHW = std::make_unique<WhiteBox<::L0::ImageCoreFamily<FamilyType::gfxCoreFamily>>>();
auto ret = imageHW->initialize(device, &desc);
@@ -793,9 +802,11 @@ HWTEST_F(ImageCreateWithMemoryManagerNTHandleMock, givenNTHandleWhenCreatingNV12
importNTHandle.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMPORT_WIN32;
desc.pNext = &importNTHandle;
L0UltHelper::cleanupUsmAllocPools(driverHandle.get());
delete driverHandle->svmAllocsManager;
driverHandle->setMemoryManager(execEnv->memoryManager.get());
driverHandle->svmAllocsManager = new NEO::SVMAllocsManager(execEnv->memoryManager.get());
L0UltHelper::initUsmAllocPools(driverHandle.get());
auto imageHW = std::make_unique<WhiteBox<::L0::ImageCoreFamily<FamilyType::gfxCoreFamily>>>();
auto ret = imageHW->initialize(device, &desc);
@@ -843,9 +854,11 @@ HWTEST_F(ImageCreateWithFailMemoryManagerMock, givenImageDescWhenFailImageAlloca
backupSipInitType = true;
}
L0UltHelper::cleanupUsmAllocPools(driverHandle.get());
delete driverHandle->svmAllocsManager;
driverHandle->setMemoryManager(execEnv->memoryManager.get());
driverHandle->svmAllocsManager = new NEO::SVMAllocsManager(execEnv->memoryManager.get());
L0UltHelper::initUsmAllocPools(driverHandle.get());
L0::Image *imageHandle = nullptr;
static_cast<FailMemoryManagerMock *>(execEnv->memoryManager.get())->fail = true;

View File

@@ -33,6 +33,7 @@
#include "level_zero/core/source/image/image_hw.h"
#include "level_zero/core/source/memory/memory_operations_helper.h"
#include "level_zero/core/source/module/module.h"
#include "level_zero/core/test/common/ult_helpers_l0.h"
#include "level_zero/core/test/unit_tests/fixtures/device_fixture.h"
#include "level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h"
#include "level_zero/core/test/unit_tests/mocks/mock_built_ins.h"
@@ -40,7 +41,6 @@
#include "level_zero/core/test/unit_tests/mocks/mock_context.h"
#include "level_zero/core/test/unit_tests/mocks/mock_kernel.h"
#include "level_zero/driver_experimental/zex_memory.h"
namespace L0 {
namespace ult {
@@ -495,14 +495,20 @@ TEST_F(MemoryTest, givenDevicePointerThenDriverGetAllocPropertiesReturnsExpected
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
EXPECT_EQ(memoryProperties.type, ZE_MEMORY_TYPE_DEVICE);
EXPECT_EQ(deviceHandle, device->toHandle());
EXPECT_EQ(memoryProperties.id,
context->getDriverHandle()->getSvmAllocsManager()->allocationsCounter);
auto usmPool = device->getNEODevice()->getUsmMemAllocPool();
auto alloc = context->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr);
EXPECT_NE(alloc, nullptr);
EXPECT_NE(alloc->pageSizeForAlignment, 0u);
EXPECT_EQ(alloc->pageSizeForAlignment, memoryProperties.pageSize);
if (usmPool &&
usmPool->isInPool(ptr)) {
EXPECT_EQ(memoryProperties.id, alloc->getAllocId());
} else {
EXPECT_EQ(memoryProperties.id,
context->getDriverHandle()->getSvmAllocsManager()->allocationsCounter);
}
result = context->freeMem(ptr);
ASSERT_EQ(result, ZE_RESULT_SUCCESS);
}
@@ -525,14 +531,19 @@ TEST_F(MemoryTest, givenHostPointerThenDriverGetAllocPropertiesReturnsExpectedPr
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
EXPECT_EQ(memoryProperties.type, ZE_MEMORY_TYPE_HOST);
EXPECT_EQ(memoryProperties.id,
context->getDriverHandle()->getSvmAllocsManager()->allocationsCounter);
auto usmPool = &driverHandle->usmHostMemAllocPool;
auto alloc = context->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr);
EXPECT_NE(alloc, nullptr);
EXPECT_NE(alloc->pageSizeForAlignment, 0u);
EXPECT_EQ(alloc->pageSizeForAlignment, memoryProperties.pageSize);
if (usmPool->isInPool(ptr)) {
EXPECT_EQ(memoryProperties.id, alloc->getAllocId());
} else {
EXPECT_EQ(memoryProperties.id,
context->getDriverHandle()->getSvmAllocsManager()->allocationsCounter);
}
result = context->freeMem(ptr);
ASSERT_EQ(result, ZE_RESULT_SUCCESS);
}
@@ -2334,8 +2345,9 @@ struct ContextRelaxedSizeMock : public ContextImp {
struct MemoryRelaxedSizeTests : public ::testing::Test {
void SetUp() override {
// disable usm device pooling, used svm manager mock does not make svmData
// disable usm pooling, used svm manager mock does not create svmData
debugManager.flags.EnableDeviceUsmAllocationPool.set(0);
debugManager.flags.EnableHostUsmAllocationPool.set(0);
neoDevice =
NEO::MockDevice::createWithNewExecutionEnvironment<NEO::MockDevice>(NEO::defaultHwInfo.get());
auto mockBuiltIns = new MockBuiltins();
@@ -5125,6 +5137,7 @@ struct AllocHostMemoryTest : public ::testing::Test {
TEST_F(AllocHostMemoryTest,
whenCallingAllocHostMemThenAllocateGraphicsMemoryWithPropertiesIsCalledTheNumberOfTimesOfRootDevices) {
driverHandle->usmHostMemAllocPool.cleanup();
void *ptr = nullptr;
static_cast<MockMemoryManager *>(driverHandle->getMemoryManager())->isMockHostMemoryManager = true;
@@ -5142,7 +5155,7 @@ TEST_F(AllocHostMemoryTest,
TEST_F(AllocHostMemoryTest,
whenCallingAllocHostMemAndFailingOnCreatingGraphicsAllocationThenNullIsReturned) {
L0UltHelper::cleanupUsmAllocPools(driverHandle.get());
static_cast<MockMemoryManager *>(driverHandle->getMemoryManager())->isMockHostMemoryManager = true;
static_cast<MockMemoryManager *>(driverHandle->getMemoryManager())->forceFailureInPrimaryAllocation = true;
@@ -5160,7 +5173,7 @@ TEST_F(AllocHostMemoryTest,
TEST_F(AllocHostMemoryTest,
whenCallingAllocHostMemAndFailingOnCreatingGraphicsAllocationWithHostPointerThenNullIsReturned) {
driverHandle->usmHostMemAllocPool.cleanup();
static_cast<MockMemoryManager *>(driverHandle->getMemoryManager())->isMockHostMemoryManager = true;
static_cast<MockMemoryManager *>(driverHandle->getMemoryManager())->forceFailureInAllocationWithHostPointer = true;

View File

@@ -58,6 +58,11 @@ bool ProductHelperHw<gfxProduct>::isMisalignedUserPtr2WayCoherent() const {
return true;
}
template <>
bool ProductHelperHw<gfxProduct>::isHostUsmPoolAllocatorSupported() const {
return ApiSpecificConfig::OCL == ApiSpecificConfig::getApiType();
}
template <>
bool ProductHelperHw<gfxProduct>::isDeviceUsmPoolAllocatorSupported() const {
return ApiSpecificConfig::OCL == ApiSpecificConfig::getApiType();

View File

@@ -91,6 +91,11 @@ bool ProductHelperHw<gfxProduct>::isTile64With3DSurfaceOnBCSSupported(const Hard
return false;
}
template <>
bool ProductHelperHw<gfxProduct>::isHostUsmPoolAllocatorSupported() const {
return ApiSpecificConfig::OCL == ApiSpecificConfig::getApiType();
}
template <>
bool ProductHelperHw<gfxProduct>::isDeviceUsmPoolAllocatorSupported() const {
return ApiSpecificConfig::OCL == ApiSpecificConfig::getApiType();

View File

@@ -140,6 +140,17 @@ LNLTEST_F(LnlProductHelper, givenProductHelperWhenCheckingIsBufferPoolAllocatorS
EXPECT_TRUE(productHelper->isBufferPoolAllocatorSupported());
}
LNLTEST_F(LnlProductHelper, givenProductHelperWhenCheckingIsHostUsmPoolAllocatorSupportedThenCorrectValueIsReturned) {
{
VariableBackup<ApiSpecificConfig::ApiType> backup(&apiTypeForUlts, ApiSpecificConfig::OCL);
EXPECT_TRUE(productHelper->isHostUsmPoolAllocatorSupported());
}
{
VariableBackup<ApiSpecificConfig::ApiType> backup(&apiTypeForUlts, ApiSpecificConfig::L0);
EXPECT_FALSE(productHelper->isHostUsmPoolAllocatorSupported());
}
}
LNLTEST_F(LnlProductHelper, givenProductHelperWhenCheckingIsDeviceUsmPoolAllocatorSupportedThenCorrectValueIsReturned) {
{
VariableBackup<ApiSpecificConfig::ApiType> backup(&apiTypeForUlts, ApiSpecificConfig::OCL);

View File

@@ -124,6 +124,17 @@ PTLTEST_F(PtlProductHelper, givenResolveDependenciesByPipeControllsNotSupportedW
EXPECT_FALSE(productHelper->isResolveDependenciesByPipeControlsSupported(pInHwInfo, false, 2, csrRelaxed));
}
PTLTEST_F(PtlProductHelper, givenProductHelperWhenCheckingIsHostUsmPoolAllocatorSupportedThenCorrectValueIsReturned) {
{
VariableBackup<ApiSpecificConfig::ApiType> backup(&apiTypeForUlts, ApiSpecificConfig::OCL);
EXPECT_TRUE(productHelper->isDeviceUsmPoolAllocatorSupported());
}
{
VariableBackup<ApiSpecificConfig::ApiType> backup(&apiTypeForUlts, ApiSpecificConfig::L0);
EXPECT_TRUE(productHelper->isDeviceUsmPoolAllocatorSupported());
}
}
PTLTEST_F(PtlProductHelper, givenProductHelperWhenCheckingIsDeviceUsmPoolAllocatorSupportedThenCorrectValueIsReturned) {
{
VariableBackup<ApiSpecificConfig::ApiType> backup(&apiTypeForUlts, ApiSpecificConfig::OCL);

View File

@@ -87,6 +87,17 @@ MTLTEST_F(MtlProductHelper, givenProductHelperWhenCheckoverrideAllocationCpuCach
EXPECT_FALSE(productHelper->overrideAllocationCpuCacheable(allocationData));
}
MTLTEST_F(MtlProductHelper, givenProductHelperWhenCheckingIsHostUsmPoolAllocatorSupportedThenCorrectValueIsReturned) {
{
VariableBackup<ApiSpecificConfig::ApiType> backup(&apiTypeForUlts, ApiSpecificConfig::OCL);
EXPECT_TRUE(productHelper->isHostUsmPoolAllocatorSupported());
}
{
VariableBackup<ApiSpecificConfig::ApiType> backup(&apiTypeForUlts, ApiSpecificConfig::L0);
EXPECT_FALSE(productHelper->isHostUsmPoolAllocatorSupported());
}
}
MTLTEST_F(MtlProductHelper, givenProductHelperWhenCheckingIsDeviceUsmPoolAllocatorSupportedThenCorrectValueIsReturned) {
{
VariableBackup<ApiSpecificConfig::ApiType> backup(&apiTypeForUlts, ApiSpecificConfig::OCL);