Media
/
compute-runtime
mirror of https://github.com/intel/compute-runtime.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feature: add NEO_LOCAL_MEMORY_ALLOCATION_MODE 

Allow the application to force storageInfo.localOnly and get the
out-of-memory returned if not possible.

This is a windows-only feature supported on discrete platforms.

Related-To: NEO-13428
Signed-off-by: Maciej Bielski <maciej.bielski@intel.com>
This commit is contained in:
Maciej Bielski 2025-04-01 18:09:02 +00:00 committed by Compute-Runtime-Automation
parent 0f19182214
commit 0f8ee57f98
31 changed files with 363 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
level_zero/core/source/context/context_imp.cpp
View File

@ -305,16 +305,28 @@ ze_result_t ContextImp::allocDeviceMem(ze_device_handle_t hDevice,
this->driverHandle->svmAllocsManager->freeSVMAllocDeferImpl();
usmPtr =
this->driverHandle->svmAllocsManager->createUnifiedMemoryAllocation(size, unifiedMemoryProperties);
if (usmPtr) {
*ptr = usmPtr;
return ZE_RESULT_SUCCESS;
}
}
}
if (usmPtr == nullptr) {
return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY;
}
*ptr = usmPtr;
return ZE_RESULT_SUCCESS;
ze_result_t ret{ZE_RESULT_SUCCESS};
if (this->driverHandle->getMemoryManager()->isLocalOnlyAllocationMode()) {
auto *allocData{this->driverHandle->svmAllocsManager->getSVMAlloc(usmPtr)};
DEBUG_BREAK_IF(allocData == nullptr);
auto *gpuAllocation{allocData->gpuAllocations.getDefaultGraphicsAllocation()};
DEBUG_BREAK_IF(gpuAllocation == nullptr);
if (allocData->memoryType == InternalMemoryType::deviceUnifiedMemory && gpuAllocation->storageInfo.localOnlyRequired) {
ret = this->makeMemoryResident(hDevice, usmPtr, size);
}
}
if (ret == ZE_RESULT_SUCCESS) {
*ptr = usmPtr;
}
return ret;
}
ze_result_t ContextImp::allocSharedMem(ze_device_handle_t hDevice,

70
level_zero/core/test/unit_tests/sources/context/test_context.cpp
View File

@ -700,6 +700,76 @@ TEST_F(ContextMakeMemoryResidentTests,
context->freeMem(ptr);
}
TEST_F(ContextMakeMemoryResidentTests, givenDeviceUnifiedMemoryAndLocalOnlyAllocationModeThenCallMakeMemoryResidentImmediately) {
const size_t size = 4096;
void *ptr = nullptr;
ze_device_mem_alloc_desc_t deviceDesc = {};
auto *driverHandleImp{static_cast<DriverHandleImp *>(hostDriverHandle.get())};
driverHandleImp->memoryManager->usmDeviceAllocationMode = NEO::LocalMemAllocationMode::localOnly;
static_cast<MockMemoryManager *>(driverHandleImp->memoryManager)->returnFakeAllocation = true;
EXPECT_EQ(0U, mockMemoryInterface->makeResidentCalled);
mockMemoryInterface->makeResidentResult = NEO::MemoryOperationsStatus::success;
ze_result_t res1 = context->allocDeviceMem(device->toHandle(),
&deviceDesc,
size,
0,
&ptr);
EXPECT_EQ(ZE_RESULT_SUCCESS, res1);
auto allocData{driverHandleImp->svmAllocsManager->getSVMAlloc(ptr)};
EXPECT_NE(allocData, nullptr);
const bool lmemAllocationModeSupported{allocData->gpuAllocations.getDefaultGraphicsAllocation()->storageInfo.localOnlyRequired};
EXPECT_EQ(mockMemoryInterface->makeResidentCalled, (lmemAllocationModeSupported ? 1U : 0U));
EXPECT_EQ(ZE_RESULT_SUCCESS, context->freeMem(ptr));
mockMemoryInterface->makeResidentResult = NEO::MemoryOperationsStatus::outOfMemory;
ze_result_t res2 = context->allocDeviceMem(device->toHandle(),
&deviceDesc,
size,
0,
&ptr);
EXPECT_EQ(res2, (lmemAllocationModeSupported ? ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY : ZE_RESULT_SUCCESS));
EXPECT_EQ(mockMemoryInterface->makeResidentCalled, (lmemAllocationModeSupported ? 2U : 0U));
EXPECT_EQ(ZE_RESULT_SUCCESS, context->freeMem(ptr));
}
TEST_F(ContextMakeMemoryResidentTests, givenNonDeviceUnifiedMemoryWhenAllocDeviceMemCalledThenMakeMemoryResidentIsNotImmediatelyCalled) {
const size_t size = 4096;
void *ptr = nullptr;
ze_device_mem_alloc_desc_t deviceDesc = {};
auto *driverHandleImp{static_cast<DriverHandleImp *>(hostDriverHandle.get())};
driverHandleImp->memoryManager->usmDeviceAllocationMode = NEO::LocalMemAllocationMode::localOnly;
static_cast<MockMemoryManager *>(driverHandleImp->memoryManager)->returnFakeAllocation = true;
auto *origSvmAllocsManager{driverHandleImp->svmAllocsManager};
auto fakeAllocationAddr{reinterpret_cast<void *>(0x1234)};
MockGraphicsAllocation mockUnifiedAllocation{};
SvmAllocationData allocData(0U);
allocData.gpuAllocations.addAllocation(&mockUnifiedAllocation);
allocData.memoryType = InternalMemoryType::notSpecified;
MockSVMAllocsManager mockSvmAllocsManager{driverHandleImp->memoryManager};
mockSvmAllocsManager.createUnifiedMemoryAllocationCallBase = false;
mockSvmAllocsManager.createUnifiedMemoryAllocationReturnValue = fakeAllocationAddr;
mockSvmAllocsManager.insertSVMAlloc(fakeAllocationAddr, allocData);
driverHandleImp->svmAllocsManager = &mockSvmAllocsManager;
EXPECT_EQ(0U, mockMemoryInterface->makeResidentCalled);
mockMemoryInterface->makeResidentResult = NEO::MemoryOperationsStatus::success;
ze_result_t res1 = context->allocDeviceMem(device->toHandle(),
&deviceDesc,
size,
0,
&ptr);
EXPECT_EQ(ZE_RESULT_SUCCESS, res1);
EXPECT_EQ(mockMemoryInterface->makeResidentCalled, 0U);
driverHandleImp->svmAllocsManager = origSvmAllocsManager;
}
struct ContextMakeMemoryResidentAndMigrationTests : public ContextMakeMemoryResidentTests {
struct MockResidentTestsPageFaultManager : public MockPageFaultManager {
void moveAllocationToGpuDomain(void *ptr) override {

12
level_zero/core/test/unit_tests/sources/memory/test_memory.cpp
View File

@ -2295,7 +2295,15 @@ struct SVMAllocsManagerRelaxedSizeMock : public NEO::SVMAllocsManager {
void *createUnifiedMemoryAllocation(size_t size,
const UnifiedMemoryProperties &svmProperties) override {
validateMemoryProperties(svmProperties);
return alignedMalloc(4096u, 4096u);
auto retPtr{alignedMalloc(4096u, 4096u)};
SvmAllocationData allocData(svmProperties.getRootDeviceIndex());
mockUnifiedMemoryAllocation.setGpuPtr(reinterpret_cast<uint64_t>(retPtr));
mockUnifiedMemoryAllocation.setAllocationOffset(0U);
allocData.gpuAllocations.addAllocation(&mockUnifiedMemoryAllocation);
insertSVMAlloc(retPtr, allocData);
return retPtr;
}
void *createSharedUnifiedMemoryAllocation(size_t size,
@ -2311,6 +2319,8 @@ struct SVMAllocsManagerRelaxedSizeMock : public NEO::SVMAllocsManager {
return alignedMalloc(4096u, 4096u);
}
std::function<void(const UnifiedMemoryProperties &)> validateMemoryProperties = [](const UnifiedMemoryProperties &properties) -> void {};
MockGraphicsAllocation mockUnifiedMemoryAllocation{};
};
struct ContextRelaxedSizeMock : public ContextImp {

5
level_zero/doc/experimental_extensions/DRIVER_EXPERIMENTAL_EXTENSIONS.md
View File

@ -1,6 +1,6 @@
<!---
Copyright (C) 2022 Intel Corporation
Copyright (C) 2022-2025 Intel Corporation
SPDX-License-Identifier: MIT
@ -25,4 +25,5 @@ ze_result_t res = zeDriverGetExtensionFunctionAddress(hDriver, "zexMemOpenIpcHan
```
### [Multiple IPC Handles](MULTIPLE_IPC_HANDLES.md)
### [Multi-CCS Modes](MULTI_CCS_MODES.md)
### [Multi-CCS Modes](MULTI_CCS_MODES.md)
### [Local memory allocation mode](LOCAL_MEMORY_ALLOCATION_MODE.md)

20
level_zero/doc/experimental_extensions/LOCAL_MEMORY_ALLOCATION_MODE.md Normal file
View File

@ -0,0 +1,20 @@
<!---
Copyright (C) 2025 Intel Corporation
SPDX-License-Identifier: MIT
-->
# Local memory allocation mode
At the moment this is supported on Windows only.
In Level Zero, the API function to allocate a device (local) memory is [zeMemAllocDevice](https://oneapi-src.github.io/level-zero-spec/level-zero/latest/core/api.html#zememallocdevice). For such (device USM) allocations, there are two policies of handling the scenario enough resources being available:
- Local-preferred mode: A region is primarily allocated out of local-memory resources, however system-memory may be used as a fallback.
- Local-only mode: A can only be allocated out of local-memory resources. If the amount of available local-memory is not sufficient the `ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY` error is returned.
Depending on the hardware, one of these two modes is being used by default or a specific one can be enforced by setting an environment variable `NEO_LOCAL_MEMORY_ALLOCATION_MODE` to respective value:
- `NEO_LOCAL_MEMORY_ALLOCATION_MODE=0`: HW-default (value of the flag when it is not set)
- `NEO_LOCAL_MEMORY_ALLOCATION_MODE=1`: Local-only
- `NEO_LOCAL_MEMORY_ALLOCATION_MODE=2`: Local-preferred

3
shared/source/debug_settings/release_variables.inl
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2024 Intel Corporation
* Copyright (C) 2020-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -15,3 +15,4 @@ DECLARE_DEBUG_VARIABLE(bool, NEO_CAL_ENABLED, false, "Set by the Compute Aggrega
DECLARE_DEBUG_VARIABLE(std::string, ZE_AFFINITY_MASK, std::string("default"), "Refer to the Level Zero Specification for a description")
DECLARE_DEBUG_VARIABLE(std::string, ZEX_NUMBER_OF_CCS, std::string("default"), "Define number of CCS engines per root device, e.g. setting Root Device Index 0 to 4 CCS, and Root Device Index 1 To 1 CCS: ZEX_NUMBER_OF_CCS=0:4,1:1")
DECLARE_DEBUG_VARIABLE(bool, ZE_ENABLE_PCI_ID_DEVICE_ORDER, false, "Refer to the Level Zero Specification for a description")
DECLARE_DEBUG_VARIABLE(int32_t, NEO_LOCAL_MEMORY_ALLOCATION_MODE, 0, "Specify device-USM allocation policy. 0: default for given HW; 1: require local-memory (return out-of-memory error otherwise); 2: prefer local-memory but refer to system-memory as a fallback");

11
shared/source/helpers/common_types.h
View File

@ -121,6 +121,17 @@ enum class SynchronizedDispatchMode : uint32_t {
limited = 2
};
enum class LocalMemAllocationMode : uint32_t {
hwDefault = 0U,
localOnly = 1U,
localPreferred = 2U,
count = 3U
};
constexpr inline auto toLocalMemAllocationMode(std::underlying_type_t<LocalMemAllocationMode> modeFlag) {
DEBUG_BREAK_IF(modeFlag >= toUnderlying(LocalMemAllocationMode::count));
return toEnum<LocalMemAllocationMode>(modeFlag);
}
namespace InterruptId {
static constexpr uint32_t notUsed = std::numeric_limits<uint32_t>::max();
}

25
shared/source/helpers/windows/product_helper_dg2_and_later_discrete.inl Normal file
View File

@ -0,0 +1,25 @@
/*
* Copyright (C) 2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/debug_settings/debug_settings_manager.h"
namespace NEO {
template <>
bool ProductHelperHw<gfxProduct>::getStorageInfoLocalOnlyFlag(LocalMemAllocationMode usmDeviceAllocationMode, bool defaultValue) const {
switch (usmDeviceAllocationMode) {
case LocalMemAllocationMode::hwDefault:
return defaultValue;
case LocalMemAllocationMode::localOnly:
return true;
case LocalMemAllocationMode::localPreferred:
return false;
default:
UNRECOVERABLE_IF(true);
}
}
} // namespace NEO

21
shared/source/memory_manager/definitions/storage_info.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2024 Intel Corporation
* Copyright (C) 2020-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -29,9 +29,11 @@ StorageInfo MemoryManager::createStorageInfoFromProperties(const AllocationPrope
return storageInfo;
}
const auto deviceCount = GfxCoreHelper::getSubDevicesCount(executionEnvironment.rootDeviceEnvironments[properties.rootDeviceIndex]->getHardwareInfo());
const auto *rootDeviceEnv{executionEnvironment.rootDeviceEnvironments[properties.rootDeviceIndex].get()};
const auto deviceCount = GfxCoreHelper::getSubDevicesCount(rootDeviceEnv->getHardwareInfo());
const auto leastOccupiedBank = getLocalMemoryUsageBankSelector(properties.allocationType, properties.rootDeviceIndex)->getLeastOccupiedBank(properties.subDevicesBitfield);
const auto subDevicesMask = executionEnvironment.rootDeviceEnvironments[properties.rootDeviceIndex]->deviceAffinityMask.getGenericSubDevicesMask().to_ulong();
const auto subDevicesMask = rootDeviceEnv->deviceAffinityMask.getGenericSubDevicesMask().to_ulong();
const DeviceBitfield allTilesValue(properties.subDevicesBitfield.count() == 1 ? maxNBitValue(deviceCount) & subDevicesMask : properties.subDevicesBitfield);
DeviceBitfield preferredTile;
@ -50,7 +52,6 @@ StorageInfo MemoryManager::createStorageInfoFromProperties(const AllocationPrope
AppResourceHelper::copyResourceTagStr(storageInfo.resourceTag, properties.allocationType,
sizeof(storageInfo.resourceTag));
auto releaseHelper = executionEnvironment.rootDeviceEnvironments[properties.rootDeviceIndex]->getReleaseHelper();
switch (properties.allocationType) {
case AllocationType::constantSurface:
case AllocationType::kernelIsa:
@ -128,9 +129,6 @@ StorageInfo MemoryManager::createStorageInfoFromProperties(const AllocationPrope
storageInfo.colouringPolicy = colouringPolicy;
storageInfo.colouringGranularity = granularity;
}
if (!releaseHelper || releaseHelper->isLocalOnlyAllowed()) {
storageInfo.localOnlyRequired = true;
}
if (properties.flags.shareable) {
storageInfo.isLockable = false;
@ -140,9 +138,12 @@ StorageInfo MemoryManager::createStorageInfoFromProperties(const AllocationPrope
default:
break;
}
if (properties.flags.preferCompressed && (!releaseHelper || releaseHelper->isLocalOnlyAllowed())) {
storageInfo.localOnlyRequired = true;
}
storageInfo.localOnlyRequired = getLocalOnlyRequired(properties.allocationType,
rootDeviceEnv->getProductHelper(),
rootDeviceEnv->getReleaseHelper(),
properties.flags.preferCompressed);
if (debugManager.flags.ForceMultiTileAllocPlacement.get()) {
UNRECOVERABLE_IF(properties.allocationType == AllocationType::unknown);
if ((1llu << (static_cast<int64_t>(properties.allocationType) - 1)) & debugManager.flags.ForceMultiTileAllocPlacement.get()) {

12
shared/source/memory_manager/memory_manager.cpp
View File

@ -42,6 +42,7 @@
#include "shared/source/os_interface/os_interface.h"
#include "shared/source/os_interface/product_helper.h"
#include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
#include "shared/source/release_helper/release_helper.h"
#include "shared/source/utilities/logger_neo_only.h"
namespace NEO {
@ -653,11 +654,12 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, const Allo
case AllocationType::svmGpu:
case AllocationType::image:
if (false == allocationData.flags.uncacheable && useLocalPreferredForCacheableBuffers) {
if (!allocationData.flags.preferCompressed) {
if ((usmDeviceAllocationMode == LocalMemAllocationMode::hwDefault) && !allocationData.flags.preferCompressed) {
allocationData.storageInfo.localOnlyRequired = false;
}
allocationData.storageInfo.systemMemoryPlacement = false;
}
break;
default:
break;
}
@ -1250,4 +1252,12 @@ void MemoryManager::removeCustomHeapAllocatorConfig(AllocationType allocationTyp
customHeapAllocators.erase({allocationType, isFrontWindowPool});
}
bool MemoryManager::getLocalOnlyRequired(AllocationType allocationType, const ProductHelper &productHelper, const ReleaseHelper *releaseHelper, bool preferCompressed) const {
const bool enabledForRelease{!releaseHelper || releaseHelper->isLocalOnlyAllowed()};
if (allocationType == AllocationType::buffer || allocationType == AllocationType::svmGpu) {
return productHelper.getStorageInfoLocalOnlyFlag(usmDeviceAllocationMode, enabledForRelease);
}
return (preferCompressed ? enabledForRelease : false);
}
} // namespace NEO

5
shared/source/memory_manager/memory_manager.h
View File

@ -6,6 +6,7 @@
*/
#pragma once
#include "shared/source/helpers/common_types.h"
#include "shared/source/helpers/constants.h"
#include "shared/source/helpers/engine_control.h"
#include "shared/source/helpers/heap_assigner.h"
@ -46,6 +47,7 @@ class HostPtrManager;
class OsContext;
class PrefetchManager;
class HeapAllocator;
class ReleaseHelper;
enum AllocationUsage {
TEMPORARY_ALLOCATION,
@ -342,6 +344,8 @@ class MemoryManager {
void initUsmReuseLimits();
UsmReuseInfo usmReuseInfo;
LocalMemAllocationMode usmDeviceAllocationMode = LocalMemAllocationMode::hwDefault;
bool isLocalOnlyAllocationMode() const { return usmDeviceAllocationMode == LocalMemAllocationMode::localOnly; }
bool shouldLimitAllocationsReuse() const {
return getUsedSystemMemorySize() >= usmReuseInfo.getLimitAllocationsReuseThreshold();
@ -387,6 +391,7 @@ class MemoryManager {
void zeroCpuMemoryIfRequested(const AllocationData &allocationData, void *cpuPtr, size_t size);
void updateLatestContextIdForRootDevice(uint32_t rootDeviceIndex);
virtual DeviceBitfield computeStorageInfoMemoryBanks(const AllocationProperties &properties, DeviceBitfield preferredBank, DeviceBitfield allBanks);
virtual bool getLocalOnlyRequired(AllocationType allocationType, const ProductHelper &productHelper, const ReleaseHelper *releaseHelper, bool preferCompressed) const;
bool initialized = false;
bool forceNonSvmForExternalHostPtr = false;

11
shared/source/os_interface/linux/drm_memory_manager.cpp
View File

@ -44,6 +44,7 @@
#include "shared/source/os_interface/linux/sys_calls.h"
#include "shared/source/os_interface/os_interface.h"
#include "shared/source/os_interface/product_helper.h"
#include "shared/source/release_helper/release_helper.h"
#include <cstring>
#include <memory>
@ -3098,4 +3099,14 @@ bool DrmMemoryManager::reInitDeviceSpecificGfxPartition(uint32_t rootDeviceIndex
void DrmMemoryManager::releaseDeviceSpecificGfxPartition(uint32_t rootDeviceIndex) {
gfxPartitions.at(rootDeviceIndex).reset();
}
bool DrmMemoryManager::getLocalOnlyRequired(AllocationType allocationType, const ProductHelper &productHelper, const ReleaseHelper *releaseHelper, bool preferCompressed) const {
const bool enabledForRelease{!releaseHelper || releaseHelper->isLocalOnlyAllowed()};
if (preferCompressed || allocationType == AllocationType::buffer || allocationType == AllocationType::svmGpu) {
return enabledForRelease;
}
return false;
}
} // namespace NEO

1
shared/source/os_interface/linux/drm_memory_manager.h
View File

@ -198,6 +198,7 @@ class DrmMemoryManager : public MemoryManager {
bool retrieveMmapOffsetForBufferObject(uint32_t rootDeviceIndex, BufferObject &bo, uint64_t flags, uint64_t &offset);
BufferObject::BOType getBOTypeFromPatIndex(uint64_t patIndex, bool isPatIndexSupported) const;
void setLocalMemBanksCount(uint32_t rootDeviceIndex);
bool getLocalOnlyRequired(AllocationType allocationType, const ProductHelper &productHelper, const ReleaseHelper *releaseHelper, bool preferCompressed) const override;
std::vector<BufferObject *> pinBBs;
std::vector<void *> memoryForPinBBs;

3
shared/source/os_interface/product_helper.h
View File

@ -52,6 +52,7 @@ enum class GfxMemoryAllocationMethod : uint32_t;
enum class AllocationType;
enum class CacheRegion : uint16_t;
enum class CachePolicy : uint32_t;
enum class LocalMemAllocationMode : uint32_t;
using ProductHelperCreateFunctionType = std::unique_ptr<ProductHelper> (*)();
extern ProductHelperCreateFunctionType productHelperFactory[IGFX_MAX_PRODUCT];
@ -268,7 +269,7 @@ class ProductHelper {
virtual bool isCompressionForbidden(const HardwareInfo &hwInfo) const = 0;
virtual bool isExposingSubdevicesAllowed() const = 0;
virtual bool useAdditionalBlitProperties() const = 0;
virtual bool getStorageInfoLocalOnlyFlag(LocalMemAllocationMode usmDeviceAllocationMode, bool defaultValue) const = 0;
virtual ~ProductHelper() = default;
protected:

4
shared/source/os_interface/product_helper.inl
View File

@ -1085,4 +1085,8 @@ bool ProductHelperHw<gfxProduct>::useAdditionalBlitProperties() const {
return false;
}
template <PRODUCT_FAMILY gfxProduct>
bool ProductHelperHw<gfxProduct>::getStorageInfoLocalOnlyFlag(LocalMemAllocationMode usmDeviceAllocationMode, bool defaultValue) const {
return defaultValue;
}
} // namespace NEO

1
shared/source/os_interface/product_helper_hw.h
View File

@ -206,6 +206,7 @@ class ProductHelperHw : public ProductHelper {
bool isCompressionForbidden(const HardwareInfo &hwInfo) const override;
bool isExposingSubdevicesAllowed() const override;
bool useAdditionalBlitProperties() const override;
bool getStorageInfoLocalOnlyFlag(LocalMemAllocationMode usmDeviceAllocationMode, bool defaultValue) const override;
~ProductHelperHw() override = default;

1
shared/source/os_interface/windows/wddm_memory_manager.cpp
View File

@ -76,6 +76,7 @@ WddmMemoryManager::WddmMemoryManager(ExecutionEnvironment &executionEnvironment)
alignmentSelector.addCandidateAlignment(customAlignment, false, AlignmentSelector::anyWastage);
}
osMemory = OSMemory::create();
usmDeviceAllocationMode = toLocalMemAllocationMode(debugManager.flags.NEO_LOCAL_MEMORY_ALLOCATION_MODE.get());
initialized = true;
}

1
shared/source/os_interface/windows/wddm_memory_operations_handler.cpp
View File

@ -85,7 +85,6 @@ MemoryOperationsStatus WddmMemoryOperationsHandler::isResident(Device *device, G
MemoryOperationsStatus WddmMemoryOperationsHandler::free(Device *device, GraphicsAllocation &gfxAllocation) {
if (gfxAllocation.isExplicitlyMadeResident()) {
WddmAllocation &wddmAllocation = reinterpret_cast<WddmAllocation &>(gfxAllocation);
if (wddmAllocation.fragmentsStorage.fragmentCount > 0) {

1
shared/source/xe2_hpg_core/windows/product_helper_bmg.cpp
View File

@ -11,6 +11,7 @@
constexpr static auto gfxProduct = IGFX_BMG;
#include "shared/source/helpers/windows/product_helper_dg2_and_later_discrete.inl"
#include "shared/source/xe2_hpg_core/bmg/os_agnostic_product_helper_bmg.inl"
#include "shared/source/xe2_hpg_core/os_agnostic_product_helper_xe2_hpg_core.inl"

1
shared/source/xe_hpg_core/windows/product_helper_dg2.cpp
View File

@ -11,6 +11,7 @@
constexpr static auto gfxProduct = IGFX_DG2;
#include "shared/source/helpers/windows/product_helper_dg2_and_later_discrete.inl"
#include "shared/source/xe_hpg_core/dg2/os_agnostic_product_helper_dg2.inl"
#include "shared/source/xe_hpg_core/os_agnostic_product_helper_xe_hpg_core.inl"

1
shared/test/common/mocks/linux/mock_drm_memory_manager.h
View File

@ -58,6 +58,7 @@ class TestedDrmMemoryManager : public MemoryManagerCreate<DrmMemoryManager> {
using DrmMemoryManager::getBOTypeFromPatIndex;
using DrmMemoryManager::getDefaultDrmContextId;
using DrmMemoryManager::getDrm;
using DrmMemoryManager::getLocalOnlyRequired;
using DrmMemoryManager::getRootDeviceIndex;
using DrmMemoryManager::getUserptrAlignment;
using DrmMemoryManager::gfxPartitions;

7
shared/test/common/mocks/mock_memory_manager.cpp
View File

@ -94,7 +94,12 @@ GraphicsAllocation *MockMemoryManager::allocateGraphicsMemoryWithProperties(cons
validateAllocateProperties(properties);
lastAllocationProperties.reset(new AllocationProperties(properties));
if (returnFakeAllocation) {
return new GraphicsAllocation(properties.rootDeviceIndex, 1u /*num gmms*/, properties.allocationType, const_cast<void *>(ptr), dummyAddress, properties.size, 0, MemoryPool::system4KBPages, maxOsContextCount);
auto *allocation{new GraphicsAllocation(properties.rootDeviceIndex, 1u /*num gmms*/, properties.allocationType, const_cast<void *>(ptr), dummyAddress, properties.size, 0, MemoryPool::system4KBPages, maxOsContextCount)};
AllocationData allocationData;
getAllocationData(allocationData, properties, const_cast<void *>(ptr), createStorageInfoFromProperties(properties));
allocation->storageInfo = allocationData.storageInfo;
return allocation;
}
if (isMockHostMemoryManager) {
allocateGraphicsMemoryWithPropertiesCount++;

1
shared/test/common/mocks/mock_memory_manager.h
View File

@ -39,6 +39,7 @@ class MockMemoryManager : public MemoryManagerCreate<OsAgnosticMemoryManager> {
using MemoryManager::defaultEngineIndex;
using MemoryManager::externalLocalMemoryUsageBankSelector;
using MemoryManager::getAllocationData;
using MemoryManager::getLocalOnlyRequired;
using MemoryManager::gfxPartitions;
using MemoryManager::internalLocalMemoryUsageBankSelector;
using MemoryManager::isNonSvmBuffer;

3
shared/test/common/mocks/mock_product_helper.h
View File

@ -12,6 +12,8 @@
namespace NEO {
enum class LocalMemAllocationMode : uint32_t;
struct MockProductHelper : ProductHelperHw<IGFX_UNKNOWN> {
using ProductHelper::setupPreemptionSurfaceSize;
MockProductHelper() = default;
@ -27,5 +29,6 @@ struct MockProductHelper : ProductHelperHw<IGFX_UNKNOWN> {
ADDMETHOD_CONST_NOBASE(isDeviceUsmPoolAllocatorSupported, bool, false, ());
ADDMETHOD_CONST_NOBASE(is2MBLocalMemAlignmentEnabled, bool, false, ());
ADDMETHOD_CONST_NOBASE(isDisableScratchPagesRequiredForDebugger, bool, true, ());
ADDMETHOD_CONST_NOBASE(getStorageInfoLocalOnlyFlag, bool, false, (LocalMemAllocationMode, bool));
};
} // namespace NEO

8
shared/test/common/mocks/mock_svm_manager.h
View File

@ -15,6 +15,7 @@
namespace NEO {
struct MockSVMAllocsManager : public SVMAllocsManager {
public:
using SVMAllocsManager::insertSVMAlloc;
using SVMAllocsManager::memoryManager;
using SVMAllocsManager::mtxForIndirectAccess;
using SVMAllocsManager::svmAllocs;
@ -32,9 +33,14 @@ struct MockSVMAllocsManager : public SVMAllocsManager {
void *createUnifiedMemoryAllocation(size_t size, const UnifiedMemoryProperties &memoryProperties) override {
requestedZeroedOutAllocation = memoryProperties.isInternalAllocation;
return SVMAllocsManager::createUnifiedMemoryAllocation(size, memoryProperties);
if (createUnifiedMemoryAllocationCallBase) {
return SVMAllocsManager::createUnifiedMemoryAllocation(size, memoryProperties);
}
return createUnifiedMemoryAllocationReturnValue;
}
bool requestedZeroedOutAllocation = false;
bool createUnifiedMemoryAllocationCallBase = true;
void *createUnifiedMemoryAllocationReturnValue = nullptr;
};
template <bool enableLocalMemory>

1
shared/test/common/test_files/igdrcl.config
View File

@ -15,6 +15,7 @@ ZE_AFFINITY_MASK = default
ZEX_NUMBER_OF_CCS = default
ZE_ENABLE_PCI_ID_DEVICE_ORDER = 0
NEO_CAL_ENABLED = 0
NEO_LOCAL_MEMORY_ALLOCATION_MODE = 0
AUBDumpFilterNamedKernelStartIdx = 0
AUBDumpFilterNamedKernelEndIdx = -1
AUBDumpSubCaptureMode = 0

24
shared/test/unit_test/memory_manager/memory_manager_tests.cpp
View File

@ -3167,6 +3167,30 @@ HWTEST_F(MemoryAllocatorTest, givenUseLocalPreferredForCacheableBuffersAndCompre
}
}
HWTEST_F(MemoryAllocatorTest, givenNonDefaultLocalMemoryAllocationModeAndLocalPreferredForCacheableBuffersWhenGettingAllocDataForDeviceUsmThenLocalOnlyRequiredIsNotOverriden) {
DebugManagerStateRestore restorer;
debugManager.flags.UseLocalPreferredForCacheableBuffers.set(1);
AllocationProperties properties(mockRootDeviceIndex, 1, AllocationType::buffer, mockDeviceBitfield);
properties.flags.uncacheable = false;
MockMemoryManager mockMemoryManager;
auto releaseHelper = std::make_unique<MockReleaseHelper>();
mockMemoryManager.executionEnvironment.rootDeviceEnvironments[properties.rootDeviceIndex]->releaseHelper.reset(releaseHelper.get());
for (const auto debugKeyValue : std::to_array({1, 2})) {
mockMemoryManager.usmDeviceAllocationMode = toLocalMemAllocationMode(debugKeyValue);
releaseHelper->isLocalOnlyAllowedResult = (debugKeyValue == 1);
auto storageInfo{mockMemoryManager.createStorageInfoFromProperties(properties)};
bool expectedValue{storageInfo.localOnlyRequired};
AllocationData allocData;
mockMemoryManager.getAllocationData(allocData, properties, nullptr, storageInfo);
EXPECT_EQ(expectedValue, allocData.storageInfo.localOnlyRequired);
}
releaseHelper.release();
}
TEST(MemoryTransferHelperTest, WhenBlitterIsSelectedButBlitCopyFailsThenFallbackToCopyOnCPU) {
constexpr uint32_t dataSize = 16;
uint8_t destData[dataSize] = {};

51
shared/test/unit_test/memory_manager/storage_info_tests.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2021-2024 Intel Corporation
* Copyright (C) 2021-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -18,6 +18,7 @@
#include "shared/test/common/mocks/mock_device.h"
#include "shared/test/common/mocks/mock_graphics_allocation.h"
#include "shared/test/common/mocks/mock_memory_manager.h"
#include "shared/test/common/mocks/mock_product_helper.h"
#include "shared/test/common/mocks/mock_release_helper.h"
#include "shared/test/common/mocks/ult_device_factory.h"
#include "shared/test/common/test_macros/hw_test.h"
@ -746,3 +747,51 @@ TEST_F(MultiDeviceStorageInfoTest, givenDirectSubmissionForceLocalMemoryStorageE
}
}
}
TEST_F(MultiDeviceStorageInfoTest, givenBufferOrSvmGpuAllocationWhenLocalOnlyFlagValueComputedThenProductHelperIsUsed) {
constexpr bool preferCompressed{false};
MockProductHelper productHelper{};
MockReleaseHelper mockReleaseHelper{};
EXPECT_EQ(0U, productHelper.getStorageInfoLocalOnlyFlagCalled);
const auto isEnabledForRelease{[](ReleaseHelper *releaseHelper) { return (!releaseHelper || releaseHelper->isLocalOnlyAllowed()); }};
for (const auto allocationType : std::array{AllocationType::buffer, AllocationType::svmGpu}) {
productHelper.getStorageInfoLocalOnlyFlagResult = isEnabledForRelease(nullptr);
EXPECT_EQ(memoryManager->getLocalOnlyRequired(allocationType, productHelper, nullptr, preferCompressed),
productHelper.getStorageInfoLocalOnlyFlagResult);
for (const bool allowed : std::array{false, true}) {
mockReleaseHelper.isLocalOnlyAllowedResult = allowed;
productHelper.getStorageInfoLocalOnlyFlagResult = isEnabledForRelease(&mockReleaseHelper);
EXPECT_EQ(memoryManager->getLocalOnlyRequired(allocationType, productHelper, &mockReleaseHelper, preferCompressed),
productHelper.getStorageInfoLocalOnlyFlagResult);
}
}
EXPECT_EQ(6U, productHelper.getStorageInfoLocalOnlyFlagCalled);
}
TEST_F(MultiDeviceStorageInfoTest, givenNeitherBufferNorSvmGpuAllocationWhenLocalOnlyFlagValueComputedThenProductHelperIsNotUsed) {
MockProductHelper productHelper{};
MockReleaseHelper mockReleaseHelper{};
EXPECT_EQ(0U, productHelper.getStorageInfoLocalOnlyFlagCalled);
const auto allocationType{AllocationType::unknown};
bool preferCompressed{true};
for (const bool allowed : std::array{false, true}) {
mockReleaseHelper.isLocalOnlyAllowedResult = allowed;
EXPECT_EQ(memoryManager->getLocalOnlyRequired(allocationType, productHelper, &mockReleaseHelper, preferCompressed),
mockReleaseHelper.isLocalOnlyAllowedResult);
}
preferCompressed = false;
for (const bool allowed : std::array{false, true}) {
mockReleaseHelper.isLocalOnlyAllowedResult = allowed;
EXPECT_EQ(memoryManager->getLocalOnlyRequired(allocationType, productHelper, &mockReleaseHelper, preferCompressed),
false);
}
EXPECT_EQ(0U, productHelper.getStorageInfoLocalOnlyFlagCalled);
}

28
shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp
View File

@ -18,6 +18,7 @@
#include "shared/source/os_interface/linux/drm_memory_operations_handler_bind.h"
#include "shared/source/os_interface/linux/i915.h"
#include "shared/source/os_interface/linux/os_context_linux.h"
#include "shared/source/release_helper/release_helper.h"
#include "shared/test/common/helpers/engine_descriptor_helper.h"
#include "shared/test/common/helpers/gtest_helpers.h"
#include "shared/test/common/mocks/linux/mock_drm_allocation.h"
@ -33,6 +34,7 @@
#include "shared/test/common/mocks/mock_gmm_resource_info.h"
#include "shared/test/common/mocks/mock_host_ptr_manager.h"
#include "shared/test/common/mocks/mock_product_helper.h"
// #include "shared/test/common/mocks/mock_release_helper.h"
#include "shared/test/common/os_interface/linux/drm_memory_manager_fixture.h"
#include "shared/test/common/os_interface/linux/drm_mock_cache_info.h"
#include "shared/test/common/os_interface/linux/drm_mock_memory_info.h"
@ -5664,6 +5666,17 @@ TEST(DrmMemoryManagerSimpleTest, WhenDrmIsCreatedThenQueryPageFaultSupportIsCall
using DrmMemoryManagerWithLocalMemoryTest = Test<DrmMemoryManagerWithLocalMemoryFixture>;
HWTEST_TEMPLATED_F(DrmMemoryManagerWithLocalMemoryTest, givenDrmMemoryManagerWithoutLocalMemoryWhenPreferCompressedIsSetThenLocalOnlyRequriedDeterminedByReleaseHelper) {
TestedDrmMemoryManager memoryManager(false, false, false, *executionEnvironment);
AllocationProperties properties{1, true, 4096, AllocationType::unknown, false, 0b10};
properties.flags.preferCompressed = true;
auto storageInfo = memoryManager.createStorageInfoFromProperties(properties);
const auto *releaseHelper{executionEnvironment->rootDeviceEnvironments[0]->getReleaseHelper()};
EXPECT_EQ(storageInfo.localOnlyRequired, (!releaseHelper || releaseHelper->isLocalOnlyAllowed()));
}
HWTEST_TEMPLATED_F(DrmMemoryManagerWithLocalMemoryTest, givenDrmMemoryManagerWithLocalMemoryWhenLockResourceIsCalledOnAllocationInLocalMemoryThenReturnNullPtr) {
DrmAllocation drmAllocation(rootDeviceIndex, 1u /*num gmms*/, AllocationType::unknown, nullptr, nullptr, 0u, 0u, MemoryPool::localMemory);
@ -8998,4 +9011,17 @@ HWTEST_TEMPLATED_F(DrmMemoryManagerTest, givenGfxPartitionWhenReleasedAndReiniti
EXPECT_EQ(heapExternalDeviceFrontWindow, heapExternalDeviceFrontWindow2);
EXPECT_EQ(heapInternalFrontWindow, heapInternalFrontWindow2);
EXPECT_EQ(heapInternalDeviceFrontWindow, heapInternalDeviceFrontWindow2);
}
}
HWTEST_TEMPLATED_F(DrmMemoryManagerTest, givenDeviceUsmAllocationWhenLocalOnlyFlagValueComputedThenProductHelperIsNotUsed) {
constexpr bool preferCompressed{false};
MockProductHelper productHelper{};
EXPECT_EQ(0U, productHelper.getStorageInfoLocalOnlyFlagCalled);
productHelper.getStorageInfoLocalOnlyFlagResult = false;
EXPECT_EQ(memoryManager->getLocalOnlyRequired(AllocationType::buffer, productHelper, nullptr, preferCompressed), true);
EXPECT_EQ(memoryManager->getLocalOnlyRequired(AllocationType::svmGpu, productHelper, nullptr, preferCompressed), true);
EXPECT_EQ(0U, productHelper.getStorageInfoLocalOnlyFlagCalled);
}

24
shared/test/unit_test/os_interface/windows/product_helper_win_tests.cpp
View File

@ -12,6 +12,7 @@
#include "shared/source/os_interface/os_interface.h"
#include "shared/source/os_interface/product_helper.h"
#include "shared/source/os_interface/windows/wddm/wddm.h"
#include "shared/test/common/helpers/debug_manager_state_restore.h"
#include "shared/test/common/mocks/mock_execution_environment.h"
#include "shared/test/common/test_macros/hw_test.h"
@ -116,4 +117,27 @@ HWTEST2_F(ProductHelperTestWindows, givenE2ECompressionWhenConfiguringHwInfoWddm
EXPECT_FALSE(outHwInfo.capabilityTable.ftrRenderCompressedImages);
}
HWTEST_F(ProductHelperTestWindows, givenProductFamilyWhenLocalMemAllocationModeSupportedThenLocalOnlyFlagIsSetAccordingly) {
DebugManagerStateRestore restore;
EXPECT_EQ(0, debugManager.flags.NEO_LOCAL_MEMORY_ALLOCATION_MODE.get());
const auto productFamily{defaultHwInfo->platform.eProductFamily};
const bool isLocalMemModeKeySupported{productFamily == IGFX_DG2 || productFamily == IGFX_BMG};
EXPECT_TRUE(productHelper->getStorageInfoLocalOnlyFlag(LocalMemAllocationMode::hwDefault, true));
EXPECT_FALSE(productHelper->getStorageInfoLocalOnlyFlag(LocalMemAllocationMode::hwDefault, false));
if (isLocalMemModeKeySupported) {
EXPECT_TRUE(productHelper->getStorageInfoLocalOnlyFlag(LocalMemAllocationMode::localOnly, true));
EXPECT_TRUE(productHelper->getStorageInfoLocalOnlyFlag(LocalMemAllocationMode::localOnly, false));
EXPECT_FALSE(productHelper->getStorageInfoLocalOnlyFlag(LocalMemAllocationMode::localPreferred, true));
EXPECT_FALSE(productHelper->getStorageInfoLocalOnlyFlag(LocalMemAllocationMode::localPreferred, false));
} else {
EXPECT_TRUE(productHelper->getStorageInfoLocalOnlyFlag(LocalMemAllocationMode::localOnly, true));
EXPECT_FALSE(productHelper->getStorageInfoLocalOnlyFlag(LocalMemAllocationMode::localOnly, false));
EXPECT_TRUE(productHelper->getStorageInfoLocalOnlyFlag(LocalMemAllocationMode::localPreferred, true));
EXPECT_FALSE(productHelper->getStorageInfoLocalOnlyFlag(LocalMemAllocationMode::localPreferred, false));
}
}
} // namespace NEO

10
shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp
View File

@ -107,6 +107,16 @@ class WddmMemoryManagerTests : public ::testing::Test {
}
};
TEST_F(WddmMemoryManagerTests, GivenLocalMemoryAllocationModeReleaseKeyWhenWddmMemoryManagerConstructedThenUsmDeviceAllocationModeProperlySet) {
DebugManagerStateRestore restorer;
for (const int32_t releaseKeyVal : std::array{0, 1, 2}) {
debugManager.flags.NEO_LOCAL_MEMORY_ALLOCATION_MODE.set(releaseKeyVal);
WddmMemoryManager memoryManager{*executionEnvironment};
EXPECT_EQ(memoryManager.usmDeviceAllocationMode, toLocalMemAllocationMode(releaseKeyVal));
}
}
TEST_F(WddmMemoryManagerTests, GivenAllocDataWithSVMCPUSetWhenAllocateGraphicsMemoryWithAlignmentThenProperFunctionIsUsed) {
AllocationProperties allocationProperties{0u, 0u, NEO::AllocationType::svmCpu, {}};
NEO::AllocationData allocData = {};