fix: report ZE_MEMORY_ACCESS_CAP_FLAG_CONCURRENT correctly

At the moment the capability is returned only based on the value
returned by the `productHelper`, which is too liberal. The capability
must also consider the support reported by `memoryManager`. Only then
the support reported is aligned with actual logic of handling
USM-allocations.

Related-To: NEO-10040
Signed-off-by: Maciej Bielski <maciej.bielski@intel.com>
This commit is contained in:
Maciej Bielski
2025-01-24 16:39:50 +00:00
committed by Compute-Runtime-Automation
parent 1abb48c3e0
commit a8779c2387
16 changed files with 112 additions and 39 deletions

View File

@@ -768,21 +768,23 @@ ze_result_t DeviceImp::getMemoryProperties(uint32_t *pCount, ze_device_memory_pr
ze_result_t DeviceImp::getMemoryAccessProperties(ze_device_memory_access_properties_t *pMemAccessProperties) {
auto &hwInfo = this->getHwInfo();
auto &productHelper = this->getProductHelper();
pMemAccessProperties->hostAllocCapabilities =
static_cast<ze_memory_access_cap_flags_t>(productHelper.getHostMemCapabilities(&hwInfo));
pMemAccessProperties->deviceAllocCapabilities =
static_cast<ze_memory_access_cap_flags_t>(productHelper.getDeviceMemCapabilities());
auto memoryManager{this->getDriverHandle()->getMemoryManager()};
const bool isKmdMigrationAvailable{memoryManager->isKmdMigrationAvailable(this->getRootDeviceIndex())};
pMemAccessProperties->sharedSingleDeviceAllocCapabilities =
static_cast<ze_memory_access_cap_flags_t>(productHelper.getSingleDeviceSharedMemCapabilities());
static_cast<ze_memory_access_cap_flags_t>(productHelper.getSingleDeviceSharedMemCapabilities(isKmdMigrationAvailable));
pMemAccessProperties->sharedCrossDeviceAllocCapabilities = {};
if (this->getNEODevice()->getHardwareInfo().capabilityTable.p2pAccessSupported) {
pMemAccessProperties->sharedCrossDeviceAllocCapabilities = ZE_MEMORY_ACCESS_CAP_FLAG_RW;
auto memoryManager = this->getDriverHandle()->getMemoryManager();
if (memoryManager->isKmdMigrationAvailable(this->getRootDeviceIndex()) &&
if (isKmdMigrationAvailable &&
memoryManager->hasPageFaultsEnabled(*this->getNEODevice()) &&
NEO::debugManager.flags.EnableConcurrentSharedCrossP2PDeviceAccess.get() == 1) {
pMemAccessProperties->sharedCrossDeviceAllocCapabilities |= ZE_MEMORY_ACCESS_CAP_FLAG_CONCURRENT;

View File

@@ -1,5 +1,5 @@
#
# Copyright (C) 2021-2024 Intel Corporation
# Copyright (C) 2021-2025 Intel Corporation
#
# SPDX-License-Identifier: MIT
#
@@ -10,5 +10,6 @@ if(UNIX)
${CMAKE_CURRENT_SOURCE_DIR}/test_device_uuid.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_device_pci_bus_info_linux.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_device_pci_speed_info_linux.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_l0_drm_device.cpp
)
endif()

View File

@@ -0,0 +1,52 @@
/*
* Copyright (C) 2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/test/common/libult/linux/drm_mock.h"
#include "shared/test/common/mocks/linux/mock_drm_memory_manager.h"
#include "shared/test/common/test_macros/test.h"
#include "level_zero/core/test/unit_tests/fixtures/device_fixture.h"
#include "gtest/gtest.h"
namespace L0 {
namespace ult {
using DrmDeviceTests = Test<DeviceFixture>;
TEST_F(DrmDeviceTests, whenMemoryAccessPropertiesQueriedThenConcurrentDeviceSharedMemSupportDependsOnMemoryManagerHelper) {
constexpr auto rootDeviceIndex{0U};
execEnv->rootDeviceEnvironments[rootDeviceIndex]->osInterface.reset(new NEO::OSInterface);
auto drm{new DrmMock{*execEnv->rootDeviceEnvironments[rootDeviceIndex]}};
execEnv->rootDeviceEnvironments[rootDeviceIndex]->osInterface->setDriverModel(std::unique_ptr<DriverModel>{drm});
auto *origMemoryManager{device->getDriverHandle()->getMemoryManager()};
auto *proxyMemoryManager{new TestedDrmMemoryManager{*execEnv}};
device->getDriverHandle()->setMemoryManager(proxyMemoryManager);
auto &productHelper = device->getProductHelper();
ze_device_memory_access_properties_t properties;
for (auto pfSupported : std::array{false, true}) {
drm->pageFaultSupported = pfSupported;
bool isKmdMigrationAvailable{proxyMemoryManager->isKmdMigrationAvailable(rootDeviceIndex)};
proxyMemoryManager->isKmdMigrationAvailableCalled = 0U;
auto result = device->getMemoryAccessProperties(&properties);
EXPECT_EQ(ZE_RESULT_SUCCESS, result);
EXPECT_EQ(proxyMemoryManager->isKmdMigrationAvailableCalled, 1U);
auto expectedSharedSingleDeviceAllocCapabilities = static_cast<ze_memory_access_cap_flags_t>(productHelper.getSingleDeviceSharedMemCapabilities(isKmdMigrationAvailable));
EXPECT_EQ(expectedSharedSingleDeviceAllocCapabilities, properties.sharedSingleDeviceAllocCapabilities);
}
device->getDriverHandle()->setMemoryManager(origMemoryManager);
delete proxyMemoryManager;
}
} // namespace ult
} // namespace L0

View File

@@ -2801,7 +2801,8 @@ TEST_F(DeviceTests, WhenGettingMemoryAccessPropertiesThenSuccessIsReturned) {
auto expectedDeviceAllocCapabilities = static_cast<ze_memory_access_cap_flags_t>(productHelper.getDeviceMemCapabilities());
EXPECT_EQ(expectedDeviceAllocCapabilities, properties.deviceAllocCapabilities);
auto expectedSharedSingleDeviceAllocCapabilities = static_cast<ze_memory_access_cap_flags_t>(productHelper.getSingleDeviceSharedMemCapabilities());
bool isKmdMigrationSupported{false};
auto expectedSharedSingleDeviceAllocCapabilities = static_cast<ze_memory_access_cap_flags_t>(productHelper.getSingleDeviceSharedMemCapabilities(isKmdMigrationSupported));
EXPECT_EQ(expectedSharedSingleDeviceAllocCapabilities, properties.sharedSingleDeviceAllocCapabilities);
auto expectedSharedSystemAllocCapabilities = static_cast<ze_memory_access_cap_flags_t>(productHelper.getSharedSystemMemCapabilities(&hwInfo));

View File

@@ -1105,7 +1105,7 @@ TEST_F(MemoryTest, whenCallingSetAtomicAccessAttributeWithInsufficientCapability
uint64_t getHostMemCapabilities(const HardwareInfo *hwInfo) const override {
return 0;
}
uint64_t getSingleDeviceSharedMemCapabilities() const override {
uint64_t getSingleDeviceSharedMemCapabilities(bool) const override {
return 0;
}
};
@@ -1218,7 +1218,7 @@ TEST_F(MemoryTest, whenCallingSetAtomicAccessAttributeForSystemAccessSharedSingl
struct MockProductHelperAtomic : NEO::ProductHelperHw<IGFX_UNKNOWN> {
MockProductHelperAtomic() = default;
uint64_t getSingleDeviceSharedMemCapabilities() const override {
uint64_t getSingleDeviceSharedMemCapabilities(bool) const override {
return 15;
}
};

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2024 Intel Corporation
* Copyright (C) 2020-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -15,6 +15,7 @@
#include "shared/source/helpers/hw_info_helper.h"
#include "shared/source/helpers/string.h"
#include "shared/source/kernel/kernel_properties.h"
#include "shared/source/memory_manager/memory_manager.h"
#include "shared/source/os_interface/driver_info.h"
#include "opencl/source/cl_device/cl_device.h"
@@ -404,7 +405,9 @@ void ClDevice::initializeCaps() {
deviceInfo.hostMemCapabilities = productHelper.getHostMemCapabilities(&hwInfo);
deviceInfo.deviceMemCapabilities = productHelper.getDeviceMemCapabilities();
deviceInfo.singleDeviceSharedMemCapabilities = productHelper.getSingleDeviceSharedMemCapabilities();
const bool isKmdMigrationAvailable{getMemoryManager()->isKmdMigrationAvailable(getRootDeviceIndex())};
deviceInfo.singleDeviceSharedMemCapabilities = productHelper.getSingleDeviceSharedMemCapabilities(isKmdMigrationAvailable);
deviceInfo.crossDeviceSharedMemCapabilities = productHelper.getCrossDeviceSharedMemCapabilities();
deviceInfo.sharedSystemMemCapabilities = productHelper.getSharedSystemMemCapabilities(&hwInfo);

View File

@@ -6,6 +6,7 @@
*/
#pragma once
#include "shared/source/memory_manager/memory_manager.h"
#include "shared/test/common/mocks/mock_device.h"
#include "opencl/test/unit_test/mocks/mock_cl_device.h"
@@ -22,7 +23,8 @@ struct GetDeviceInfoMemCapabilitiesTest : ::testing::Test {
void check(std::vector<TestParams> &params) {
auto device = std::make_unique<MockClDevice>(MockDevice::createWithNewExecutionEnvironment<MockDevice>(nullptr));
auto &productHelper = device->getProductHelper();
const bool isKmdMigrationAvailable{device->getMemoryManager()->isKmdMigrationAvailable(device->getRootDeviceIndex())};
for (auto &param : params) {
cl_unified_shared_memory_capabilities_intel unifiedSharedMemoryCapabilities{};
size_t paramRetSize;
@@ -31,7 +33,7 @@ struct GetDeviceInfoMemCapabilitiesTest : ::testing::Test {
sizeof(cl_unified_shared_memory_capabilities_intel),
&unifiedSharedMemoryCapabilities, &paramRetSize);
EXPECT_EQ(CL_SUCCESS, retVal);
if ((param.paramName == CL_DEVICE_SINGLE_DEVICE_SHARED_MEM_CAPABILITIES_INTEL) && (productHelper.isKmdMigrationSupported())) {
if ((param.paramName == CL_DEVICE_SINGLE_DEVICE_SHARED_MEM_CAPABILITIES_INTEL) && (isKmdMigrationAvailable)) {
EXPECT_EQ((param.expectedCapabilities | CL_UNIFIED_SHARED_MEMORY_CONCURRENT_ACCESS_INTEL | CL_UNIFIED_SHARED_MEMORY_CONCURRENT_ATOMIC_ACCESS_INTEL), unifiedSharedMemoryCapabilities);
} else {
EXPECT_EQ(param.expectedCapabilities, unifiedSharedMemoryCapabilities);

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2021-2024 Intel Corporation
* Copyright (C) 2021-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -32,6 +32,7 @@ PVCTEST_F(GetDeviceInfoMemCapabilitiesTest, GivenValidParametersWhenGetDeviceInf
PVCTEST_F(GetDeviceInfoMemCapabilitiesTest, GivenEnableUsmConcurrentAccessSupportWhenGetDeviceInfoIsCalledForPVCThenClSuccessIsReturned) {
DebugManagerStateRestore restorer;
debugManager.flags.EnableUsmConcurrentAccessSupport.set(0b1110);
debugManager.flags.UseKmdMigration.set(true);
std::vector<TestParams> params = {
{CL_DEVICE_HOST_MEM_CAPABILITIES_INTEL, CL_UNIFIED_SHARED_MEMORY_ACCESS_INTEL},

View File

@@ -26,6 +26,7 @@ struct KmdNotifyProperties;
struct AllocationData;
class CommandStreamReceiver;
class Device;
class Drm;
enum class LocalMemoryAccessMode;
struct FrontEndPropertiesSupport;
struct HardwareInfo;
@@ -82,7 +83,7 @@ class ProductHelper {
virtual void adjustSamplerState(void *sampler, const HardwareInfo &hwInfo) const = 0;
virtual uint64_t getHostMemCapabilities(const HardwareInfo *hwInfo) const = 0;
virtual uint64_t getDeviceMemCapabilities() const = 0;
virtual uint64_t getSingleDeviceSharedMemCapabilities() const = 0;
virtual uint64_t getSingleDeviceSharedMemCapabilities(bool isKmdMigrationAvailable) const = 0;
virtual uint64_t getCrossDeviceSharedMemCapabilities() const = 0;
virtual uint64_t getSharedSystemMemCapabilities(const HardwareInfo *hwInfo) const = 0;
virtual std::vector<int32_t> getKernelSupportedThreadArbitrationPolicies() const = 0;

View File

@@ -148,10 +148,10 @@ uint64_t ProductHelperHw<gfxProduct>::getDeviceMemCapabilities() const {
}
template <PRODUCT_FAMILY gfxProduct>
uint64_t ProductHelperHw<gfxProduct>::getSingleDeviceSharedMemCapabilities() const {
uint64_t ProductHelperHw<gfxProduct>::getSingleDeviceSharedMemCapabilities(bool isKmdMigrationAvailable) const {
uint64_t capabilities = UnifiedSharedMemoryFlags::access | UnifiedSharedMemoryFlags::atomicAccess;
if (isKmdMigrationSupported() || getConcurrentAccessMemCapabilitiesSupported(UsmAccessCapabilities::sharedSingleDevice)) {
if (isKmdMigrationAvailable || getConcurrentAccessMemCapabilitiesSupported(UsmAccessCapabilities::sharedSingleDevice)) {
capabilities |= UnifiedSharedMemoryFlags::concurrentAccess | UnifiedSharedMemoryFlags::concurrentAtomicAccess;
}

View File

@@ -24,7 +24,7 @@ class ProductHelperHw : public ProductHelper {
void adjustSamplerState(void *sampler, const HardwareInfo &hwInfo) const override;
uint64_t getHostMemCapabilities(const HardwareInfo *hwInfo) const override;
uint64_t getDeviceMemCapabilities() const override;
uint64_t getSingleDeviceSharedMemCapabilities() const override;
uint64_t getSingleDeviceSharedMemCapabilities(bool isKmdMigrationAvailable) const override;
uint64_t getCrossDeviceSharedMemCapabilities() const override;
uint64_t getSharedSystemMemCapabilities(const HardwareInfo *hwInfo) const override;
std::vector<int32_t> getKernelSupportedThreadArbitrationPolicies() const override;

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2024 Intel Corporation
* Copyright (C) 2018-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -172,6 +172,7 @@ class TestedDrmMemoryManager : public MemoryManagerCreate<DrmMemoryManager> {
return DrmMemoryManager::acquireGpuRangeWithCustomAlignment(size, rootDeviceIndex, heapIndex, alignment);
}
ADDMETHOD(isLimitedRange, bool, true, false, (uint32_t rootDeviceIndex), (rootDeviceIndex));
ADDMETHOD(isKmdMigrationAvailable, bool, true, false, (uint32_t rootDeviceIndex), (rootDeviceIndex));
DeviceBitfield computeStorageInfoMemoryBanks(const AllocationProperties &properties, DeviceBitfield preferredBank, DeviceBitfield allBanks) override {
++computeStorageInfoMemoryBanksCalled;

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2024 Intel Corporation
* Copyright (C) 2018-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -43,6 +43,7 @@ struct MockRootDeviceEnvironment : public RootDeviceEnvironment {
struct MockExecutionEnvironment : ExecutionEnvironment {
using ExecutionEnvironment::adjustCcsCountImpl;
using ExecutionEnvironment::directSubmissionController;
using ExecutionEnvironment::memoryManager;
using ExecutionEnvironment::rootDeviceEnvironments;
~MockExecutionEnvironment() override = default;

View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2021-2024 Intel Corporation
* Copyright (C) 2021-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -59,7 +59,7 @@ uint64_t ProductHelperHw<IGFX_UNKNOWN>::getDeviceMemCapabilities() const {
}
template <>
uint64_t ProductHelperHw<IGFX_UNKNOWN>::getSingleDeviceSharedMemCapabilities() const {
uint64_t ProductHelperHw<IGFX_UNKNOWN>::getSingleDeviceSharedMemCapabilities(bool) const {
return 0;
}

View File

@@ -13,6 +13,7 @@
#include "shared/test/common/helpers/engine_descriptor_helper.h"
#include "shared/test/common/libult/linux/drm_query_mock.h"
#include "shared/test/common/mocks/linux/mock_drm_allocation.h"
#include "shared/test/common/mocks/linux/mock_drm_memory_manager.h"
#include "shared/test/common/mocks/linux/mock_os_context_linux.h"
#include "shared/test/common/mocks/mock_execution_environment.h"
@@ -179,33 +180,35 @@ TEST(DrmVmBindTest, givenUseKmdMigrationWhenCallingBindBoOnUnifiedSharedMemoryTh
}
TEST(DrmVmBindTest, givenDrmWithPageFaultSupportWhenCallingBindBoOnUnifiedSharedMemoryThenMarkAllocationShouldPageFaultWhenKmdMigrationIsSupported) {
constexpr auto rootDeviceIndex{0U};
auto executionEnvironment = std::make_unique<MockExecutionEnvironment>();
executionEnvironment->rootDeviceEnvironments[0]->initGmm();
executionEnvironment->initializeMemoryManager();
executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->initGmm();
executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->osInterface.reset(new NEO::OSInterface);
DrmQueryMock drm(*executionEnvironment->rootDeviceEnvironments[0]);
drm.pageFaultSupported = true;
auto drm{new DrmQueryMock{*executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]}};
drm->pageFaultSupported = true;
executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->osInterface->setDriverModel(std::unique_ptr<DriverModel>{drm});
executionEnvironment->memoryManager.reset(new MockDrmMemoryManager{GemCloseWorkerMode::gemCloseWorkerInactive, false, false, *executionEnvironment});
OsContextLinux osContext(drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor());
OsContextLinux osContext(*drm, rootDeviceIndex, 0u, EngineDescriptorHelper::getDefaultDescriptor());
osContext.ensureContextInitialized(false);
uint32_t vmHandleId = 0;
MockBufferObject bo(0u, &drm, 3, 0, 0, 1);
MockDrmAllocation allocation(0u, AllocationType::unifiedSharedMemory, MemoryPool::localMemory);
MockBufferObject bo(rootDeviceIndex, drm, 3, 0, 0, 1);
MockDrmAllocation allocation(rootDeviceIndex, AllocationType::unifiedSharedMemory, MemoryPool::localMemory);
allocation.bufferObjects[0] = &bo;
allocation.bindBO(&bo, &osContext, vmHandleId, nullptr, true, false);
auto &productHelper = drm.getRootDeviceEnvironment().getHelper<ProductHelper>();
auto kmdMigrationSupported = productHelper.isKmdMigrationSupported();
const bool isKmdMigrationAvailable{executionEnvironment->memoryManager->isKmdMigrationAvailable(rootDeviceIndex)};
if (kmdMigrationSupported) {
EXPECT_TRUE(allocation.shouldAllocationPageFault(&drm));
if (isKmdMigrationAvailable) {
EXPECT_TRUE(allocation.shouldAllocationPageFault(drm));
EXPECT_FALSE(bo.isExplicitResidencyRequired());
EXPECT_EQ(DrmPrelimHelper::getImmediateVmBindFlag(), drm.context.receivedVmBind->flags);
EXPECT_EQ(DrmPrelimHelper::getImmediateVmBindFlag(), drm->context.receivedVmBind->flags);
} else {
EXPECT_FALSE(allocation.shouldAllocationPageFault(&drm));
EXPECT_FALSE(allocation.shouldAllocationPageFault(drm));
EXPECT_TRUE(bo.isExplicitResidencyRequired());
EXPECT_EQ(DrmPrelimHelper::getImmediateVmBindFlag() | DrmPrelimHelper::getMakeResidentVmBindFlag(), drm.context.receivedVmBind->flags);
EXPECT_EQ(DrmPrelimHelper::getImmediateVmBindFlag() | DrmPrelimHelper::getMakeResidentVmBindFlag(), drm->context.receivedVmBind->flags);
}
}

View File

@@ -115,7 +115,8 @@ HWTEST_F(ProductHelperTest, givenProductHelperWhenGettingMemoryCapabilitiesThenC
}
}
auto singleDeviceSharedMemCapabilities = productHelper->getSingleDeviceSharedMemCapabilities();
constexpr bool isKmdMigrationAvailable{false};
auto singleDeviceSharedMemCapabilities = productHelper->getSingleDeviceSharedMemCapabilities(isKmdMigrationAvailable);
if (singleDeviceSharedMemCapabilities > 0) {
if (capabilityBitset.test(static_cast<uint32_t>(UsmAccessCapabilities::sharedSingleDevice))) {
EXPECT_TRUE(UnifiedSharedMemoryFlags::concurrentAccess & singleDeviceSharedMemCapabilities);
@@ -142,11 +143,15 @@ HWTEST_F(ProductHelperTest, givenProductHelperWhenGettingMemoryCapabilitiesThenC
}
HWTEST_F(ProductHelperTest, givenProductHelperAndSingleDeviceSharedMemAccessConcurrentAtomicEnabledIfKmdMigrationEnabled) {
DebugManagerStateRestore restore;
debugManager.flags.EnableUsmConcurrentAccessSupport.set(0);
auto singleDeviceSharedMemCapabilities = productHelper->getSingleDeviceSharedMemCapabilities();
if ((singleDeviceSharedMemCapabilities > 0) && (productHelper->isKmdMigrationSupported())) {
EXPECT_TRUE(UnifiedSharedMemoryFlags::concurrentAccess & singleDeviceSharedMemCapabilities);
EXPECT_TRUE(UnifiedSharedMemoryFlags::concurrentAtomicAccess & singleDeviceSharedMemCapabilities);
for (const bool isKmdMigrationAvailable : std::array<bool, 2>{false, true}) {
auto singleDeviceSharedMemCapabilities = productHelper->getSingleDeviceSharedMemCapabilities(isKmdMigrationAvailable);
if (singleDeviceSharedMemCapabilities > 0) {
EXPECT_EQ(isKmdMigrationAvailable, !!(UnifiedSharedMemoryFlags::concurrentAccess & singleDeviceSharedMemCapabilities));
EXPECT_EQ(isKmdMigrationAvailable, !!(UnifiedSharedMemoryFlags::concurrentAtomicAccess & singleDeviceSharedMemCapabilities));
}
}
}