Media/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-09-15 13:01:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: regression caused by tbx fault mngr

Browse Source 
Addresses regressions from the reverted merge
of the tbx fault manager for host memory.

Recursive locking of mutex caused deadlock.

To fix, separate tbx fault data from base
cpu fault data, allowing separate mutexes
for each, eliminating recursive locks on
the same mutex.

By separating, we also help ensure that tbx-related
changes don't affect the original cpu fault manager code
paths.

As an added safe guard preventing critical regressions
and avoiding another auto-revert, the tbx fault manager
is hidden behind a new debug flag which is disabled by default.

Related-To: NEO-12268
Signed-off-by: Jack Myers <jack.myers@intel.com>
This commit is contained in:
Jack Myers
2024-12-23 21:42:43 +00:00
committed by Compute-Runtime-Automation
parent b8157a2547
commit 7f9fadc314
36 changed files with 874 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
level_zero/core/source/device/device_imp.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2024 Intel Corporation
* Copyright (C) 2020-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -196,6 +196,6 @@ struct DeviceImp : public Device, NEO::NonCopyableOrMovableClass {
std::unique_ptr<DebugSession> debugSession;
};
void transferAndUnprotectMemoryWithHints(NEO::PageFaultManager *pageFaultHandler, void *allocPtr, NEO::PageFaultManager::PageFaultData &pageFaultData);
void transferAndUnprotectMemoryWithHints(NEO::CpuPageFaultManager *pageFaultHandler, void *allocPtr, NEO::CpuPageFaultManager::PageFaultData &pageFaultData);
} // namespace L0

14
level_zero/core/source/memory/cpu_page_fault_memory_manager.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2024 Intel Corporation
* Copyright (C) 2020-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -16,7 +16,7 @@
#include "level_zero/core/source/driver/driver_handle_imp.h"
namespace NEO {
void PageFaultManager::transferToCpu(void *ptr, size_t size, void *device) {
void CpuPageFaultManager::transferToCpu(void *ptr, size_t size, void *device) {
L0::DeviceImp *deviceImp = static_cast<L0::DeviceImp *>(device);
deviceImp->getNEODevice()->stopDirectSubmissionForCopyEngine();
@ -29,7 +29,7 @@ void PageFaultManager::transferToCpu(void *ptr, size_t size, void *device) {
allocData->size, true);
UNRECOVERABLE_IF(ret);
}
void PageFaultManager::transferToGpu(void *ptr, void *device) {
void CpuPageFaultManager::transferToGpu(void *ptr, void *device) {
L0::DeviceImp *deviceImp = static_cast<L0::DeviceImp *>(device);
deviceImp->getNEODevice()->stopDirectSubmissionForCopyEngine();
@ -44,7 +44,7 @@ void PageFaultManager::transferToGpu(void *ptr, void *device) {
this->evictMemoryAfterImplCopy(allocData->cpuAllocation, deviceImp->getNEODevice());
}
void PageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
void CpuPageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
L0::DeviceImp *deviceImp = static_cast<L0::DeviceImp *>(pageFaultData.cmdQ);
CommandStreamReceiver *csr = nullptr;
@ -61,9 +61,9 @@ void PageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultDa
} // namespace NEO
namespace L0 {
void transferAndUnprotectMemoryWithHints(NEO::PageFaultManager *pageFaultHandler, void *allocPtr, NEO::PageFaultManager::PageFaultData &pageFaultData) {
void transferAndUnprotectMemoryWithHints(NEO::CpuPageFaultManager *pageFaultHandler, void *allocPtr, NEO::CpuPageFaultManager::PageFaultData &pageFaultData) {
bool migration = true;
if (pageFaultData.domain == NEO::PageFaultManager::AllocationDomain::gpu) {
if (pageFaultData.domain == NEO::CpuPageFaultManager::AllocationDomain::gpu) {
L0::DeviceImp *deviceImp = static_cast<L0::DeviceImp *>(pageFaultData.cmdQ);
NEO::SvmAllocationData *allocData = deviceImp->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(allocPtr);
@ -87,7 +87,7 @@ void transferAndUnprotectMemoryWithHints(NEO::PageFaultManager *pageFaultHandler
}
}
if (migration) {
pageFaultData.domain = NEO::PageFaultManager::AllocationDomain::cpu;
pageFaultData.domain = NEO::CpuPageFaultManager::AllocationDomain::cpu;
}
pageFaultHandler->allowCPUMemoryAccess(allocPtr, pageFaultData.size);
}

28
level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2024 Intel Corporation
* Copyright (C) 2020-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -545,9 +545,9 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
EXPECT_EQ(handlerWithHints, reinterpret_cast<void *>(mockPageFaultManager->gpuDomainHandler));
NEO::PageFaultManager::PageFaultData pageData;
NEO::CpuPageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
pageData.domain = NEO::PageFaultManager::AllocationDomain::gpu;
pageData.domain = NEO::CpuPageFaultManager::AllocationDomain::gpu;
mockPageFaultManager->gpuDomainHandler(mockPageFaultManager, ptr, pageData);
flags = deviceImp->memAdviseSharedAllocations[allocData];
EXPECT_EQ(1, flags.cpuMigrationBlocked);
@ -586,9 +586,9 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
EXPECT_EQ(handlerWithHints, reinterpret_cast<void *>(mockPageFaultManager->gpuDomainHandler));
NEO::PageFaultManager::PageFaultData pageData;
NEO::CpuPageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
pageData.domain = NEO::PageFaultManager::AllocationDomain::gpu;
pageData.domain = NEO::CpuPageFaultManager::AllocationDomain::gpu;
pageData.unifiedMemoryManager = device->getDriverHandle()->getSvmAllocsManager();
EXPECT_EQ(0u, device->getDriverHandle()->getSvmAllocsManager()->nonGpuDomainAllocs.size());
mockPageFaultManager->gpuDomainHandler(mockPageFaultManager, ptr, pageData);
@ -661,9 +661,9 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
testing::internal::CaptureStdout(); // start capturing
NEO::PageFaultManager::PageFaultData pageData;
NEO::CpuPageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
pageData.domain = NEO::PageFaultManager::AllocationDomain::gpu;
pageData.domain = NEO::CpuPageFaultManager::AllocationDomain::gpu;
pageData.unifiedMemoryManager = device->getDriverHandle()->getSvmAllocsManager();
mockPageFaultManager->gpuDomainHandler(mockPageFaultManager, ptr, pageData);
flags = deviceImp->memAdviseSharedAllocations[allocData];
@ -715,9 +715,9 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
EXPECT_EQ(handlerWithHints, reinterpret_cast<void *>(mockPageFaultManager->gpuDomainHandler));
NEO::PageFaultManager::PageFaultData pageData;
NEO::CpuPageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
pageData.domain = NEO::PageFaultManager::AllocationDomain::gpu;
pageData.domain = NEO::CpuPageFaultManager::AllocationDomain::gpu;
pageData.unifiedMemoryManager = device->getDriverHandle()->getSvmAllocsManager();
mockPageFaultManager->gpuDomainHandler(mockPageFaultManager, ptr, pageData);
flags = deviceImp->memAdviseSharedAllocations[allocData];
@ -762,9 +762,9 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
EXPECT_EQ(handlerWithHints, reinterpret_cast<void *>(mockPageFaultManager->gpuDomainHandler));
NEO::PageFaultManager::PageFaultData pageData;
NEO::CpuPageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
pageData.domain = NEO::PageFaultManager::AllocationDomain::cpu;
pageData.domain = NEO::CpuPageFaultManager::AllocationDomain::cpu;
pageData.unifiedMemoryManager = device->getDriverHandle()->getSvmAllocsManager();
mockPageFaultManager->gpuDomainHandler(mockPageFaultManager, ptr, pageData);
flags = deviceImp->memAdviseSharedAllocations[allocData];
@ -809,9 +809,9 @@ TEST_F(CommandListMemAdvisePageFault, givenInvalidPtrAndPageFaultHandlerAndGpuDo
EXPECT_EQ(handlerWithHints, reinterpret_cast<void *>(mockPageFaultManager->gpuDomainHandler));
NEO::PageFaultManager::PageFaultData pageData;
NEO::CpuPageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
pageData.domain = NEO::PageFaultManager::AllocationDomain::gpu;
pageData.domain = NEO::CpuPageFaultManager::AllocationDomain::gpu;
pageData.unifiedMemoryManager = device->getDriverHandle()->getSvmAllocsManager();
void *alloc = reinterpret_cast<void *>(0x1);
mockPageFaultManager->gpuDomainHandler(mockPageFaultManager, alloc, pageData);
@ -838,7 +838,7 @@ TEST_F(CommandListMemAdvisePageFault, givenUnifiedMemoryAllocWhenAllowCPUMemoryE
L0::DeviceImp *deviceImp = static_cast<L0::DeviceImp *>((L0::Device::fromHandle(device)));
NEO::PageFaultManager::PageFaultData pageData;
NEO::CpuPageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
mockPageFaultManager->baseAllowCPUMemoryEviction(true, ptr, pageData);

8
opencl/source/memory_manager/cpu_page_fault_manager_memory_sync.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2024 Intel Corporation
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -16,14 +16,14 @@
#include "opencl/source/command_queue/csr_selection_args.h"
namespace NEO {
void PageFaultManager::transferToCpu(void *ptr, size_t size, void *cmdQ) {
void CpuPageFaultManager::transferToCpu(void *ptr, size_t size, void *cmdQ) {
auto commandQueue = static_cast<CommandQueue *>(cmdQ);
commandQueue->getDevice().stopDirectSubmissionForCopyEngine();
auto retVal = commandQueue->enqueueSVMMap(true, CL_MAP_WRITE, ptr, size, 0, nullptr, nullptr, false);
UNRECOVERABLE_IF(retVal);
}
void PageFaultManager::transferToGpu(void *ptr, void *cmdQ) {
void CpuPageFaultManager::transferToGpu(void *ptr, void *cmdQ) {
auto commandQueue = static_cast<CommandQueue *>(cmdQ);
commandQueue->getDevice().stopDirectSubmissionForCopyEngine();
@ -37,7 +37,7 @@ void PageFaultManager::transferToGpu(void *ptr, void *cmdQ) {
UNRECOVERABLE_IF(allocData == nullptr);
this->evictMemoryAfterImplCopy(allocData->cpuAllocation, &commandQueue->getDevice());
}
void PageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
void CpuPageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
auto commandQueue = static_cast<CommandQueue *>(pageFaultData.cmdQ);
auto allocData = memoryData[ptr].unifiedMemoryManager->getSVMAlloc(ptr);

4
opencl/test/unit_test/memory_manager/cpu_page_fault_manager_memory_sync_tests.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2024 Intel Corporation
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -121,7 +121,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenAllowCPUMemoryEvictionIs
cmdQ->device = device.get();
pageFaultManager->insertAllocation(alloc, 256, svmAllocsManager.get(), cmdQ.get(), {});
NEO::PageFaultManager::PageFaultData pageData;
NEO::CpuPageFaultManager::PageFaultData pageData;
pageData.cmdQ = cmdQ.get();
pageFaultManager->baseAllowCPUMemoryEviction(true, alloc, pageData);

13
shared/source/command_stream/tbx_command_stream_receiver_hw.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2024 Intel Corporation
* Copyright (C) 2018-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -18,6 +18,7 @@ namespace NEO {
class AubSubCaptureManager;
class TbxStream;
class CpuPageFaultManager;
template <typename GfxFamily>
class TbxCommandStreamReceiverHw : public CommandStreamReceiverSimulatedHw<GfxFamily> {
@ -36,6 +37,12 @@ class TbxCommandStreamReceiverHw : public CommandStreamReceiverSimulatedHw<GfxFa
return 2000; // 2s
}
bool isAllocTbxFaultable(GraphicsAllocation *gfxAlloc);
void registerAllocationWithTbxFaultMngrIfTbxFaultable(GraphicsAllocation *gfxAllocation, void *cpuAddress, size_t size);
void allowCPUMemoryAccessIfTbxFaultable(GraphicsAllocation *gfxAllocation, void *cpuAddress, size_t size);
void protectCPUMemoryAccessIfTbxFaultable(GraphicsAllocation *gfxAllocation, void *cpuAddress, size_t size);
void protectCPUMemoryFromWritesIfTbxFaultable(GraphicsAllocation *gfxAllocation, void *cpuAddress, size_t size);
public:
using CommandStreamReceiverSimulatedCommonHw<GfxFamily>::initAdditionalMMIO;
using CommandStreamReceiverSimulatedCommonHw<GfxFamily>::aubManager;
@ -81,10 +88,12 @@ class TbxCommandStreamReceiverHw : public CommandStreamReceiverSimulatedHw<GfxFa
void initializeEngine() override;
MemoryManager *getMemoryManager() {
MOCKABLE_VIRTUAL MemoryManager *getMemoryManager() {
return CommandStreamReceiver::getMemoryManager();
}
MOCKABLE_VIRTUAL CpuPageFaultManager *getTbxPageFaultManager();
TbxStream tbxStream;
std::unique_ptr<AubSubCaptureManager> subCaptureManager;
uint32_t aubDeviceId;

95
shared/source/command_stream/tbx_command_stream_receiver_hw.inl
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2024 Intel Corporation
* Copyright (C) 2018-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -26,14 +26,22 @@
#include "shared/source/helpers/gfx_core_helper.h"
#include "shared/source/helpers/hw_info.h"
#include "shared/source/helpers/ptr_math.h"
#include "shared/source/memory_manager/allocation_type.h"
#include "shared/source/memory_manager/graphics_allocation.h"
#include "shared/source/memory_manager/memory_manager.h"
#include "shared/source/os_interface/aub_memory_operations_handler.h"
#include "shared/source/os_interface/product_helper.h"
#include "shared/source/page_fault_manager/tbx_page_fault_manager.h"
#include <cstring>
namespace NEO {
template <typename GfxFamily>
CpuPageFaultManager *TbxCommandStreamReceiverHw<GfxFamily>::getTbxPageFaultManager() {
return this->getMemoryManager()->getPageFaultManager();
}
template <typename GfxFamily>
TbxCommandStreamReceiverHw<GfxFamily>::TbxCommandStreamReceiverHw(ExecutionEnvironment &executionEnvironment,
uint32_t rootDeviceIndex,
@ -74,6 +82,56 @@ TbxCommandStreamReceiverHw<GfxFamily>::~TbxCommandStreamReceiverHw() {
this->freeEngineInfo(gttRemap);
}
template <typename GfxFamily>
bool TbxCommandStreamReceiverHw<GfxFamily>::isAllocTbxFaultable(GraphicsAllocation *gfxAlloc) {
// indicates host memory not managed by the driver
if (gfxAlloc->getDriverAllocatedCpuPtr() == nullptr || !debugManager.isTbxPageFaultManagerEnabled() || this->getTbxPageFaultManager() == nullptr) {
return false;
}
auto allocType = gfxAlloc->getAllocationType();
return allocType == AllocationType::bufferHostMemory;
}
template <typename GfxFamily>
void TbxCommandStreamReceiverHw<GfxFamily>::registerAllocationWithTbxFaultMngrIfTbxFaultable(GraphicsAllocation *gfxAlloc, void *cpuAddress, size_t size) {
if (!isAllocTbxFaultable(gfxAlloc)) {
return;
}
auto bank = this->getMemoryBank(gfxAlloc);
if (bank == 0u || gfxAlloc->storageInfo.cloningOfPageTables) {
bank = GraphicsAllocation::defaultBank;
}
auto faultManager = getTbxPageFaultManager();
faultManager->insertAllocation(this, gfxAlloc, bank, cpuAddress, size);
}
template <typename GfxFamily>
void TbxCommandStreamReceiverHw<GfxFamily>::allowCPUMemoryAccessIfTbxFaultable(GraphicsAllocation *gfxAlloc, void *cpuAddress, size_t size) {
if (!isAllocTbxFaultable(gfxAlloc)) {
return;
}
auto faultManager = getTbxPageFaultManager();
faultManager->allowCPUMemoryAccess(cpuAddress, size);
}
template <typename GfxFamily>
void TbxCommandStreamReceiverHw<GfxFamily>::protectCPUMemoryAccessIfTbxFaultable(GraphicsAllocation *gfxAlloc, void *cpuAddress, size_t size) {
if (!isAllocTbxFaultable(gfxAlloc)) {
return;
}
auto faultManager = getTbxPageFaultManager();
faultManager->protectCPUMemoryAccess(cpuAddress, size);
}
template <typename GfxFamily>
void TbxCommandStreamReceiverHw<GfxFamily>::protectCPUMemoryFromWritesIfTbxFaultable(GraphicsAllocation *gfxAlloc, void *cpuAddress, size_t size) {
if (!isAllocTbxFaultable(gfxAlloc)) {
return;
}
auto faultManager = getTbxPageFaultManager();
faultManager->protectCpuMemoryFromWrites(cpuAddress, size);
}
template <typename GfxFamily>
void TbxCommandStreamReceiverHw<GfxFamily>::initializeEngine() {
isEngineInitialized = true;
@ -431,21 +489,27 @@ void TbxCommandStreamReceiverHw<GfxFamily>::writeMemory(uint64_t gpuAddress, voi
template <typename GfxFamily>
bool TbxCommandStreamReceiverHw<GfxFamily>::writeMemory(GraphicsAllocation &gfxAllocation, bool isChunkCopy, uint64_t gpuVaChunkOffset, size_t chunkSize) {
uint64_t gpuAddress;
void *cpuAddress;
size_t size;
if (!this->getParametersForMemory(gfxAllocation, gpuAddress, cpuAddress, size)) {
return false;
}
auto allocType = gfxAllocation.getAllocationType();
this->registerAllocationWithTbxFaultMngrIfTbxFaultable(&gfxAllocation, cpuAddress, size);
if (!this->isTbxWritable(gfxAllocation)) {
return false;
}
this->allowCPUMemoryAccessIfTbxFaultable(&gfxAllocation, cpuAddress, size);
if (!isEngineInitialized) {
initializeEngine();
}
uint64_t gpuAddress;
void *cpuAddress;
size_t size;
if (!this->getParametersForMemory(gfxAllocation, gpuAddress, cpuAddress, size)) {
return false;
}
if (aubManager) {
this->writeMemoryWithAubManager(gfxAllocation, isChunkCopy, gpuVaChunkOffset, chunkSize);
} else {
@ -457,9 +521,10 @@ bool TbxCommandStreamReceiverHw<GfxFamily>::writeMemory(GraphicsAllocation &gfxA
writeMemory(gpuAddress, cpuAddress, size, this->getMemoryBank(&gfxAllocation), this->getPPGTTAdditionalBits(&gfxAllocation));
}
if (AubHelper::isOneTimeAubWritableAllocationType(gfxAllocation.getAllocationType())) {
if (AubHelper::isOneTimeAubWritableAllocationType(allocType)) {
this->setTbxWritable(false, gfxAllocation);
}
this->protectCPUMemoryAccessIfTbxFaultable(&gfxAllocation, cpuAddress, size);
return true;
}
@ -533,6 +598,7 @@ void TbxCommandStreamReceiverHw<GfxFamily>::processEviction() {
template <typename GfxFamily>
SubmissionStatus TbxCommandStreamReceiverHw<GfxFamily>::processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId) {
for (auto &gfxAllocation : allocationsForResidency) {
if (dumpTbxNonWritable) {
this->setTbxWritable(true, *gfxAllocation);
@ -554,15 +620,19 @@ SubmissionStatus TbxCommandStreamReceiverHw<GfxFamily>::processResidency(Residen
template <typename GfxFamily>
void TbxCommandStreamReceiverHw<GfxFamily>::downloadAllocationTbx(GraphicsAllocation &gfxAllocation) {
uint64_t gpuAddress = 0;
void *cpuAddress = nullptr;
size_t size = 0;
this->getParametersForMemory(gfxAllocation, gpuAddress, cpuAddress, size);
this->allowCPUMemoryAccessIfTbxFaultable(&gfxAllocation, cpuAddress, size);
if (hardwareContextController) {
hardwareContextController->readMemory(gpuAddress, cpuAddress, size,
this->getMemoryBank(&gfxAllocation), gfxAllocation.getUsedPageSize());
this->protectCPUMemoryFromWritesIfTbxFaultable(&gfxAllocation, cpuAddress, size);
return;
}
@ -573,6 +643,7 @@ void TbxCommandStreamReceiverHw<GfxFamily>::downloadAllocationTbx(GraphicsAlloca
};
ppgtt->pageWalk(static_cast<uintptr_t>(gpuAddress), size, 0, 0, walker, this->getMemoryBank(&gfxAllocation));
}
this->protectCPUMemoryFromWritesIfTbxFaultable(&gfxAllocation, cpuAddress, size);
}
template <typename GfxFamily>
@ -671,6 +742,12 @@ void TbxCommandStreamReceiverHw<GfxFamily>::dumpAllocation(GraphicsAllocation &g
template <typename GfxFamily>
void TbxCommandStreamReceiverHw<GfxFamily>::removeDownloadAllocation(GraphicsAllocation *alloc) {
auto lockCSR = this->obtainUniqueOwnership();
this->allocationsForDownload.erase(alloc);
auto faultManager = getTbxPageFaultManager();
if (faultManager != nullptr) {
faultManager->removeAllocation(alloc);
}
}
} // namespace NEO

10
shared/source/debug_settings/debug_settings_manager.h
View File

@ -1,11 +1,12 @@
/*
* Copyright (C) 2018-2024 Intel Corporation
* Copyright (C) 2018-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "shared/source/helpers/options.h"
#include "shared/source/helpers/string.h"
#include "shared/source/utilities/io_functions.h"
@ -161,6 +162,13 @@ class DebugSettingsManager {
}
}
inline bool isTbxPageFaultManagerEnabled() {
auto setCsr = flags.SetCommandStreamReceiver.get();
auto tbxMngrFlag = flags.EnableTbxPageFaultManager.get();
auto isTbxMode = (setCsr == static_cast<int32_t>(CommandStreamReceiverType::tbx)) || (setCsr == static_cast<int32_t>(CommandStreamReceiverType::tbxWithAub));
return tbxMngrFlag && isTbxMode;
}
protected:
std::unique_ptr<SettingsReader> readerImpl;
bool isLoopAtDriverInitEnabled() const {

1
shared/source/debug_settings/debug_variables_base.inl
View File

@ -41,6 +41,7 @@ DECLARE_DEBUG_VARIABLE(bool, AUBDumpAllocsOnEnqueueSVMMemcpyOnly, false, "Force
DECLARE_DEBUG_VARIABLE(bool, AUBDumpForceAllToLocalMemory, false, "Force placing every allocation in local memory address space")
DECLARE_DEBUG_VARIABLE(bool, GenerateAubFilePerProcessId, true, "Generate aub file with process id")
DECLARE_DEBUG_VARIABLE(bool, SetBufferHostMemoryAlwaysAubWritable, false, "Make buffer host memory allocation always uploaded to AUB/TBX")
DECLARE_DEBUG_VARIABLE(bool, EnableTbxPageFaultManager, false, "Enables experiemental page fault manager for host buffer types, improves upon SetBufferHostMemoryAlwaysAubWritable")
/*DEBUG FLAGS*/
DECLARE_DEBUG_VARIABLE(bool, EnableSWTags, false, "Enable software tagging in batch buffer")

10
shared/source/memory_manager/memory_manager.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2024 Intel Corporation
* Copyright (C) 2018-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -83,9 +83,11 @@ MemoryManager::MemoryManager(ExecutionEnvironment &executionEnvironment) : execu
localMemAllocsSize[rootDeviceIndex].store(0u);
}
if (anyLocalMemorySupported) {
pageFaultManager = PageFaultManager::create();
prefetchManager = PrefetchManager::create();
if (anyLocalMemorySupported || debugManager.isTbxPageFaultManagerEnabled()) {
pageFaultManager = CpuPageFaultManager::create();
if (anyLocalMemorySupported) {
prefetchManager = PrefetchManager::create();
}
}
if (debugManager.flags.EnableMultiStorageResources.get() != -1) {

8
shared/source/memory_manager/memory_manager.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2024 Intel Corporation
* Copyright (C) 2018-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -29,7 +29,7 @@ namespace NEO {
using SubDeviceIdsVec = StackVec<uint32_t, 4>;
class MultiGraphicsAllocation;
class PageFaultManager;
class CpuPageFaultManager;
class GfxPartition;
struct ImageInfo;
struct AllocationData;
@ -192,7 +192,7 @@ class MemoryManager {
return deferredDeleter.get();
}
PageFaultManager *getPageFaultManager() const {
MOCKABLE_VIRTUAL CpuPageFaultManager *getPageFaultManager() const {
return pageFaultManager.get();
}
@ -413,7 +413,7 @@ class MemoryManager {
std::vector<std::unique_ptr<LocalMemoryUsageBankSelector>> internalLocalMemoryUsageBankSelector;
std::vector<std::unique_ptr<LocalMemoryUsageBankSelector>> externalLocalMemoryUsageBankSelector;
void *reservedMemory = nullptr;
std::unique_ptr<PageFaultManager> pageFaultManager;
std::unique_ptr<CpuPageFaultManager> pageFaultManager;
std::unique_ptr<PrefetchManager> prefetchManager;
OSMemory::ReservedCpuAddressRange reservedCpuAddressRange;
std::vector<std::unique_ptr<HeapAssigner>> heapAssigners;

4
shared/source/page_fault_manager/CMakeLists.txt
View File

@ -1,5 +1,5 @@
#
# Copyright (C) 2019-2024 Intel Corporation
# Copyright (C) 2019-2025 Intel Corporation
#
# SPDX-License-Identifier: MIT
#
@ -8,6 +8,8 @@ set(NEO_CORE_PAGE_FAULT_MANAGER
${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
${CMAKE_CURRENT_SOURCE_DIR}/cpu_page_fault_manager.cpp
${CMAKE_CURRENT_SOURCE_DIR}/cpu_page_fault_manager.h
${CMAKE_CURRENT_SOURCE_DIR}/tbx_page_fault_manager.cpp
${CMAKE_CURRENT_SOURCE_DIR}/tbx_page_fault_manager.h
)
set_property(GLOBAL PROPERTY NEO_CORE_PAGE_FAULT_MANAGER ${NEO_CORE_PAGE_FAULT_MANAGER})

69
shared/source/page_fault_manager/cpu_page_fault_manager.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2024 Intel Corporation
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -16,19 +16,25 @@
#include <algorithm>
namespace NEO {
void PageFaultManager::insertAllocation(void *ptr, size_t size, SVMAllocsManager *unifiedMemoryManager, void *cmdQ, const MemoryProperties &memoryProperties) {
void CpuPageFaultManager::insertAllocation(void *ptr, size_t size, SVMAllocsManager *unifiedMemoryManager, void *cmdQ, const MemoryProperties &memoryProperties) {
auto initialPlacement = MemoryPropertiesHelper::getUSMInitialPlacement(memoryProperties);
const auto domain = (initialPlacement == GraphicsAllocation::UsmInitialPlacement::CPU) ? AllocationDomain::cpu : AllocationDomain::none;
std::unique_lock<SpinLock> lock{mtx};
this->memoryData.insert(std::make_pair(ptr, PageFaultData{size, unifiedMemoryManager, cmdQ, domain}));
PageFaultData faultData{};
faultData.size = size;
faultData.unifiedMemoryManager = unifiedMemoryManager;
faultData.cmdQ = cmdQ;
faultData.domain = domain;
this->memoryData.insert(std::make_pair(ptr, faultData));
unifiedMemoryManager->nonGpuDomainAllocs.push_back(ptr);
if (initialPlacement != GraphicsAllocation::UsmInitialPlacement::CPU) {
this->protectCPUMemoryAccess(ptr, size);
}
unifiedMemoryManager->nonGpuDomainAllocs.push_back(ptr);
}
void PageFaultManager::removeAllocation(void *ptr) {
void CpuPageFaultManager::removeAllocation(void *ptr) {
std::unique_lock<SpinLock> lock{mtx};
auto alloc = memoryData.find(ptr);
if (alloc != memoryData.end()) {
@ -45,12 +51,12 @@ void PageFaultManager::removeAllocation(void *ptr) {
}
}
void PageFaultManager::moveAllocationToGpuDomain(void *ptr) {
void CpuPageFaultManager::moveAllocationToGpuDomain(void *ptr) {
std::unique_lock<SpinLock> lock{mtx};
auto alloc = memoryData.find(ptr);
if (alloc != memoryData.end()) {
auto &pageFaultData = alloc->second;
if (pageFaultData.domain != AllocationDomain::gpu) {
if (pageFaultData.domain == AllocationDomain::cpu || pageFaultData.domain == AllocationDomain::none) {
this->migrateStorageToGpuDomain(ptr, pageFaultData);
auto &cpuAllocs = pageFaultData.unifiedMemoryManager->nonGpuDomainAllocs;
@ -61,7 +67,7 @@ void PageFaultManager::moveAllocationToGpuDomain(void *ptr) {
}
}
void PageFaultManager::moveAllocationsWithinUMAllocsManagerToGpuDomain(SVMAllocsManager *unifiedMemoryManager) {
void CpuPageFaultManager::moveAllocationsWithinUMAllocsManagerToGpuDomain(SVMAllocsManager *unifiedMemoryManager) {
std::unique_lock<SpinLock> lock{mtx};
for (auto allocPtr : unifiedMemoryManager->nonGpuDomainAllocs) {
auto &pageFaultData = this->memoryData[allocPtr];
@ -70,7 +76,7 @@ void PageFaultManager::moveAllocationsWithinUMAllocsManagerToGpuDomain(SVMAllocs
unifiedMemoryManager->nonGpuDomainAllocs.clear();
}
inline void PageFaultManager::migrateStorageToGpuDomain(void *ptr, PageFaultData &pageFaultData) {
inline void CpuPageFaultManager::migrateStorageToGpuDomain(void *ptr, PageFaultData &pageFaultData) {
if (pageFaultData.domain == AllocationDomain::cpu) {
this->setCpuAllocEvictable(false, ptr, pageFaultData.unifiedMemoryManager);
this->allowCPUMemoryEviction(false, ptr, pageFaultData);
@ -96,39 +102,40 @@ inline void PageFaultManager::migrateStorageToGpuDomain(void *ptr, PageFaultData
pageFaultData.domain = AllocationDomain::gpu;
}
bool PageFaultManager::verifyAndHandlePageFault(void *ptr, bool handlePageFault) {
std::unique_lock<SpinLock> lock{mtx};
for (auto &alloc : this->memoryData) {
auto allocPtr = alloc.first;
auto &pageFaultData = alloc.second;
if (ptr >= allocPtr && ptr < ptrOffset(allocPtr, pageFaultData.size)) {
if (handlePageFault) {
this->setAubWritable(true, allocPtr, pageFaultData.unifiedMemoryManager);
gpuDomainHandler(this, allocPtr, pageFaultData);
}
return true;
}
}
return false;
void CpuPageFaultManager::handlePageFault(void *ptr, PageFaultData &faultData) {
this->setAubWritable(true, ptr, faultData.unifiedMemoryManager);
gpuDomainHandler(this, ptr, faultData);
}
void PageFaultManager::setGpuDomainHandler(gpuDomainHandlerFunc gpuHandlerFuncPtr) {
bool CpuPageFaultManager::verifyAndHandlePageFault(void *ptr, bool handleFault) {
std::unique_lock<SpinLock> lock{mtx};
auto allocPtr = getFaultData(memoryData, ptr, handleFault);
if (allocPtr == nullptr) {
return false;
}
if (handleFault) {
handlePageFault(allocPtr, memoryData[allocPtr]);
}
return true;
}
void CpuPageFaultManager::setGpuDomainHandler(gpuDomainHandlerFunc gpuHandlerFuncPtr) {
this->gpuDomainHandler = gpuHandlerFuncPtr;
}
void PageFaultManager::transferAndUnprotectMemory(PageFaultManager *pageFaultHandler, void *allocPtr, PageFaultData &pageFaultData) {
void CpuPageFaultManager::transferAndUnprotectMemory(CpuPageFaultManager *pageFaultHandler, void *allocPtr, PageFaultData &pageFaultData) {
pageFaultHandler->migrateStorageToCpuDomain(allocPtr, pageFaultData);
pageFaultHandler->allowCPUMemoryAccess(allocPtr, pageFaultData.size);
pageFaultHandler->setCpuAllocEvictable(true, allocPtr, pageFaultData.unifiedMemoryManager);
pageFaultHandler->allowCPUMemoryEviction(true, allocPtr, pageFaultData);
}
void PageFaultManager::unprotectAndTransferMemory(PageFaultManager *pageFaultHandler, void *allocPtr, PageFaultData &pageFaultData) {
void CpuPageFaultManager::unprotectAndTransferMemory(CpuPageFaultManager *pageFaultHandler, void *allocPtr, PageFaultData &pageFaultData) {
pageFaultHandler->allowCPUMemoryAccess(allocPtr, pageFaultData.size);
pageFaultHandler->migrateStorageToCpuDomain(allocPtr, pageFaultData);
}
inline void PageFaultManager::migrateStorageToCpuDomain(void *ptr, PageFaultData &pageFaultData) {
inline void CpuPageFaultManager::migrateStorageToCpuDomain(void *ptr, PageFaultData &pageFaultData) {
if (pageFaultData.domain == AllocationDomain::gpu) {
std::chrono::steady_clock::time_point start;
std::chrono::steady_clock::time_point end;
@ -144,19 +151,19 @@ inline void PageFaultManager::migrateStorageToCpuDomain(void *ptr, PageFaultData
pageFaultData.domain = AllocationDomain::cpu;
}
void PageFaultManager::selectGpuDomainHandler() {
void CpuPageFaultManager::selectGpuDomainHandler() {
if (debugManager.flags.SetCommandStreamReceiver.get() > static_cast<int32_t>(CommandStreamReceiverType::hardware) || debugManager.flags.NEO_CAL_ENABLED.get()) {
this->gpuDomainHandler = &PageFaultManager::unprotectAndTransferMemory;
this->gpuDomainHandler = &CpuPageFaultManager::unprotectAndTransferMemory;
}
}
void PageFaultManager::setAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
void CpuPageFaultManager::setAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
UNRECOVERABLE_IF(ptr == nullptr);
auto gpuAlloc = unifiedMemoryManager->getSVMAlloc(ptr)->gpuAllocations.getDefaultGraphicsAllocation();
gpuAlloc->setAubWritable(writable, GraphicsAllocation::allBanks);
}
void PageFaultManager::setCpuAllocEvictable(bool evictable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
void CpuPageFaultManager::setCpuAllocEvictable(bool evictable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
UNRECOVERABLE_IF(ptr == nullptr);
auto cpuAlloc = unifiedMemoryManager->getSVMAlloc(ptr)->cpuAllocation;
cpuAlloc->setEvictable(evictable);

60
shared/source/page_fault_manager/cpu_page_fault_manager.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2024 Intel Corporation
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -8,6 +8,7 @@
#pragma once
#include "shared/source/helpers/non_copyable_or_moveable.h"
#include "shared/source/helpers/ptr_math.h"
#include "shared/source/utilities/spinlock.h"
#include <memory>
@ -21,57 +22,80 @@ class Device;
class SVMAllocsManager;
class OSInterface;
class PageFaultManager : public NonCopyableOrMovableClass {
class CpuPageFaultManager : public NonCopyableClass {
public:
static std::unique_ptr<PageFaultManager> create();
static std::unique_ptr<CpuPageFaultManager> create();
virtual ~PageFaultManager() = default;
CpuPageFaultManager() = default;
virtual ~CpuPageFaultManager() = default;
virtual void allowCPUMemoryAccess(void *ptr, size_t size) = 0;
virtual void protectCPUMemoryAccess(void *ptr, size_t size) = 0;
virtual void protectCpuMemoryFromWrites(void *ptr, size_t size) = 0;
MOCKABLE_VIRTUAL void moveAllocationToGpuDomain(void *ptr);
MOCKABLE_VIRTUAL void moveAllocationsWithinUMAllocsManagerToGpuDomain(SVMAllocsManager *unifiedMemoryManager);
void insertAllocation(void *ptr, size_t size, SVMAllocsManager *unifiedMemoryManager, void *cmdQ, const MemoryProperties &memoryProperties);
void removeAllocation(void *ptr);
virtual void insertAllocation(CommandStreamReceiver *csr, GraphicsAllocation *alloc, uint32_t bank, void *ptr, size_t size) {}
virtual void removeAllocation(GraphicsAllocation *alloc) {}
enum class AllocationDomain {
none,
cpu,
gpu,
none
};
struct PageFaultData {
size_t size;
SVMAllocsManager *unifiedMemoryManager;
void *cmdQ;
AllocationDomain domain;
AllocationDomain domain = AllocationDomain::none;
size_t size = 0;
SVMAllocsManager *unifiedMemoryManager = nullptr;
void *cmdQ = nullptr;
};
typedef void (*gpuDomainHandlerFunc)(PageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData);
typedef void (*gpuDomainHandlerFunc)(CpuPageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData);
void setGpuDomainHandler(gpuDomainHandlerFunc gpuHandlerFuncPtr);
virtual void allowCPUMemoryAccess(void *ptr, size_t size) = 0;
virtual void protectCPUMemoryAccess(void *ptr, size_t size) = 0;
MOCKABLE_VIRTUAL void transferToCpu(void *ptr, size_t size, void *cmdQ);
protected:
virtual bool checkFaultHandlerFromPageFaultManager() = 0;
virtual void registerFaultHandler() = 0;
virtual void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) = 0;
virtual void allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) = 0;
MOCKABLE_VIRTUAL bool verifyAndHandlePageFault(void *ptr, bool handlePageFault);
virtual bool checkFaultHandlerFromPageFaultManager() = 0;
virtual void registerFaultHandler() = 0;
virtual bool verifyAndHandlePageFault(void *ptr, bool handlePageFault);
template <class FaultDataType>
void *getFaultData(std::unordered_map<void *, FaultDataType> &memData, void *ptr, bool handleFault) {
for (auto &alloc : memData) {
auto allocPtr = alloc.first;
auto &pageFaultData = alloc.second;
if (ptr >= allocPtr && ptr < ptrOffset(allocPtr, pageFaultData.size)) {
return allocPtr;
}
}
return nullptr;
}
void handlePageFault(void *ptr, PageFaultData &faultData);
MOCKABLE_VIRTUAL void transferToGpu(void *ptr, void *cmdQ);
MOCKABLE_VIRTUAL void setAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager);
MOCKABLE_VIRTUAL void setCpuAllocEvictable(bool evictable, void *ptr, SVMAllocsManager *unifiedMemoryManager);
MOCKABLE_VIRTUAL void allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData);
static void transferAndUnprotectMemory(PageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData);
static void unprotectAndTransferMemory(PageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData);
static void transferAndUnprotectMemory(CpuPageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData);
static void unprotectAndTransferMemory(CpuPageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData);
void selectGpuDomainHandler();
inline void migrateStorageToGpuDomain(void *ptr, PageFaultData &pageFaultData);
inline void migrateStorageToCpuDomain(void *ptr, PageFaultData &pageFaultData);
decltype(&transferAndUnprotectMemory) gpuDomainHandler = &transferAndUnprotectMemory;
using gpuDomainHandlerType = decltype(&transferAndUnprotectMemory);
gpuDomainHandlerType gpuDomainHandler = &transferAndUnprotectMemory;
std::unordered_map<void *, PageFaultData> memoryData;
SpinLock mtx;

4
shared/source/page_fault_manager/linux/CMakeLists.txt
View File

@ -1,5 +1,5 @@
#
# Copyright (C) 2019-2024 Intel Corporation
# Copyright (C) 2019-2025 Intel Corporation
#
# SPDX-License-Identifier: MIT
#
@ -8,6 +8,8 @@ set(NEO_CORE_PAGE_FAULT_MANAGER_LINUX
${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
${CMAKE_CURRENT_SOURCE_DIR}/cpu_page_fault_manager_linux.cpp
${CMAKE_CURRENT_SOURCE_DIR}/cpu_page_fault_manager_linux.h
${CMAKE_CURRENT_SOURCE_DIR}/tbx_page_fault_manager_linux.cpp
${CMAKE_CURRENT_SOURCE_DIR}/tbx_page_fault_manager_linux.h
)
set_property(GLOBAL PROPERTY NEO_CORE_PAGE_FAULT_MANAGER_LINUX ${NEO_CORE_PAGE_FAULT_MANAGER_LINUX})

18
shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2024 Intel Corporation
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -10,15 +10,20 @@
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/device/device.h"
#include "shared/source/execution_environment/root_device_environment.h"
#include "shared/source/helpers/debug_helpers.h"
#include "shared/source/memory_manager/memory_operations_handler.h"
#include "shared/source/page_fault_manager/linux/tbx_page_fault_manager_linux.h"
#include <algorithm>
#include <sys/mman.h>
namespace NEO {
std::unique_ptr<PageFaultManager> PageFaultManager::create() {
auto pageFaultManager = std::make_unique<PageFaultManagerLinux>();
std::unique_ptr<CpuPageFaultManager> CpuPageFaultManager::create() {
auto pageFaultManager = [&]() -> std::unique_ptr<CpuPageFaultManager> {
if (debugManager.isTbxPageFaultManagerEnabled()) {
return TbxPageFaultManager::create();
}
return std::make_unique<PageFaultManagerLinux>();
}();
pageFaultManager->selectGpuDomainHandler();
return pageFaultManager;
@ -90,6 +95,11 @@ void PageFaultManagerLinux::protectCPUMemoryAccess(void *ptr, size_t size) {
UNRECOVERABLE_IF(retVal != 0);
}
void PageFaultManagerLinux::protectCpuMemoryFromWrites(void *ptr, size_t size) {
auto retVal = mprotect(ptr, size, PROT_READ);
UNRECOVERABLE_IF(retVal != 0);
}
void PageFaultManagerLinux::callPreviousHandler(int signal, siginfo_t *info, void *context) {
handlerIndex++;
UNRECOVERABLE_IF(handlerIndex < 0 && handlerIndex >= static_cast<int>(previousPageFaultHandlers.size()));

8
shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2024 Intel Corporation
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -14,7 +14,7 @@
#include <vector>
namespace NEO {
class PageFaultManagerLinux : public PageFaultManager {
class PageFaultManagerLinux : public virtual CpuPageFaultManager {
public:
PageFaultManagerLinux();
~PageFaultManagerLinux() override;
@ -24,6 +24,7 @@ class PageFaultManagerLinux : public PageFaultManager {
protected:
void allowCPUMemoryAccess(void *ptr, size_t size) override;
void protectCPUMemoryAccess(void *ptr, size_t size) override;
void protectCpuMemoryFromWrites(void *ptr, size_t size) override;
void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override;
void allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override;
@ -41,4 +42,7 @@ class PageFaultManagerLinux : public PageFaultManager {
bool evictMemoryAfterCopy = false;
int handlerIndex = 0;
};
class CpuPageFaultManagerLinux final : public PageFaultManagerLinux {};
} // namespace NEO

15
shared/source/page_fault_manager/linux/tbx_page_fault_manager_linux.cpp Normal file
View File

@ -0,0 +1,15 @@
/*
* Copyright (C) 2024-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/page_fault_manager/linux/tbx_page_fault_manager_linux.h"
namespace NEO {
std::unique_ptr<TbxPageFaultManager> TbxPageFaultManager::create() {
return std::make_unique<TbxPageFaultManagerLinux>();
}
} // namespace NEO

20
shared/source/page_fault_manager/linux/tbx_page_fault_manager_linux.h Normal file
View File

@ -0,0 +1,20 @@
/*
* Copyright (C) 2024-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h"
#include "shared/source/page_fault_manager/tbx_page_fault_manager.h"
namespace NEO {
class TbxPageFaultManagerLinux final : public PageFaultManagerLinux, public TbxPageFaultManager {
public:
TbxPageFaultManagerLinux() : PageFaultManagerLinux(), TbxPageFaultManager() {}
};
} // namespace NEO

75
shared/source/page_fault_manager/tbx_page_fault_manager.cpp Normal file
View File

@ -0,0 +1,75 @@
/*
* Copyright (C) 2024-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/page_fault_manager/tbx_page_fault_manager.h"
#include "shared/source/command_stream/command_stream_receiver.h"
#include "shared/source/memory_manager/graphics_allocation.h"
#include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
namespace NEO {
bool TbxPageFaultManager::verifyAndHandlePageFault(void *ptr, bool handleFault) {
std::unique_lock<SpinLock> lock{mtxTbx};
auto allocPtr = getFaultData(memoryDataTbx, ptr, handleFault);
if (allocPtr == nullptr) {
return CpuPageFaultManager::verifyAndHandlePageFault(ptr, handleFault);
}
if (handleFault) {
handlePageFault(allocPtr, memoryDataTbx[allocPtr]);
}
return true;
}
void TbxPageFaultManager::handlePageFault(void *ptr, PageFaultDataTbx &faultData) {
auto &graphicsAllocation = *faultData.gfxAllocation;
auto bank = faultData.bank;
auto hasBeenDownloaded = faultData.hasBeenDownloaded;
auto size = faultData.size;
auto csr = faultData.csr;
if (!hasBeenDownloaded) {
this->allowCPUMemoryAccess(ptr, size);
csr->downloadAllocation(graphicsAllocation);
this->protectCpuMemoryFromWrites(ptr, size);
faultData.hasBeenDownloaded = true;
} else {
graphicsAllocation.setTbxWritable(true, bank);
this->allowCPUMemoryAccess(ptr, size);
this->memoryDataTbx.erase(ptr);
}
}
void TbxPageFaultManager::removeAllocation(GraphicsAllocation *alloc) {
std::unique_lock<SpinLock> lock{mtxTbx};
for (auto &data : memoryDataTbx) {
auto allocPtr = data.first;
auto faultData = data.second;
if (faultData.gfxAllocation == alloc) {
memoryDataTbx.erase(allocPtr);
this->allowCPUMemoryAccess(allocPtr, faultData.size);
return;
}
}
}
void TbxPageFaultManager::insertAllocation(CommandStreamReceiver *csr, GraphicsAllocation *alloc, uint32_t bank, void *ptr, size_t size) {
std::unique_lock<SpinLock> lock{mtxTbx};
if (this->memoryDataTbx.find(ptr) == this->memoryDataTbx.end()) {
PageFaultDataTbx pageFaultData{};
pageFaultData.size = size;
pageFaultData.gfxAllocation = alloc;
pageFaultData.bank = bank;
pageFaultData.csr = csr;
memoryDataTbx[ptr] = pageFaultData;
}
auto &faultData = this->memoryDataTbx[ptr];
faultData.hasBeenDownloaded = false;
this->protectCPUMemoryAccess(ptr, size);
}
} // namespace NEO

44
shared/source/page_fault_manager/tbx_page_fault_manager.h Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (C) 2024-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
namespace NEO {
class CommandStreamReceiver;
class GraphicsAllocation;
class TbxPageFaultManager : public virtual CpuPageFaultManager {
public:
struct PageFaultDataTbx {
size_t size;
bool hasBeenDownloaded = false;
GraphicsAllocation *gfxAllocation = nullptr;
uint32_t bank = 0;
CommandStreamReceiver *csr = nullptr;
};
static std::unique_ptr<TbxPageFaultManager> create();
TbxPageFaultManager() = default;
using CpuPageFaultManager::insertAllocation;
using CpuPageFaultManager::removeAllocation;
void insertAllocation(CommandStreamReceiver *csr, GraphicsAllocation *alloc, uint32_t bank, void *ptr, size_t size) override;
void removeAllocation(GraphicsAllocation *alloc) override;
using CpuPageFaultManager::checkFaultHandlerFromPageFaultManager;
bool verifyAndHandlePageFault(void *ptr, bool handlePageFault) override;
protected:
void handlePageFault(void *ptr, PageFaultDataTbx &faultData);
std::unordered_map<void *, PageFaultDataTbx> memoryDataTbx;
SpinLock mtxTbx;
};
} // namespace NEO

4
shared/source/page_fault_manager/windows/CMakeLists.txt
View File

@ -1,5 +1,5 @@
#
# Copyright (C) 2019-2024 Intel Corporation
# Copyright (C) 2019-2025 Intel Corporation
#
# SPDX-License-Identifier: MIT
#
@ -8,6 +8,8 @@ set(NEO_CORE_PAGE_FAULT_MANAGER_WINDOWS
${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
${CMAKE_CURRENT_SOURCE_DIR}/cpu_page_fault_manager_windows.cpp
${CMAKE_CURRENT_SOURCE_DIR}/cpu_page_fault_manager_windows.h
${CMAKE_CURRENT_SOURCE_DIR}/tbx_page_fault_manager_windows.cpp
${CMAKE_CURRENT_SOURCE_DIR}/tbx_page_fault_manager_windows.h
)
if(WIN32)

24
shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2024 Intel Corporation
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -8,21 +8,27 @@
#include "shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h"
#include "shared/source/command_stream/command_stream_receiver.h"
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/device/device.h"
#include "shared/source/helpers/debug_helpers.h"
#include "shared/source/memory_manager/unified_memory_manager.h"
#include "shared/source/os_interface/os_interface.h"
#include "shared/source/os_interface/windows/os_context_win.h"
#include "shared/source/page_fault_manager/windows/tbx_page_fault_manager_windows.h"
namespace NEO {
std::unique_ptr<PageFaultManager> PageFaultManager::create() {
auto pageFaultManager = std::make_unique<PageFaultManagerWindows>();
std::unique_ptr<CpuPageFaultManager> CpuPageFaultManager::create() {
auto pageFaultManager = [&]() -> std::unique_ptr<CpuPageFaultManager> {
if (debugManager.isTbxPageFaultManagerEnabled()) {
return TbxPageFaultManager::create();
}
return std::make_unique<CpuPageFaultManagerWindows>();
}();
pageFaultManager->selectGpuDomainHandler();
return pageFaultManager;
}
std::function<LONG(struct _EXCEPTION_POINTERS *exceptionInfo)> PageFaultManagerWindows::pageFaultHandler;
std::function<LONG(struct _EXCEPTION_POINTERS *exceptionInfo)> PageFaultManagerWindows::pageFaultHandler = nullptr;
PageFaultManagerWindows::PageFaultManagerWindows() {
PageFaultManagerWindows::registerFaultHandler();
@ -67,10 +73,16 @@ void PageFaultManagerWindows::protectCPUMemoryAccess(void *ptr, size_t size) {
UNRECOVERABLE_IF(!retVal);
}
void PageFaultManagerWindows::protectCpuMemoryFromWrites(void *ptr, size_t size) {
DWORD previousState;
auto retVal = VirtualProtect(ptr, size, PAGE_READONLY, &previousState);
UNRECOVERABLE_IF(!retVal);
}
void PageFaultManagerWindows::evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) {}
void PageFaultManagerWindows::allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) {
NEO::SvmAllocationData *allocData = memoryData[ptr].unifiedMemoryManager->getSVMAlloc(ptr);
NEO::SvmAllocationData *allocData = this->memoryData[ptr].unifiedMemoryManager->getSVMAlloc(ptr);
UNRECOVERABLE_IF(allocData == nullptr);
if (osInterface) {

8
shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2024 Intel Corporation
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -13,7 +13,8 @@
#include <functional>
namespace NEO {
class PageFaultManagerWindows : public PageFaultManager {
class PageFaultManagerWindows : public virtual CpuPageFaultManager {
public:
PageFaultManagerWindows();
~PageFaultManagerWindows() override;
@ -23,6 +24,7 @@ class PageFaultManagerWindows : public PageFaultManager {
protected:
void allowCPUMemoryAccess(void *ptr, size_t size) override;
void protectCPUMemoryAccess(void *ptr, size_t size) override;
void protectCpuMemoryFromWrites(void *ptr, size_t size) override;
void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override;
void allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override;
@ -34,4 +36,6 @@ class PageFaultManagerWindows : public PageFaultManager {
PVOID previousHandler;
};
class CpuPageFaultManagerWindows final : public PageFaultManagerWindows {};
} // namespace NEO

15
shared/source/page_fault_manager/windows/tbx_page_fault_manager_windows.cpp Normal file
View File

@ -0,0 +1,15 @@
/*
* Copyright (C) 2024-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/page_fault_manager/windows/tbx_page_fault_manager_windows.h"
namespace NEO {
std::unique_ptr<TbxPageFaultManager> TbxPageFaultManager::create() {
return std::make_unique<TbxPageFaultManagerWindows>();
}
} // namespace NEO

23
shared/source/page_fault_manager/windows/tbx_page_fault_manager_windows.h Normal file
View File

@ -0,0 +1,23 @@
/*
* Copyright (C) 2024-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "shared/source/page_fault_manager/tbx_page_fault_manager.h"
#include "shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h"
// Known false positive for valid virtual inheritance
#pragma warning(disable : 4250)
namespace NEO {
class TbxPageFaultManagerWindows final : public PageFaultManagerWindows, public TbxPageFaultManager {
public:
TbxPageFaultManagerWindows() : PageFaultManagerWindows(), TbxPageFaultManager() {}
};
} // namespace NEO

6
shared/test/common/fixtures/tbx_command_stream_fixture.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2024 Intel Corporation
* Copyright (C) 2018-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -10,6 +10,7 @@
#include "shared/source/command_stream/command_stream_receiver.h"
#include "shared/source/command_stream/tbx_command_stream_receiver_hw.h"
#include "shared/source/memory_manager/os_agnostic_memory_manager.h"
#include "shared/test/common/helpers/debug_manager_state_restore.h"
#include "shared/test/common/mocks/mock_device.h"
#include "gtest/gtest.h"
@ -18,6 +19,9 @@ namespace NEO {
void TbxCommandStreamFixture::setUp(MockDevice *pDevice) {
// Create our TBX command stream receiver based on HW type
DebugManagerStateRestore dbgRestore;
debugManager.flags.SetCommandStreamReceiver.set(static_cast<int32_t>(CommandStreamReceiverType::tbx));
debugManager.flags.EnableTbxPageFaultManager.set(true);
pCommandStreamReceiver = TbxCommandStreamReceiver::create("", false, *pDevice->executionEnvironment, pDevice->getRootDeviceIndex(), pDevice->getDeviceBitfield());
ASSERT_NE(nullptr, pCommandStreamReceiver);
memoryManager = new OsAgnosticMemoryManager(*pDevice->executionEnvironment);

3
shared/test/common/mocks/mock_allocation_properties.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2024 Intel Corporation
* Copyright (C) 2018-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -19,5 +19,6 @@ struct MockAllocationProperties : public AllocationProperties {
MockAllocationProperties(uint32_t rootDeviceIndex, bool allocateMemory, size_t size, DeviceBitfield deviceBitfield) : AllocationProperties(rootDeviceIndex, allocateMemory, size, AllocationType::internalHostMemory, false, deviceBitfield) {}
MockAllocationProperties(uint32_t rootDeviceIndex, bool allocateMemory, size_t size, AllocationType allocationType) : AllocationProperties(rootDeviceIndex, allocateMemory, size, allocationType, false, mockDeviceBitfield) {}
MockAllocationProperties(uint32_t rootDeviceIndex, bool allocateMemory, size_t size, AllocationType allocationType, DeviceBitfield deviceBitfield) : AllocationProperties(rootDeviceIndex, allocateMemory, size, allocationType, false, deviceBitfield) {}
MockAllocationProperties(uint32_t rootDeviceIndex, AllocationType allocType, size_t size) : AllocationProperties(rootDeviceIndex, true, size, allocType, false, mockDeviceBitfield) {}
};
} // namespace NEO

43
shared/test/common/mocks/mock_cpu_page_fault_manager.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2024 Intel Corporation
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -14,16 +14,17 @@
using namespace NEO;
class MockPageFaultManager : public PageFaultManager {
template <class BaseFaultManager>
class MockPageFaultManagerImpl : public BaseFaultManager {
public:
using PageFaultManager::gpuDomainHandler;
using PageFaultManager::memoryData;
using PageFaultManager::PageFaultData;
using PageFaultManager::PageFaultManager;
using PageFaultManager::selectGpuDomainHandler;
using PageFaultManager::transferAndUnprotectMemory;
using PageFaultManager::unprotectAndTransferMemory;
using PageFaultManager::verifyAndHandlePageFault;
using BaseFaultManager::BaseFaultManager;
using BaseFaultManager::gpuDomainHandler;
using BaseFaultManager::memoryData;
using PageFaultData = typename BaseFaultManager::PageFaultData;
using BaseFaultManager::selectGpuDomainHandler;
using BaseFaultManager::transferAndUnprotectMemory;
using BaseFaultManager::unprotectAndTransferMemory;
using BaseFaultManager::verifyAndHandlePageFault;
bool checkFaultHandlerFromPageFaultManager() override {
checkFaultHandlerCalled++;
@ -42,6 +43,9 @@ class MockPageFaultManager : public PageFaultManager {
protectedMemoryAccessAddress = ptr;
protectedSize = size;
}
void protectCpuMemoryFromWrites(void *ptr, size_t size) override {
protectWritesCalled++;
}
void transferToCpu(void *ptr, size_t size, void *cmdQ) override {
transferToCpuCalled++;
transferToCpuAddress = ptr;
@ -62,19 +66,19 @@ class MockPageFaultManager : public PageFaultManager {
allowCPUMemoryEvictionCalled++;
}
void baseAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
PageFaultManager::setAubWritable(writable, ptr, unifiedMemoryManager);
BaseFaultManager::setAubWritable(writable, ptr, unifiedMemoryManager);
}
void baseCpuTransfer(void *ptr, size_t size, void *cmdQ) {
PageFaultManager::transferToCpu(ptr, size, cmdQ);
BaseFaultManager::transferToCpu(ptr, size, cmdQ);
}
void baseGpuTransfer(void *ptr, void *cmdQ) {
PageFaultManager::transferToGpu(ptr, cmdQ);
BaseFaultManager::transferToGpu(ptr, cmdQ);
}
void baseCpuAllocEvictable(bool evictable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
PageFaultManager::setCpuAllocEvictable(evictable, ptr, unifiedMemoryManager);
BaseFaultManager::setCpuAllocEvictable(evictable, ptr, unifiedMemoryManager);
}
void baseAllowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
PageFaultManager::allowCPUMemoryEviction(evict, ptr, pageFaultData);
BaseFaultManager::allowCPUMemoryEviction(evict, ptr, pageFaultData);
}
void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override {}
@ -85,21 +89,22 @@ class MockPageFaultManager : public PageFaultManager {
}
void *getHwHandlerAddress() {
return reinterpret_cast<void *>(PageFaultManager::transferAndUnprotectMemory);
return reinterpret_cast<void *>(BaseFaultManager::transferAndUnprotectMemory);
}
void *getAubAndTbxHandlerAddress() {
return reinterpret_cast<void *>(PageFaultManager::unprotectAndTransferMemory);
return reinterpret_cast<void *>(BaseFaultManager::unprotectAndTransferMemory);
}
void moveAllocationToGpuDomain(void *ptr) override {
moveAllocationToGpuDomainCalled++;
PageFaultManager::moveAllocationToGpuDomain(ptr);
BaseFaultManager::moveAllocationToGpuDomain(ptr);
}
int checkFaultHandlerCalled = 0;
int registerFaultHandlerCalled = 0;
int allowMemoryAccessCalled = 0;
int protectMemoryCalled = 0;
int protectWritesCalled = 0;
int transferToCpuCalled = 0;
int transferToGpuCalled = 0;
int moveAllocationToGpuDomainCalled = 0;
@ -120,6 +125,8 @@ class MockPageFaultManager : public PageFaultManager {
EngineUsage engineUsage = EngineUsage::engineUsageCount;
};
class MockPageFaultManager : public MockPageFaultManagerImpl<CpuPageFaultManager> {};
template <class T>
class MockPageFaultManagerHandlerInvoke : public T {
public:

7
shared/test/common/mocks/mock_tbx_csr.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2024 Intel Corporation
* Copyright (C) 2018-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -10,6 +10,8 @@
#include "shared/source/command_stream/submission_status.h"
#include "shared/source/command_stream/tbx_command_stream_receiver_hw.h"
#include "shared/source/execution_environment/execution_environment.h"
#include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
#include "shared/source/page_fault_manager/tbx_page_fault_manager.h"
namespace NEO {
@ -18,6 +20,8 @@ class MockTbxCsr : public TbxCommandStreamReceiverHw<GfxFamily> {
public:
using TbxCommandStreamReceiverHw<GfxFamily>::writeMemory;
using TbxCommandStreamReceiverHw<GfxFamily>::allocationsForDownload;
using TbxCommandStreamReceiverHw<GfxFamily>::getParametersForMemory;
using TbxCommandStreamReceiverHw<GfxFamily>::getTbxPageFaultManager;
MockTbxCsr(ExecutionEnvironment &executionEnvironment, const DeviceBitfield deviceBitfield)
: TbxCommandStreamReceiverHw<GfxFamily>(executionEnvironment, 0, deviceBitfield) {
this->downloadAllocationImpl = [this](GraphicsAllocation &gfxAllocation) {
@ -64,6 +68,7 @@ class MockTbxCsr : public TbxCommandStreamReceiverHw<GfxFamily> {
TbxCommandStreamReceiverHw<GfxFamily>::dumpAllocation(gfxAllocation);
dumpAllocationCalled = true;
}
bool initializeEngineCalled = false;
bool writeMemoryWithAubManagerCalled = false;
bool writeMemoryCalled = false;

1
shared/test/common/test_files/igdrcl.config
View File

@ -35,6 +35,7 @@ AUBDumpAllocsOnEnqueueSVMMemcpyOnly = 0
AUBDumpForceAllToLocalMemory = 0
GenerateAubFilePerProcessId = 1
SetBufferHostMemoryAlwaysAubWritable = 0
EnableTbxPageFaultManager = 0
EnableSWTags = 0
DumpSWTagsBXML = 0
ForceDeviceId = unk

267
shared/test/unit_test/command_stream/tbx_command_stream_tests.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2024 Intel Corporation
* Copyright (C) 2018-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -13,9 +13,11 @@
#include "shared/source/helpers/engine_node_helper.h"
#include "shared/source/helpers/gfx_core_helper.h"
#include "shared/source/helpers/hardware_context_controller.h"
#include "shared/source/helpers/options.h"
#include "shared/source/helpers/ptr_math.h"
#include "shared/source/memory_manager/memory_banks.h"
#include "shared/source/memory_manager/os_agnostic_memory_manager.h"
#include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
#include "shared/test/common/fixtures/device_fixture.h"
#include "shared/test/common/fixtures/mock_aub_center_fixture.h"
#include "shared/test/common/fixtures/tbx_command_stream_fixture.h"
@ -1281,3 +1283,266 @@ HWTEST_F(TbxCommandStreamTests, givenTimestampBufferAllocationWhenTbxWriteMemory
memoryManager->freeGraphicsMemory(timestampAllocation);
}
template <typename FamilyType>
class MockTbxCsrForPageFaultTests : public MockTbxCsr<FamilyType> {
public:
using MockTbxCsr<FamilyType>::MockTbxCsr;
CpuPageFaultManager *getTbxPageFaultManager() override {
return this->tbxFaultManager.get();
}
using MockTbxCsr<FamilyType>::isAllocTbxFaultable;
std::unique_ptr<TbxPageFaultManager> tbxFaultManager = TbxPageFaultManager::create();
};
HWTEST_F(TbxCommandStreamTests, givenTbxModeWhenHostWritesHostAllocThenAllocShouldBeDownloadedAndWritable) {
DebugManagerStateRestore stateRestore;
debugManager.flags.SetCommandStreamReceiver.set(static_cast<int32_t>(CommandStreamReceiverType::tbx));
debugManager.flags.EnableTbxPageFaultManager.set(true);
std::unique_ptr<MockTbxCsrForPageFaultTests<FamilyType>> tbxCsr(new MockTbxCsrForPageFaultTests<FamilyType>(*pDevice->executionEnvironment, pDevice->getDeviceBitfield()));
tbxCsr->setupContext(*pDevice->getDefaultEngine().osContext);
EXPECT_TRUE(tbxCsr->tbxFaultManager->checkFaultHandlerFromPageFaultManager());
auto memoryManager = pDevice->getMemoryManager();
NEO::GraphicsAllocation *gfxAlloc1 = memoryManager->allocateGraphicsMemoryWithProperties(
{pDevice->getRootDeviceIndex(),
MemoryConstants::pageSize,
AllocationType::bufferHostMemory,
pDevice->getDeviceBitfield()});
uint64_t gpuAddress;
void *cpuAddress;
size_t size;
EXPECT_TRUE(tbxCsr->getParametersForMemory(*gfxAlloc1, gpuAddress, cpuAddress, size));
tbxCsr->writeMemory(*gfxAlloc1);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
// accessing outside address range does not affect inserted host allocs
auto ptrBelow = (void *)0x0;
EXPECT_FALSE(tbxCsr->tbxFaultManager->verifyAndHandlePageFault(ptrBelow, true));
auto ptrAbove = ptrOffset(cpuAddress, size + 1);
EXPECT_FALSE(tbxCsr->tbxFaultManager->verifyAndHandlePageFault(ptrAbove, true));
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
*reinterpret_cast<char *>(cpuAddress) = 1;
EXPECT_TRUE(tbxCsr->isTbxWritable(*gfxAlloc1));
EXPECT_TRUE(tbxCsr->makeCoherentCalled);
tbxCsr->makeCoherentCalled = false;
tbxCsr->writeMemory(*gfxAlloc1);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
// accessing address with offset that is still in alloc range should
// also make writable and download
reinterpret_cast<char *>(cpuAddress)[1] = 1;
EXPECT_TRUE(tbxCsr->isTbxWritable(*gfxAlloc1));
EXPECT_TRUE(tbxCsr->makeCoherentCalled);
tbxCsr->makeCoherentCalled = false;
// for coverage
tbxCsr->tbxFaultManager->removeAllocation(static_cast<GraphicsAllocation *>(nullptr));
tbxCsr->tbxFaultManager->removeAllocation(gfxAlloc1);
memoryManager->freeGraphicsMemory(gfxAlloc1);
}
HWTEST_F(TbxCommandStreamTests, givenTbxWithModeWhenHostBufferNotWritableAndProtectedThenDownloadShouldNotCrash) {
DebugManagerStateRestore stateRestore;
debugManager.flags.SetCommandStreamReceiver.set(static_cast<int32_t>(CommandStreamReceiverType::tbx));
debugManager.flags.EnableTbxPageFaultManager.set(true);
std::unique_ptr<MockTbxCsrForPageFaultTests<FamilyType>> tbxCsr(new MockTbxCsrForPageFaultTests<FamilyType>(*pDevice->executionEnvironment, pDevice->getDeviceBitfield()));
tbxCsr->setupContext(*pDevice->getDefaultEngine().osContext);
EXPECT_TRUE(tbxCsr->tbxFaultManager->checkFaultHandlerFromPageFaultManager());
auto memoryManager = pDevice->getMemoryManager();
NEO::GraphicsAllocation *gfxAlloc1 = memoryManager->allocateGraphicsMemoryWithProperties(
{pDevice->getRootDeviceIndex(),
MemoryConstants::pageSize,
AllocationType::bufferHostMemory,
pDevice->getDeviceBitfield()});
uint64_t gpuAddress;
void *cpuAddress;
size_t size;
EXPECT_TRUE(tbxCsr->getParametersForMemory(*gfxAlloc1, gpuAddress, cpuAddress, size));
tbxCsr->writeMemory(*gfxAlloc1);
tbxCsr->downloadAllocationTbx(*gfxAlloc1);
EXPECT_TRUE(!tbxCsr->isTbxWritable(*gfxAlloc1));
static_cast<float *>(cpuAddress)[0] = 1.0f;
memoryManager->freeGraphicsMemory(gfxAlloc1);
}
HWTEST_F(TbxCommandStreamTests, givenAllocationWithNoDriverAllocatedCpuPtrThenIsAllocTbxFaultableShouldReturnFalse) {
DebugManagerStateRestore stateRestore;
debugManager.flags.SetCommandStreamReceiver.set(static_cast<int32_t>(CommandStreamReceiverType::tbx));
debugManager.flags.EnableTbxPageFaultManager.set(true);
std::unique_ptr<MockTbxCsrForPageFaultTests<FamilyType>> tbxCsr(new MockTbxCsrForPageFaultTests<FamilyType>(*pDevice->executionEnvironment, pDevice->getDeviceBitfield()));
tbxCsr->setupContext(*pDevice->getDefaultEngine().osContext);
EXPECT_TRUE(tbxCsr->tbxFaultManager->checkFaultHandlerFromPageFaultManager());
auto memoryManager = pDevice->getMemoryManager();
NEO::GraphicsAllocation *gfxAlloc1 = memoryManager->allocateGraphicsMemoryWithProperties(
{pDevice->getRootDeviceIndex(),
MemoryConstants::pageSize,
AllocationType::bufferHostMemory,
pDevice->getDeviceBitfield()});
auto cpuPtr = gfxAlloc1->getDriverAllocatedCpuPtr();
gfxAlloc1->setDriverAllocatedCpuPtr(nullptr);
EXPECT_FALSE(tbxCsr->isAllocTbxFaultable(gfxAlloc1));
gfxAlloc1->setDriverAllocatedCpuPtr(cpuPtr);
memoryManager->freeGraphicsMemory(gfxAlloc1);
}
HWTEST_F(TbxCommandStreamTests, givenTbxModeWhenHostReadsHostAllocThenAllocShouldBeDownloadedButNotWritable) {
DebugManagerStateRestore stateRestore;
debugManager.flags.SetCommandStreamReceiver.set(static_cast<int32_t>(CommandStreamReceiverType::tbx));
debugManager.flags.EnableTbxPageFaultManager.set(true);
std::unique_ptr<MockTbxCsrForPageFaultTests<FamilyType>> tbxCsr(new MockTbxCsrForPageFaultTests<FamilyType>(*pDevice->executionEnvironment, pDevice->getDeviceBitfield()));
tbxCsr->setupContext(*pDevice->getDefaultEngine().osContext);
EXPECT_TRUE(tbxCsr->tbxFaultManager->checkFaultHandlerFromPageFaultManager());
auto memoryManager = pDevice->getMemoryManager();
NEO::GraphicsAllocation *gfxAlloc1 = memoryManager->allocateGraphicsMemoryWithProperties(
{pDevice->getRootDeviceIndex(),
MemoryConstants::pageSize,
AllocationType::bufferHostMemory,
pDevice->getDeviceBitfield()});
uint64_t gpuAddress;
void *cpuAddress;
size_t size;
EXPECT_TRUE(tbxCsr->getParametersForMemory(*gfxAlloc1, gpuAddress, cpuAddress, size));
*reinterpret_cast<char *>(cpuAddress) = 1;
tbxCsr->writeMemory(*gfxAlloc1);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
auto readVal = *reinterpret_cast<char *>(cpuAddress);
EXPECT_EQ(1, readVal);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
EXPECT_TRUE(tbxCsr->makeCoherentCalled);
tbxCsr->makeCoherentCalled = false;
tbxCsr->writeMemory(*gfxAlloc1);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
readVal = *reinterpret_cast<char *>(cpuAddress);
EXPECT_EQ(1, readVal);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
EXPECT_TRUE(tbxCsr->makeCoherentCalled);
tbxCsr->makeCoherentCalled = false;
// for coverage
tbxCsr->tbxFaultManager->removeAllocation(static_cast<GraphicsAllocation *>(nullptr));
tbxCsr->tbxFaultManager->removeAllocation(gfxAlloc1);
memoryManager->freeGraphicsMemory(gfxAlloc1);
}
HWTEST_F(TbxCommandStreamTests, givenTbxModeWhenHandleFaultFalseThenTbxFaultableTypesShouldNotBeHandled) {
DebugManagerStateRestore stateRestore;
debugManager.flags.SetCommandStreamReceiver.set(static_cast<int32_t>(CommandStreamReceiverType::tbx));
debugManager.flags.EnableTbxPageFaultManager.set(true);
std::unique_ptr<MockTbxCsrForPageFaultTests<FamilyType>> tbxCsr(new MockTbxCsrForPageFaultTests<FamilyType>(*pDevice->executionEnvironment, pDevice->getDeviceBitfield()));
tbxCsr->setupContext(*pDevice->getDefaultEngine().osContext);
EXPECT_TRUE(tbxCsr->tbxFaultManager->checkFaultHandlerFromPageFaultManager());
auto memoryManager = pDevice->getMemoryManager();
NEO::GraphicsAllocation *gfxAlloc1 = memoryManager->allocateGraphicsMemoryWithProperties(
{pDevice->getRootDeviceIndex(),
MemoryConstants::pageSize,
AllocationType::bufferHostMemory,
pDevice->getDeviceBitfield()});
uint64_t gpuAddress;
void *cpuAddress;
size_t size;
EXPECT_TRUE(tbxCsr->getParametersForMemory(*gfxAlloc1, gpuAddress, cpuAddress, size));
*reinterpret_cast<char *>(cpuAddress) = 1;
tbxCsr->writeMemory(*gfxAlloc1);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
auto readVal = *reinterpret_cast<char *>(cpuAddress);
EXPECT_EQ(1, readVal);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
EXPECT_TRUE(tbxCsr->makeCoherentCalled);
tbxCsr->makeCoherentCalled = false;
tbxCsr->writeMemory(*gfxAlloc1);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
EXPECT_TRUE(tbxCsr->tbxFaultManager->verifyAndHandlePageFault(cpuAddress, false));
EXPECT_FALSE(tbxCsr->makeCoherentCalled);
tbxCsr->tbxFaultManager->removeAllocation(gfxAlloc1);
memoryManager->freeGraphicsMemory(gfxAlloc1);
}
HWTEST_F(TbxCommandStreamTests, givenTbxModeWhenPageFaultManagerIsDisabledThenIsAllocTbxFaultableShouldReturnFalse) {
DebugManagerStateRestore stateRestore;
debugManager.flags.SetCommandStreamReceiver.set(static_cast<int32_t>(CommandStreamReceiverType::tbx));
debugManager.flags.EnableTbxPageFaultManager.set(false);
std::unique_ptr<MockTbxCsrForPageFaultTests<FamilyType>> tbxCsr(new MockTbxCsrForPageFaultTests<FamilyType>(*pDevice->executionEnvironment, pDevice->getDeviceBitfield()));
tbxCsr->setupContext(*pDevice->getDefaultEngine().osContext);
EXPECT_TRUE(tbxCsr->tbxFaultManager->checkFaultHandlerFromPageFaultManager());
auto memoryManager = pDevice->getMemoryManager();
NEO::GraphicsAllocation *gfxAlloc1 = memoryManager->allocateGraphicsMemoryWithProperties(
{pDevice->getRootDeviceIndex(),
MemoryConstants::pageSize,
AllocationType::bufferHostMemory,
pDevice->getDeviceBitfield()});
EXPECT_FALSE(tbxCsr->isAllocTbxFaultable(gfxAlloc1));
memoryManager->freeGraphicsMemory(gfxAlloc1);
}
HWTEST_F(TbxCommandStreamTests, givenTbxModeWhenPageFaultManagerIsNotAvailableThenIsAllocTbxFaultableShouldReturnFalse) {
DebugManagerStateRestore stateRestore;
debugManager.flags.SetCommandStreamReceiver.set(static_cast<int32_t>(CommandStreamReceiverType::tbx));
debugManager.flags.EnableTbxPageFaultManager.set(false);
std::unique_ptr<MockTbxCsrForPageFaultTests<FamilyType>> tbxCsr(new MockTbxCsrForPageFaultTests<FamilyType>(*pDevice->executionEnvironment, pDevice->getDeviceBitfield()));
tbxCsr->setupContext(*pDevice->getDefaultEngine().osContext);
tbxCsr->tbxFaultManager.reset(nullptr);
auto memoryManager = pDevice->getMemoryManager();
NEO::GraphicsAllocation *gfxAlloc1 = memoryManager->allocateGraphicsMemoryWithProperties(
{pDevice->getRootDeviceIndex(),
MemoryConstants::pageSize,
AllocationType::bufferHostMemory,
pDevice->getDeviceBitfield()});
EXPECT_FALSE(tbxCsr->isAllocTbxFaultable(gfxAlloc1));
memoryManager->freeGraphicsMemory(gfxAlloc1);
}

31
shared/test/unit_test/debug_settings/debug_settings_manager_tests.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2024 Intel Corporation
* Copyright (C) 2018-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -452,3 +452,32 @@ TEST(DurationLogTest, givenDurationGetTimeStringThenTimeStringIsCorrect) {
EXPECT_TRUE(std::isdigit(c) || c == '[' || c == ']' || c == '.' || c == ' ');
}
}
TEST(DebugSettingsManager, GivenTbxOrTbxWithAubCsrTypeAndTbxFaultsEnabledWhenCallingIsTbxMngrEnabledThenReturnTrue) {
DebugManagerStateRestore restorer;
NEO::debugManager.flags.EnableTbxPageFaultManager.set(true);
NEO::debugManager.flags.SetCommandStreamReceiver.set(2);
EXPECT_TRUE(NEO::debugManager.isTbxPageFaultManagerEnabled());
NEO::debugManager.flags.SetCommandStreamReceiver.set(4);
EXPECT_TRUE(NEO::debugManager.isTbxPageFaultManagerEnabled());
}
TEST(DebugSettingsManager, GivenTbxFaultsDisabledWhenCallingIsTbxMngrEnabledThenReturnFalse) {
DebugManagerStateRestore restorer;
NEO::debugManager.flags.EnableTbxPageFaultManager.set(false);
EXPECT_FALSE(NEO::debugManager.isTbxPageFaultManagerEnabled());
}
TEST(DebugSettingsManager, GivenHardwareOrHardwareWithAubCsrTypeAndTbxFaultsEnabledWhenCallingIsTbxMngrEnabledThenReturnFalse) {
DebugManagerStateRestore restorer;
NEO::debugManager.flags.EnableTbxPageFaultManager.set(true);
NEO::debugManager.flags.SetCommandStreamReceiver.set(1);
EXPECT_FALSE(NEO::debugManager.isTbxPageFaultManagerEnabled());
NEO::debugManager.flags.SetCommandStreamReceiver.set(3);
EXPECT_FALSE(NEO::debugManager.isTbxPageFaultManagerEnabled());
}

89
shared/test/unit_test/page_fault_manager/cpu_page_fault_manager_tests.cpp
View File

@ -1,10 +1,12 @@
/*
* Copyright (C) 2019-2024 Intel Corporation
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/helpers/options.h"
#include "shared/source/page_fault_manager/tbx_page_fault_manager.h"
#include "shared/test/common/fixtures/cpu_page_fault_manager_tests_fixture.h"
#include "shared/test/common/helpers/debug_manager_state_restore.h"
#include "shared/test/common/mocks/mock_graphics_allocation.h"
@ -21,7 +23,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocsWhenInsertingAllocsThenAllo
EXPECT_EQ(pageFaultManager->memoryData.size(), 1u);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].size, 10u);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].cmdQ, cmdQ);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].domain, PageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].domain, CpuPageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].unifiedMemoryManager, unifiedMemoryManager.get());
EXPECT_EQ(pageFaultManager->allowMemoryAccessCalled, 0);
EXPECT_EQ(pageFaultManager->protectMemoryCalled, 0);
@ -33,11 +35,11 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocsWhenInsertingAllocsThenAllo
EXPECT_EQ(pageFaultManager->memoryData.size(), 2u);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].size, 10u);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].cmdQ, cmdQ);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].domain, PageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].domain, CpuPageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].unifiedMemoryManager, unifiedMemoryManager.get());
EXPECT_EQ(pageFaultManager->memoryData[alloc2].size, 20u);
EXPECT_EQ(pageFaultManager->memoryData[alloc2].cmdQ, cmdQ);
EXPECT_EQ(pageFaultManager->memoryData[alloc2].domain, PageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData[alloc2].domain, CpuPageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData[alloc2].unifiedMemoryManager, unifiedMemoryManager.get());
EXPECT_EQ(pageFaultManager->allowMemoryAccessCalled, 0);
EXPECT_EQ(pageFaultManager->protectMemoryCalled, 0);
@ -62,7 +64,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenRemovingProtectedAllocTh
EXPECT_EQ(pageFaultManager->memoryData[alloc1].size, 10u);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].unifiedMemoryManager, unifiedMemoryManager.get());
pageFaultManager->memoryData[alloc1].domain = PageFaultManager::AllocationDomain::gpu;
pageFaultManager->memoryData[alloc1].domain = CpuPageFaultManager::AllocationDomain::gpu;
pageFaultManager->removeAllocation(alloc1);
EXPECT_EQ(pageFaultManager->allowMemoryAccessCalled, 1);
@ -91,7 +93,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenRemovingAllocsThenNonGpu
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 1u);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs[0], alloc2);
pageFaultManager->memoryData.at(alloc2).domain = PageFaultManager::AllocationDomain::gpu;
pageFaultManager->memoryData.at(alloc2).domain = CpuPageFaultManager::AllocationDomain::gpu;
pageFaultManager->removeAllocation(alloc2);
EXPECT_EQ(pageFaultManager->allowMemoryAccessCalled, 1);
EXPECT_EQ(pageFaultManager->allowedMemoryAccessAddress, alloc2);
@ -109,7 +111,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenAllocNotPresentInNonGpuD
pageFaultManager->moveAllocationToGpuDomain(alloc1);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 0u);
pageFaultManager->memoryData.at(alloc1).domain = PageFaultManager::AllocationDomain::cpu;
pageFaultManager->memoryData.at(alloc1).domain = CpuPageFaultManager::AllocationDomain::cpu;
pageFaultManager->removeAllocation(alloc1);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 0u);
}
@ -127,7 +129,7 @@ TEST_F(PageFaultManagerTest, givenNonGpuAllocsContainerWhenMovingToGpuDomainMult
pageFaultManager->moveAllocationToGpuDomain(alloc1);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 1u);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs[0], alloc2);
pageFaultManager->memoryData.at(alloc1).domain = PageFaultManager::AllocationDomain::none;
pageFaultManager->memoryData.at(alloc1).domain = CpuPageFaultManager::AllocationDomain::none;
pageFaultManager->moveAllocationToGpuDomain(alloc1);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 1u);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs[0], alloc2);
@ -146,7 +148,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocsWhenMovingToGpuDomainAllocs
pageFaultManager->insertAllocation(alloc3, 30, unifiedMemoryManager.get(), cmdQ, {});
EXPECT_EQ(pageFaultManager->transferToCpuCalled, 0);
EXPECT_EQ(pageFaultManager->memoryData.size(), 3u);
pageFaultManager->memoryData.at(alloc3).domain = PageFaultManager::AllocationDomain::gpu;
pageFaultManager->memoryData.at(alloc3).domain = CpuPageFaultManager::AllocationDomain::gpu;
pageFaultManager->moveAllocationsWithinUMAllocsManagerToGpuDomain(unifiedMemoryManager.get());
@ -166,9 +168,9 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenMovingAllocsOfGivenUMMan
pageFaultManager->insertAllocation(alloc1, 10u, unifiedMemoryManager.get(), nullptr, {});
pageFaultManager->insertAllocation(alloc2, 20u, unifiedMemoryManager.get(), nullptr, {});
pageFaultManager->memoryData.at(alloc2).domain = PageFaultManager::AllocationDomain::gpu;
pageFaultManager->memoryData.at(alloc2).domain = CpuPageFaultManager::AllocationDomain::gpu;
pageFaultManager->moveAllocationsWithinUMAllocsManagerToGpuDomain(unifiedMemoryManager.get());
EXPECT_EQ(pageFaultManager->memoryData.at(alloc1).domain, PageFaultManager::AllocationDomain::gpu);
EXPECT_EQ(pageFaultManager->memoryData.at(alloc1).domain, CpuPageFaultManager::AllocationDomain::gpu);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 0u);
}
@ -187,7 +189,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocsWhenMovingToGpuDomainWithPr
pageFaultManager->insertAllocation(alloc3, 30, unifiedMemoryManager.get(), cmdQ, {});
EXPECT_EQ(pageFaultManager->transferToCpuCalled, 0);
EXPECT_EQ(pageFaultManager->memoryData.size(), 3u);
pageFaultManager->memoryData.at(alloc3).domain = PageFaultManager::AllocationDomain::gpu;
pageFaultManager->memoryData.at(alloc3).domain = CpuPageFaultManager::AllocationDomain::gpu;
testing::internal::CaptureStdout(); // start capturing
@ -219,8 +221,8 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocsWhenMovingToGpuDomainAllocs
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 2u);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs[0], alloc1);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs[1], alloc2);
pageFaultManager->memoryData.at(alloc1).domain = PageFaultManager::AllocationDomain::cpu;
pageFaultManager->memoryData.at(alloc2).domain = PageFaultManager::AllocationDomain::none;
pageFaultManager->memoryData.at(alloc1).domain = CpuPageFaultManager::AllocationDomain::cpu;
pageFaultManager->memoryData.at(alloc2).domain = CpuPageFaultManager::AllocationDomain::none;
pageFaultManager->moveAllocationsWithinUMAllocsManagerToGpuDomain(unifiedMemoryManager.get());
@ -366,13 +368,13 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenMovingToGpuDomainThenUpd
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 2u);
pageFaultManager->moveAllocationToGpuDomain(alloc1);
EXPECT_EQ(pageFaultManager->memoryData.at(alloc1).domain, PageFaultManager::AllocationDomain::gpu);
EXPECT_EQ(pageFaultManager->memoryData.at(alloc1).domain, CpuPageFaultManager::AllocationDomain::gpu);
const auto &allocsVec = unifiedMemoryManager->nonGpuDomainAllocs;
EXPECT_EQ(std::find(allocsVec.cbegin(), allocsVec.cend(), alloc1), allocsVec.cend());
EXPECT_EQ(allocsVec.size(), 1u);
EXPECT_EQ(allocsVec[0], alloc2);
pageFaultManager->memoryData.at(alloc2).domain = PageFaultManager::AllocationDomain::gpu;
pageFaultManager->memoryData.at(alloc2).domain = CpuPageFaultManager::AllocationDomain::gpu;
pageFaultManager->moveAllocationToGpuDomain(alloc2);
EXPECT_NE(std::find(allocsVec.cbegin(), allocsVec.cend(), alloc2), allocsVec.cend());
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 1u);
@ -413,7 +415,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocInGpuDomainWhenMovingToGpuDo
pageFaultManager->insertAllocation(alloc, 10, unifiedMemoryManager.get(), cmdQ, {});
EXPECT_EQ(pageFaultManager->transferToCpuCalled, 0);
EXPECT_EQ(pageFaultManager->memoryData.size(), 1u);
pageFaultManager->memoryData.at(alloc).domain = PageFaultManager::AllocationDomain::gpu;
pageFaultManager->memoryData.at(alloc).domain = CpuPageFaultManager::AllocationDomain::gpu;
pageFaultManager->moveAllocationToGpuDomain(alloc);
EXPECT_EQ(pageFaultManager->allowMemoryAccessCalled, 0);
@ -539,15 +541,15 @@ TEST_F(PageFaultManagerTest, givenAllocsFromCpuDomainWhenVerifyingPageFaultThenD
pageFaultManager->moveAllocationsWithinUMAllocsManagerToGpuDomain(unifiedMemoryManager.get());
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 0u);
pageFaultManager->memoryData.at(alloc2).domain = PageFaultManager::AllocationDomain::none;
pageFaultManager->memoryData.at(alloc2).domain = CpuPageFaultManager::AllocationDomain::none;
pageFaultManager->verifyAndHandlePageFault(alloc2, true);
EXPECT_EQ(pageFaultManager->memoryData.at(alloc2).domain, PageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData.at(alloc2).domain, CpuPageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 0u);
pageFaultManager->gpuDomainHandler = &MockPageFaultManager::unprotectAndTransferMemory;
pageFaultManager->memoryData.at(alloc1).domain = PageFaultManager::AllocationDomain::none;
pageFaultManager->memoryData.at(alloc1).domain = CpuPageFaultManager::AllocationDomain::none;
pageFaultManager->verifyAndHandlePageFault(alloc1, true);
EXPECT_EQ(pageFaultManager->memoryData.at(alloc1).domain, PageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData.at(alloc1).domain, CpuPageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 0u);
}
@ -902,7 +904,7 @@ struct PageFaultManagerTestWithDebugFlag : public ::testing::TestWithParam<uint3
TEST_P(PageFaultManagerTestWithDebugFlag, givenDebugFlagWhenInsertingAllocationThenItOverridesHints) {
DebugManagerStateRestore restore;
debugManager.flags.UsmInitialPlacement.set(GetParam()); // Should be ignored by the driver, when passing hints
const auto expectedDomain = GetParam() == 1 ? PageFaultManager::AllocationDomain::none : PageFaultManager::AllocationDomain::cpu;
const auto expectedDomain = GetParam() == 1 ? CpuPageFaultManager::AllocationDomain::none : CpuPageFaultManager::AllocationDomain::cpu;
void *allocs[] = {
reinterpret_cast<void *>(0x1),
@ -953,24 +955,59 @@ TEST_F(PageFaultManagerTest, givenNoUsmInitialPlacementFlagsWHenInsertingUsmAllo
memoryProperties.allocFlags.usmInitialPlacementCpu = 0;
memoryProperties.allocFlags.usmInitialPlacementGpu = 0;
pageFaultManager->insertAllocation(allocs[0], 10, unifiedMemoryManager.get(), cmdQ, memoryProperties);
EXPECT_EQ(PageFaultManager::AllocationDomain::cpu, pageFaultManager->memoryData.at(allocs[0]).domain);
EXPECT_EQ(CpuPageFaultManager::AllocationDomain::cpu, pageFaultManager->memoryData.at(allocs[0]).domain);
EXPECT_EQ(allocs[0], unifiedMemoryManager->nonGpuDomainAllocs[0]);
memoryProperties.allocFlags.usmInitialPlacementCpu = 0;
memoryProperties.allocFlags.usmInitialPlacementGpu = 1;
pageFaultManager->insertAllocation(allocs[1], 10, unifiedMemoryManager.get(), cmdQ, memoryProperties);
EXPECT_EQ(PageFaultManager::AllocationDomain::none, pageFaultManager->memoryData.at(allocs[1]).domain);
EXPECT_EQ(CpuPageFaultManager::AllocationDomain::none, pageFaultManager->memoryData.at(allocs[1]).domain);
EXPECT_EQ(allocs[1], unifiedMemoryManager->nonGpuDomainAllocs[1]);
memoryProperties.allocFlags.usmInitialPlacementCpu = 1;
memoryProperties.allocFlags.usmInitialPlacementGpu = 0;
pageFaultManager->insertAllocation(allocs[2], 10, unifiedMemoryManager.get(), cmdQ, memoryProperties);
EXPECT_EQ(PageFaultManager::AllocationDomain::cpu, pageFaultManager->memoryData.at(allocs[2]).domain);
EXPECT_EQ(CpuPageFaultManager::AllocationDomain::cpu, pageFaultManager->memoryData.at(allocs[2]).domain);
EXPECT_EQ(allocs[2], unifiedMemoryManager->nonGpuDomainAllocs[2]);
memoryProperties.allocFlags.usmInitialPlacementCpu = 1;
memoryProperties.allocFlags.usmInitialPlacementGpu = 1;
pageFaultManager->insertAllocation(allocs[3], 10, unifiedMemoryManager.get(), cmdQ, memoryProperties);
EXPECT_EQ(PageFaultManager::AllocationDomain::cpu, pageFaultManager->memoryData.at(allocs[3]).domain);
EXPECT_EQ(CpuPageFaultManager::AllocationDomain::cpu, pageFaultManager->memoryData.at(allocs[3]).domain);
EXPECT_EQ(allocs[3], unifiedMemoryManager->nonGpuDomainAllocs[3]);
}
TEST_F(PageFaultManagerTest, givenTbxModeWhenAllocateSharedMemoryThenTbxFaultManagerShouldBehaveLikeCpuFaultManager) {
auto memoryManager2 = std::make_unique<MockMemoryManager>(executionEnvironment);
auto unifiedMemoryManager2 = std::make_unique<SVMAllocsManager>(memoryManager2.get(), false);
auto pageFaultManager2 = std::make_unique<MockPageFaultManagerImpl<TbxPageFaultManager>>();
void *cmdQ = reinterpret_cast<void *>(0xFFFF);
RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex};
std::map<uint32_t, DeviceBitfield> deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}};
auto properties = SVMAllocsManager::UnifiedMemoryProperties(InternalMemoryType::sharedUnifiedMemory, 1, rootDeviceIndices, deviceBitfields);
void *ptr = unifiedMemoryManager2->createSharedUnifiedMemoryAllocation(10, properties, cmdQ);
pageFaultManager2->insertAllocation(ptr, 10, unifiedMemoryManager2.get(), cmdQ, {});
EXPECT_TRUE(pageFaultManager2->verifyAndHandlePageFault(ptr, true));
EXPECT_EQ(pageFaultManager2->protectWritesCalled, 0);
unifiedMemoryManager2->freeSVMAlloc(ptr);
}
TEST_F(PageFaultManagerTest, givenHardwareModeWhenCallTbxInsertOrRemoveApiThenNothing) {
auto pageFaultManager2 = std::make_unique<MockPageFaultManagerImpl<CpuPageFaultManager>>();
auto ptr = reinterpret_cast<void *>(0XFFFF);
auto gfxAlloc = reinterpret_cast<GraphicsAllocation *>(0xFFFF);
auto csr = reinterpret_cast<CommandStreamReceiver *>(0xFFFF);
pageFaultManager2->insertAllocation(csr, gfxAlloc, 0, ptr, 0);
EXPECT_EQ(pageFaultManager2->memoryData.find(ptr), pageFaultManager2->memoryData.end());
pageFaultManager2->memoryData[ptr] = {};
pageFaultManager2->removeAllocation(gfxAlloc);
EXPECT_FALSE(pageFaultManager2->memoryData.find(ptr) == pageFaultManager2->memoryData.end());
pageFaultManager2->memoryData.erase(ptr);
}

4
shared/test/unit_test/page_fault_manager/windows/cpu_page_fault_manager_windows_tests.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2024 Intel Corporation
* Copyright (C) 2019-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -156,4 +156,4 @@ TEST_F(PageFaultManagerTest,
EXPECT_EQ(0u, csr->getEvictionAllocations().size());
unifiedMemoryManager->freeSVMAlloc(ptr);
}
}

6
shared/test/unit_test/ult_specific_config.cpp
View File

@ -29,11 +29,11 @@ const char *neoMockSettingsFileName = "neo_mock.config";
bool CompressionSelector::preferCompressedAllocation(const AllocationProperties &properties) {
return false;
}
void PageFaultManager::transferToCpu(void *ptr, size_t size, void *cmdQ) {
void CpuPageFaultManager::transferToCpu(void *ptr, size_t size, void *cmdQ) {
}
void PageFaultManager::transferToGpu(void *ptr, void *cmdQ) {
void CpuPageFaultManager::transferToGpu(void *ptr, void *cmdQ) {
}
void PageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
void CpuPageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
}
void RootDeviceEnvironment::initApiGfxCoreHelper() {