intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-12-20 00:24:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert "fix: regression caused by tbx fault mngr"

Browse Source 
This reverts commit 9a14fe2478.

Signed-off-by: Compute-Runtime-Validation <compute-runtime-validation@intel.com>
This commit is contained in:
Compute-Runtime-Validation
2024-12-19 16:09:50 +01:00
committed by Compute-Runtime-Automation
parent 136e5b4f6c
commit 124e755b9d
34 changed files with 131 additions and 757 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
level_zero/core/source/device/device_imp.h
View File

@@ -196,6 +196,6 @@ struct DeviceImp : public Device, NEO::NonCopyableOrMovableClass {
std::unique_ptr<DebugSession> debugSession;
};
void transferAndUnprotectMemoryWithHints(NEO::CpuPageFaultManager *pageFaultHandler, void *allocPtr, NEO::CpuPageFaultManager::PageFaultData &pageFaultData);
void transferAndUnprotectMemoryWithHints(NEO::PageFaultManager *pageFaultHandler, void *allocPtr, NEO::PageFaultManager::PageFaultData &pageFaultData);
} // namespace L0

12
level_zero/core/source/memory/cpu_page_fault_memory_manager.cpp
View File

@@ -16,7 +16,7 @@
#include "level_zero/core/source/driver/driver_handle_imp.h"
namespace NEO {
void CpuPageFaultManager::transferToCpu(void *ptr, size_t size, void *device) {
void PageFaultManager::transferToCpu(void *ptr, size_t size, void *device) {
L0::DeviceImp *deviceImp = static_cast<L0::DeviceImp *>(device);
deviceImp->getNEODevice()->stopDirectSubmissionForCopyEngine();
@@ -29,7 +29,7 @@ void CpuPageFaultManager::transferToCpu(void *ptr, size_t size, void *device) {
allocData->size, true);
UNRECOVERABLE_IF(ret);
}
void CpuPageFaultManager::transferToGpu(void *ptr, void *device) {
void PageFaultManager::transferToGpu(void *ptr, void *device) {
L0::DeviceImp *deviceImp = static_cast<L0::DeviceImp *>(device);
deviceImp->getNEODevice()->stopDirectSubmissionForCopyEngine();
@@ -44,7 +44,7 @@ void CpuPageFaultManager::transferToGpu(void *ptr, void *device) {
this->evictMemoryAfterImplCopy(allocData->cpuAllocation, deviceImp->getNEODevice());
}
void CpuPageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
void PageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
L0::DeviceImp *deviceImp = static_cast<L0::DeviceImp *>(pageFaultData.cmdQ);
CommandStreamReceiver *csr = nullptr;
@@ -61,9 +61,9 @@ void CpuPageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaul
} // namespace NEO
namespace L0 {
void transferAndUnprotectMemoryWithHints(NEO::CpuPageFaultManager *pageFaultHandler, void *allocPtr, NEO::CpuPageFaultManager::PageFaultData &pageFaultData) {
void transferAndUnprotectMemoryWithHints(NEO::PageFaultManager *pageFaultHandler, void *allocPtr, NEO::PageFaultManager::PageFaultData &pageFaultData) {
bool migration = true;
if (pageFaultData.domain == NEO::CpuPageFaultManager::AllocationDomain::gpu) {
if (pageFaultData.domain == NEO::PageFaultManager::AllocationDomain::gpu) {
L0::DeviceImp *deviceImp = static_cast<L0::DeviceImp *>(pageFaultData.cmdQ);
NEO::SvmAllocationData *allocData = deviceImp->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(allocPtr);
@@ -87,7 +87,7 @@ void transferAndUnprotectMemoryWithHints(NEO::CpuPageFaultManager *pageFaultHand
}
}
if (migration) {
pageFaultData.domain = NEO::CpuPageFaultManager::AllocationDomain::cpu;
pageFaultData.domain = NEO::PageFaultManager::AllocationDomain::cpu;
}
pageFaultHandler->allowCPUMemoryAccess(allocPtr, pageFaultData.size);
}

26
level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp
View File

@@ -545,9 +545,9 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
EXPECT_EQ(handlerWithHints, reinterpret_cast<void *>(mockPageFaultManager->gpuDomainHandler));
NEO::CpuPageFaultManager::PageFaultData pageData;
NEO::PageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
pageData.domain = NEO::CpuPageFaultManager::AllocationDomain::gpu;
pageData.domain = NEO::PageFaultManager::AllocationDomain::gpu;
mockPageFaultManager->gpuDomainHandler(mockPageFaultManager, ptr, pageData);
flags = deviceImp->memAdviseSharedAllocations[allocData];
EXPECT_EQ(1, flags.cpuMigrationBlocked);
@@ -586,9 +586,9 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
EXPECT_EQ(handlerWithHints, reinterpret_cast<void *>(mockPageFaultManager->gpuDomainHandler));
NEO::CpuPageFaultManager::PageFaultData pageData;
NEO::PageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
pageData.domain = NEO::CpuPageFaultManager::AllocationDomain::gpu;
pageData.domain = NEO::PageFaultManager::AllocationDomain::gpu;
pageData.unifiedMemoryManager = device->getDriverHandle()->getSvmAllocsManager();
EXPECT_EQ(0u, device->getDriverHandle()->getSvmAllocsManager()->nonGpuDomainAllocs.size());
mockPageFaultManager->gpuDomainHandler(mockPageFaultManager, ptr, pageData);
@@ -661,9 +661,9 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
testing::internal::CaptureStdout(); // start capturing
NEO::CpuPageFaultManager::PageFaultData pageData;
NEO::PageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
pageData.domain = NEO::CpuPageFaultManager::AllocationDomain::gpu;
pageData.domain = NEO::PageFaultManager::AllocationDomain::gpu;
pageData.unifiedMemoryManager = device->getDriverHandle()->getSvmAllocsManager();
mockPageFaultManager->gpuDomainHandler(mockPageFaultManager, ptr, pageData);
flags = deviceImp->memAdviseSharedAllocations[allocData];
@@ -715,9 +715,9 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
EXPECT_EQ(handlerWithHints, reinterpret_cast<void *>(mockPageFaultManager->gpuDomainHandler));
NEO::CpuPageFaultManager::PageFaultData pageData;
NEO::PageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
pageData.domain = NEO::CpuPageFaultManager::AllocationDomain::gpu;
pageData.domain = NEO::PageFaultManager::AllocationDomain::gpu;
pageData.unifiedMemoryManager = device->getDriverHandle()->getSvmAllocsManager();
mockPageFaultManager->gpuDomainHandler(mockPageFaultManager, ptr, pageData);
flags = deviceImp->memAdviseSharedAllocations[allocData];
@@ -762,9 +762,9 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
EXPECT_EQ(handlerWithHints, reinterpret_cast<void *>(mockPageFaultManager->gpuDomainHandler));
NEO::CpuPageFaultManager::PageFaultData pageData;
NEO::PageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
pageData.domain = NEO::CpuPageFaultManager::AllocationDomain::cpu;
pageData.domain = NEO::PageFaultManager::AllocationDomain::cpu;
pageData.unifiedMemoryManager = device->getDriverHandle()->getSvmAllocsManager();
mockPageFaultManager->gpuDomainHandler(mockPageFaultManager, ptr, pageData);
flags = deviceImp->memAdviseSharedAllocations[allocData];
@@ -809,9 +809,9 @@ TEST_F(CommandListMemAdvisePageFault, givenInvalidPtrAndPageFaultHandlerAndGpuDo
EXPECT_EQ(handlerWithHints, reinterpret_cast<void *>(mockPageFaultManager->gpuDomainHandler));
NEO::CpuPageFaultManager::PageFaultData pageData;
NEO::PageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
pageData.domain = NEO::CpuPageFaultManager::AllocationDomain::gpu;
pageData.domain = NEO::PageFaultManager::AllocationDomain::gpu;
pageData.unifiedMemoryManager = device->getDriverHandle()->getSvmAllocsManager();
void *alloc = reinterpret_cast<void *>(0x1);
mockPageFaultManager->gpuDomainHandler(mockPageFaultManager, alloc, pageData);
@@ -838,7 +838,7 @@ TEST_F(CommandListMemAdvisePageFault, givenUnifiedMemoryAllocWhenAllowCPUMemoryE
L0::DeviceImp *deviceImp = static_cast<L0::DeviceImp *>((L0::Device::fromHandle(device)));
NEO::CpuPageFaultManager::PageFaultData pageData;
NEO::PageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
mockPageFaultManager->baseAllowCPUMemoryEviction(true, ptr, pageData);

6
opencl/source/memory_manager/cpu_page_fault_manager_memory_sync.cpp
View File

@@ -16,14 +16,14 @@
#include "opencl/source/command_queue/csr_selection_args.h"
namespace NEO {
void CpuPageFaultManager::transferToCpu(void *ptr, size_t size, void *cmdQ) {
void PageFaultManager::transferToCpu(void *ptr, size_t size, void *cmdQ) {
auto commandQueue = static_cast<CommandQueue *>(cmdQ);
commandQueue->getDevice().stopDirectSubmissionForCopyEngine();
auto retVal = commandQueue->enqueueSVMMap(true, CL_MAP_WRITE, ptr, size, 0, nullptr, nullptr, false);
UNRECOVERABLE_IF(retVal);
}
void CpuPageFaultManager::transferToGpu(void *ptr, void *cmdQ) {
void PageFaultManager::transferToGpu(void *ptr, void *cmdQ) {
auto commandQueue = static_cast<CommandQueue *>(cmdQ);
commandQueue->getDevice().stopDirectSubmissionForCopyEngine();
@@ -37,7 +37,7 @@ void CpuPageFaultManager::transferToGpu(void *ptr, void *cmdQ) {
UNRECOVERABLE_IF(allocData == nullptr);
this->evictMemoryAfterImplCopy(allocData->cpuAllocation, &commandQueue->getDevice());
}
void CpuPageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
void PageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
auto commandQueue = static_cast<CommandQueue *>(pageFaultData.cmdQ);
auto allocData = memoryData[ptr].unifiedMemoryManager->getSVMAlloc(ptr);

2
opencl/test/unit_test/memory_manager/cpu_page_fault_manager_memory_sync_tests.cpp
View File

@@ -121,7 +121,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenAllowCPUMemoryEvictionIs
cmdQ->device = device.get();
pageFaultManager->insertAllocation(alloc, 256, svmAllocsManager.get(), cmdQ.get(), {});
NEO::CpuPageFaultManager::PageFaultData pageData;
NEO::PageFaultManager::PageFaultData pageData;
pageData.cmdQ = cmdQ.get();
pageFaultManager->baseAllowCPUMemoryEviction(true, alloc, pageData);

11
shared/source/command_stream/tbx_command_stream_receiver_hw.h
View File

@@ -18,7 +18,6 @@ namespace NEO {
class AubSubCaptureManager;
class TbxStream;
class CpuPageFaultManager;
template <typename GfxFamily>
class TbxCommandStreamReceiverHw : public CommandStreamReceiverSimulatedHw<GfxFamily> {
@@ -37,12 +36,6 @@ class TbxCommandStreamReceiverHw : public CommandStreamReceiverSimulatedHw<GfxFa
return 2000; // 2s
}
bool isAllocTbxFaultable(GraphicsAllocation *gfxAlloc);
void registerAllocationWithTbxFaultMngrIfTbxFaultable(GraphicsAllocation *gfxAllocation, void *cpuAddress, size_t size);
void allowCPUMemoryAccessIfTbxFaultable(GraphicsAllocation *gfxAllocation, void *cpuAddress, size_t size);
void protectCPUMemoryAccessIfTbxFaultable(GraphicsAllocation *gfxAllocation, void *cpuAddress, size_t size);
void protectCPUMemoryFromWritesIfTbxFaultable(GraphicsAllocation *gfxAllocation, void *cpuAddress, size_t size);
public:
using CommandStreamReceiverSimulatedCommonHw<GfxFamily>::initAdditionalMMIO;
using CommandStreamReceiverSimulatedCommonHw<GfxFamily>::aubManager;
@@ -88,12 +81,10 @@ class TbxCommandStreamReceiverHw : public CommandStreamReceiverSimulatedHw<GfxFa
void initializeEngine() override;
MOCKABLE_VIRTUAL MemoryManager *getMemoryManager() {
MemoryManager *getMemoryManager() {
return CommandStreamReceiver::getMemoryManager();
}
MOCKABLE_VIRTUAL CpuPageFaultManager *getTbxPageFaultManager();
TbxStream tbxStream;
std::unique_ptr<AubSubCaptureManager> subCaptureManager;
uint32_t aubDeviceId;

104
shared/source/command_stream/tbx_command_stream_receiver_hw.inl
View File

@@ -26,22 +26,14 @@
#include "shared/source/helpers/gfx_core_helper.h"
#include "shared/source/helpers/hw_info.h"
#include "shared/source/helpers/ptr_math.h"
#include "shared/source/memory_manager/allocation_type.h"
#include "shared/source/memory_manager/graphics_allocation.h"
#include "shared/source/memory_manager/memory_manager.h"
#include "shared/source/os_interface/aub_memory_operations_handler.h"
#include "shared/source/os_interface/product_helper.h"
#include "shared/source/page_fault_manager/tbx_page_fault_manager.h"
#include <cstring>
namespace NEO {
template <typename GfxFamily>
CpuPageFaultManager *TbxCommandStreamReceiverHw<GfxFamily>::getTbxPageFaultManager() {
return this->getMemoryManager()->getPageFaultManager();
}
template <typename GfxFamily>
TbxCommandStreamReceiverHw<GfxFamily>::TbxCommandStreamReceiverHw(ExecutionEnvironment &executionEnvironment,
uint32_t rootDeviceIndex,
@@ -82,67 +74,6 @@ TbxCommandStreamReceiverHw<GfxFamily>::~TbxCommandStreamReceiverHw() {
this->freeEngineInfo(gttRemap);
}
template <typename GfxFamily>
bool TbxCommandStreamReceiverHw<GfxFamily>::isAllocTbxFaultable(GraphicsAllocation *gfxAlloc) {
// indicates host memory not managed by the driver
if (gfxAlloc->getDriverAllocatedCpuPtr() == nullptr) {
return false;
}
auto allocType = gfxAlloc->getAllocationType();
return allocType == AllocationType::bufferHostMemory;
}
template <typename GfxFamily>
void TbxCommandStreamReceiverHw<GfxFamily>::registerAllocationWithTbxFaultMngrIfTbxFaultable(GraphicsAllocation *gfxAlloc, void *cpuAddress, size_t size) {
if (!isAllocTbxFaultable(gfxAlloc)) {
return;
}
auto bank = this->getMemoryBank(gfxAlloc);
if (bank == 0u || gfxAlloc->storageInfo.cloningOfPageTables) {
bank = GraphicsAllocation::defaultBank;
}
auto faultManager = getTbxPageFaultManager();
if (faultManager == nullptr) {
return;
}
faultManager->insertAllocation(this, gfxAlloc, bank, cpuAddress, size);
}
template <typename GfxFamily>
void TbxCommandStreamReceiverHw<GfxFamily>::allowCPUMemoryAccessIfTbxFaultable(GraphicsAllocation *gfxAlloc, void *cpuAddress, size_t size) {
if (!isAllocTbxFaultable(gfxAlloc)) {
return;
}
auto faultManager = getTbxPageFaultManager();
if (faultManager == nullptr) {
return;
}
faultManager->allowCPUMemoryAccess(cpuAddress, size);
}
template <typename GfxFamily>
void TbxCommandStreamReceiverHw<GfxFamily>::protectCPUMemoryAccessIfTbxFaultable(GraphicsAllocation *gfxAlloc, void *cpuAddress, size_t size) {
if (!isAllocTbxFaultable(gfxAlloc)) {
return;
}
auto faultManager = getTbxPageFaultManager();
if (faultManager == nullptr) {
return;
}
faultManager->protectCPUMemoryAccess(cpuAddress, size);
}
template <typename GfxFamily>
void TbxCommandStreamReceiverHw<GfxFamily>::protectCPUMemoryFromWritesIfTbxFaultable(GraphicsAllocation *gfxAlloc, void *cpuAddress, size_t size) {
if (isAllocTbxFaultable(gfxAlloc)) {
auto faultManager = getTbxPageFaultManager();
if (faultManager == nullptr) {
return;
}
faultManager->protectCpuMemoryFromWrites(cpuAddress, size);
}
}
template <typename GfxFamily>
void TbxCommandStreamReceiverHw<GfxFamily>::initializeEngine() {
isEngineInitialized = true;
@@ -500,27 +431,21 @@ void TbxCommandStreamReceiverHw<GfxFamily>::writeMemory(uint64_t gpuAddress, voi
template <typename GfxFamily>
bool TbxCommandStreamReceiverHw<GfxFamily>::writeMemory(GraphicsAllocation &gfxAllocation, bool isChunkCopy, uint64_t gpuVaChunkOffset, size_t chunkSize) {
uint64_t gpuAddress;
void *cpuAddress;
size_t size;
if (!this->getParametersForMemory(gfxAllocation, gpuAddress, cpuAddress, size)) {
return false;
}
auto allocType = gfxAllocation.getAllocationType();
this->registerAllocationWithTbxFaultMngrIfTbxFaultable(&gfxAllocation, cpuAddress, size);
if (!this->isTbxWritable(gfxAllocation)) {
return false;
}
this->allowCPUMemoryAccessIfTbxFaultable(&gfxAllocation, cpuAddress, size);
if (!isEngineInitialized) {
initializeEngine();
}
uint64_t gpuAddress;
void *cpuAddress;
size_t size;
if (!this->getParametersForMemory(gfxAllocation, gpuAddress, cpuAddress, size)) {
return false;
}
if (aubManager) {
this->writeMemoryWithAubManager(gfxAllocation, isChunkCopy, gpuVaChunkOffset, chunkSize);
} else {
@@ -532,10 +457,9 @@ bool TbxCommandStreamReceiverHw<GfxFamily>::writeMemory(GraphicsAllocation &gfxA
writeMemory(gpuAddress, cpuAddress, size, this->getMemoryBank(&gfxAllocation), this->getPPGTTAdditionalBits(&gfxAllocation));
}
if (AubHelper::isOneTimeAubWritableAllocationType(allocType)) {
if (AubHelper::isOneTimeAubWritableAllocationType(gfxAllocation.getAllocationType())) {
this->setTbxWritable(false, gfxAllocation);
}
this->protectCPUMemoryAccessIfTbxFaultable(&gfxAllocation, cpuAddress, size);
return true;
}
@@ -609,7 +533,6 @@ void TbxCommandStreamReceiverHw<GfxFamily>::processEviction() {
template <typename GfxFamily>
SubmissionStatus TbxCommandStreamReceiverHw<GfxFamily>::processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId) {
for (auto &gfxAllocation : allocationsForResidency) {
if (dumpTbxNonWritable) {
this->setTbxWritable(true, *gfxAllocation);
@@ -631,19 +554,15 @@ SubmissionStatus TbxCommandStreamReceiverHw<GfxFamily>::processResidency(Residen
template <typename GfxFamily>
void TbxCommandStreamReceiverHw<GfxFamily>::downloadAllocationTbx(GraphicsAllocation &gfxAllocation) {
uint64_t gpuAddress = 0;
void *cpuAddress = nullptr;
size_t size = 0;
this->getParametersForMemory(gfxAllocation, gpuAddress, cpuAddress, size);
this->allowCPUMemoryAccessIfTbxFaultable(&gfxAllocation, cpuAddress, size);
if (hardwareContextController) {
hardwareContextController->readMemory(gpuAddress, cpuAddress, size,
this->getMemoryBank(&gfxAllocation), gfxAllocation.getUsedPageSize());
this->protectCPUMemoryFromWritesIfTbxFaultable(&gfxAllocation, cpuAddress, size);
return;
}
@@ -654,7 +573,6 @@ void TbxCommandStreamReceiverHw<GfxFamily>::downloadAllocationTbx(GraphicsAlloca
};
ppgtt->pageWalk(static_cast<uintptr_t>(gpuAddress), size, 0, 0, walker, this->getMemoryBank(&gfxAllocation));
}
this->protectCPUMemoryFromWritesIfTbxFaultable(&gfxAllocation, cpuAddress, size);
}
template <typename GfxFamily>
@@ -753,12 +671,6 @@ void TbxCommandStreamReceiverHw<GfxFamily>::dumpAllocation(GraphicsAllocation &g
template <typename GfxFamily>
void TbxCommandStreamReceiverHw<GfxFamily>::removeDownloadAllocation(GraphicsAllocation *alloc) {
auto lockCSR = this->obtainUniqueOwnership();
this->allocationsForDownload.erase(alloc);
auto faultManager = getTbxPageFaultManager();
if (faultManager != nullptr) {
faultManager->removeAllocation(alloc);
}
}
} // namespace NEO

6
shared/source/debug_settings/debug_settings_manager.h
View File

@@ -6,7 +6,6 @@
*/
#pragma once
#include "shared/source/helpers/options.h"
#include "shared/source/helpers/string.h"
#include "shared/source/utilities/io_functions.h"
@@ -162,11 +161,6 @@ class DebugSettingsManager {
}
}
inline bool isTbxMode() {
auto setCsr = flags.SetCommandStreamReceiver.get();
return (setCsr == static_cast<int32_t>(CommandStreamReceiverType::tbx)) || (setCsr == static_cast<int32_t>(CommandStreamReceiverType::tbxWithAub));
}
protected:
std::unique_ptr<SettingsReader> readerImpl;
bool isLoopAtDriverInitEnabled() const {

8
shared/source/memory_manager/memory_manager.cpp
View File

@@ -83,11 +83,9 @@ MemoryManager::MemoryManager(ExecutionEnvironment &executionEnvironment) : execu
localMemAllocsSize[rootDeviceIndex].store(0u);
}
if (anyLocalMemorySupported || debugManager.isTbxMode()) {
pageFaultManager = CpuPageFaultManager::create();
if (anyLocalMemorySupported) {
prefetchManager = PrefetchManager::create();
}
if (anyLocalMemorySupported) {
pageFaultManager = PageFaultManager::create();
prefetchManager = PrefetchManager::create();
}
if (debugManager.flags.EnableMultiStorageResources.get() != -1) {

6
shared/source/memory_manager/memory_manager.h
View File

@@ -29,7 +29,7 @@ namespace NEO {
using SubDeviceIdsVec = StackVec<uint32_t, 4>;
class MultiGraphicsAllocation;
class CpuPageFaultManager;
class PageFaultManager;
class GfxPartition;
struct ImageInfo;
struct AllocationData;
@@ -192,7 +192,7 @@ class MemoryManager {
return deferredDeleter.get();
}
MOCKABLE_VIRTUAL CpuPageFaultManager *getPageFaultManager() const {
PageFaultManager *getPageFaultManager() const {
return pageFaultManager.get();
}
@@ -413,7 +413,7 @@ class MemoryManager {
std::vector<std::unique_ptr<LocalMemoryUsageBankSelector>> internalLocalMemoryUsageBankSelector;
std::vector<std::unique_ptr<LocalMemoryUsageBankSelector>> externalLocalMemoryUsageBankSelector;
void *reservedMemory = nullptr;
std::unique_ptr<CpuPageFaultManager> pageFaultManager;
std::unique_ptr<PageFaultManager> pageFaultManager;
std::unique_ptr<PrefetchManager> prefetchManager;
OSMemory::ReservedCpuAddressRange reservedCpuAddressRange;
std::vector<std::unique_ptr<HeapAssigner>> heapAssigners;

2
shared/source/page_fault_manager/CMakeLists.txt
View File

@@ -8,8 +8,6 @@ set(NEO_CORE_PAGE_FAULT_MANAGER
${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
${CMAKE_CURRENT_SOURCE_DIR}/cpu_page_fault_manager.cpp
${CMAKE_CURRENT_SOURCE_DIR}/cpu_page_fault_manager.h
${CMAKE_CURRENT_SOURCE_DIR}/tbx_page_fault_manager.cpp
${CMAKE_CURRENT_SOURCE_DIR}/tbx_page_fault_manager.h
)
set_property(GLOBAL PROPERTY NEO_CORE_PAGE_FAULT_MANAGER ${NEO_CORE_PAGE_FAULT_MANAGER})

51
shared/source/page_fault_manager/cpu_page_fault_manager.cpp
View File

@@ -16,26 +16,19 @@
#include <algorithm>
namespace NEO {
void CpuPageFaultManager::insertAllocation(void *ptr, size_t size, SVMAllocsManager *unifiedMemoryManager, void *cmdQ, const MemoryProperties &memoryProperties) {
void PageFaultManager::insertAllocation(void *ptr, size_t size, SVMAllocsManager *unifiedMemoryManager, void *cmdQ, const MemoryProperties &memoryProperties) {
auto initialPlacement = MemoryPropertiesHelper::getUSMInitialPlacement(memoryProperties);
const auto domain = (initialPlacement == GraphicsAllocation::UsmInitialPlacement::CPU) ? AllocationDomain::cpu : AllocationDomain::none;
std::unique_lock<SpinLock> lock{mtx};
PageFaultData faultData{};
faultData.faultType = FaultMode::cpu;
faultData.size = size;
faultData.unifiedMemoryManager = unifiedMemoryManager;
faultData.cmdQ = cmdQ;
faultData.domain = domain;
this->memoryData.insert(std::make_pair(ptr, faultData));
unifiedMemoryManager->nonGpuDomainAllocs.push_back(ptr);
this->memoryData.insert(std::make_pair(ptr, PageFaultData{size, unifiedMemoryManager, cmdQ, domain}));
if (initialPlacement != GraphicsAllocation::UsmInitialPlacement::CPU) {
this->protectCPUMemoryAccess(ptr, size);
}
unifiedMemoryManager->nonGpuDomainAllocs.push_back(ptr);
}
void CpuPageFaultManager::removeAllocation(void *ptr) {
void PageFaultManager::removeAllocation(void *ptr) {
std::unique_lock<SpinLock> lock{mtx};
auto alloc = memoryData.find(ptr);
if (alloc != memoryData.end()) {
@@ -52,7 +45,7 @@ void CpuPageFaultManager::removeAllocation(void *ptr) {
}
}
void CpuPageFaultManager::moveAllocationToGpuDomain(void *ptr) {
void PageFaultManager::moveAllocationToGpuDomain(void *ptr) {
std::unique_lock<SpinLock> lock{mtx};
auto alloc = memoryData.find(ptr);
if (alloc != memoryData.end()) {
@@ -68,7 +61,7 @@ void CpuPageFaultManager::moveAllocationToGpuDomain(void *ptr) {
}
}
void CpuPageFaultManager::moveAllocationsWithinUMAllocsManagerToGpuDomain(SVMAllocsManager *unifiedMemoryManager) {
void PageFaultManager::moveAllocationsWithinUMAllocsManagerToGpuDomain(SVMAllocsManager *unifiedMemoryManager) {
std::unique_lock<SpinLock> lock{mtx};
for (auto allocPtr : unifiedMemoryManager->nonGpuDomainAllocs) {
auto &pageFaultData = this->memoryData[allocPtr];
@@ -77,7 +70,7 @@ void CpuPageFaultManager::moveAllocationsWithinUMAllocsManagerToGpuDomain(SVMAll
unifiedMemoryManager->nonGpuDomainAllocs.clear();
}
inline void CpuPageFaultManager::migrateStorageToGpuDomain(void *ptr, PageFaultData &pageFaultData) {
inline void PageFaultManager::migrateStorageToGpuDomain(void *ptr, PageFaultData &pageFaultData) {
if (pageFaultData.domain == AllocationDomain::cpu) {
this->setCpuAllocEvictable(false, ptr, pageFaultData.unifiedMemoryManager);
this->allowCPUMemoryEviction(false, ptr, pageFaultData);
@@ -103,19 +96,15 @@ inline void CpuPageFaultManager::migrateStorageToGpuDomain(void *ptr, PageFaultD
pageFaultData.domain = AllocationDomain::gpu;
}
void CpuPageFaultManager::handlePageFault(void *ptr, PageFaultData &faultData) {
this->setAubWritable(true, ptr, faultData.unifiedMemoryManager);
gpuDomainHandler(this, ptr, faultData);
}
bool CpuPageFaultManager::verifyAndHandlePageFault(void *ptr, bool handleFault) {
bool PageFaultManager::verifyAndHandlePageFault(void *ptr, bool handlePageFault) {
std::unique_lock<SpinLock> lock{mtx};
for (auto &alloc : memoryData) {
for (auto &alloc : this->memoryData) {
auto allocPtr = alloc.first;
auto &pageFaultData = alloc.second;
if (ptr >= allocPtr && ptr < ptrOffset(allocPtr, pageFaultData.size)) {
if (handleFault) {
handlePageFault(allocPtr, pageFaultData);
if (handlePageFault) {
this->setAubWritable(true, allocPtr, pageFaultData.unifiedMemoryManager);
gpuDomainHandler(this, allocPtr, pageFaultData);
}
return true;
}
@@ -123,23 +112,23 @@ bool CpuPageFaultManager::verifyAndHandlePageFault(void *ptr, bool handleFault)
return false;
}
void CpuPageFaultManager::setGpuDomainHandler(gpuDomainHandlerFunc gpuHandlerFuncPtr) {
void PageFaultManager::setGpuDomainHandler(gpuDomainHandlerFunc gpuHandlerFuncPtr) {
this->gpuDomainHandler = gpuHandlerFuncPtr;
}
void CpuPageFaultManager::transferAndUnprotectMemory(CpuPageFaultManager *pageFaultHandler, void *allocPtr, PageFaultData &pageFaultData) {
void PageFaultManager::transferAndUnprotectMemory(PageFaultManager *pageFaultHandler, void *allocPtr, PageFaultData &pageFaultData) {
pageFaultHandler->migrateStorageToCpuDomain(allocPtr, pageFaultData);
pageFaultHandler->allowCPUMemoryAccess(allocPtr, pageFaultData.size);
pageFaultHandler->setCpuAllocEvictable(true, allocPtr, pageFaultData.unifiedMemoryManager);
pageFaultHandler->allowCPUMemoryEviction(true, allocPtr, pageFaultData);
}
void CpuPageFaultManager::unprotectAndTransferMemory(CpuPageFaultManager *pageFaultHandler, void *allocPtr, PageFaultData &pageFaultData) {
void PageFaultManager::unprotectAndTransferMemory(PageFaultManager *pageFaultHandler, void *allocPtr, PageFaultData &pageFaultData) {
pageFaultHandler->allowCPUMemoryAccess(allocPtr, pageFaultData.size);
pageFaultHandler->migrateStorageToCpuDomain(allocPtr, pageFaultData);
}
inline void CpuPageFaultManager::migrateStorageToCpuDomain(void *ptr, PageFaultData &pageFaultData) {
inline void PageFaultManager::migrateStorageToCpuDomain(void *ptr, PageFaultData &pageFaultData) {
if (pageFaultData.domain == AllocationDomain::gpu) {
std::chrono::steady_clock::time_point start;
std::chrono::steady_clock::time_point end;
@@ -155,19 +144,19 @@ inline void CpuPageFaultManager::migrateStorageToCpuDomain(void *ptr, PageFaultD
pageFaultData.domain = AllocationDomain::cpu;
}
void CpuPageFaultManager::selectGpuDomainHandler() {
void PageFaultManager::selectGpuDomainHandler() {
if (debugManager.flags.SetCommandStreamReceiver.get() > static_cast<int32_t>(CommandStreamReceiverType::hardware) || debugManager.flags.NEO_CAL_ENABLED.get()) {
this->gpuDomainHandler = &CpuPageFaultManager::unprotectAndTransferMemory;
this->gpuDomainHandler = &PageFaultManager::unprotectAndTransferMemory;
}
}
void CpuPageFaultManager::setAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
void PageFaultManager::setAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
UNRECOVERABLE_IF(ptr == nullptr);
auto gpuAlloc = unifiedMemoryManager->getSVMAlloc(ptr)->gpuAllocations.getDefaultGraphicsAllocation();
gpuAlloc->setAubWritable(writable, GraphicsAllocation::allBanks);
}
void CpuPageFaultManager::setCpuAllocEvictable(bool evictable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
void PageFaultManager::setCpuAllocEvictable(bool evictable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
UNRECOVERABLE_IF(ptr == nullptr);
auto cpuAlloc = unifiedMemoryManager->getSVMAlloc(ptr)->cpuAllocation;
cpuAlloc->setEvictable(evictable);

55
shared/source/page_fault_manager/cpu_page_fault_manager.h
View File

@@ -21,27 +21,16 @@ class Device;
class SVMAllocsManager;
class OSInterface;
class CpuPageFaultManager : public NonCopyableClass {
class PageFaultManager : public NonCopyableOrMovableClass {
public:
static std::unique_ptr<CpuPageFaultManager> create();
static std::unique_ptr<PageFaultManager> create();
virtual ~CpuPageFaultManager() = default;
virtual void allowCPUMemoryAccess(void *ptr, size_t size) = 0;
virtual void protectCPUMemoryAccess(void *ptr, size_t size) = 0;
virtual void protectCpuMemoryFromWrites(void *ptr, size_t size) = 0;
virtual ~PageFaultManager() = default;
MOCKABLE_VIRTUAL void moveAllocationToGpuDomain(void *ptr);
MOCKABLE_VIRTUAL void moveAllocationsWithinUMAllocsManagerToGpuDomain(SVMAllocsManager *unifiedMemoryManager);
void insertAllocation(void *ptr, size_t size, SVMAllocsManager *unifiedMemoryManager, void *cmdQ, const MemoryProperties &memoryProperties);
void removeAllocation(void *ptr);
virtual void insertAllocation(CommandStreamReceiver *csr, GraphicsAllocation *alloc, uint32_t bank, void *ptr, size_t size) {}
virtual void removeAllocation(GraphicsAllocation *alloc) {}
enum class FaultMode {
cpu,
tbx
};
enum class AllocationDomain {
none,
@@ -50,51 +39,39 @@ class CpuPageFaultManager : public NonCopyableClass {
};
struct PageFaultData {
FaultMode faultType = FaultMode::cpu;
size_t size = 0;
// cpu fault data
SVMAllocsManager *unifiedMemoryManager = nullptr;
void *cmdQ = nullptr;
AllocationDomain domain = AllocationDomain::none;
// tbx fault data
bool hasBeenDownloaded = false;
GraphicsAllocation *gfxAllocation = nullptr;
uint32_t bank = 0;
CommandStreamReceiver *csr = nullptr;
size_t size;
SVMAllocsManager *unifiedMemoryManager;
void *cmdQ;
AllocationDomain domain;
};
typedef void (*gpuDomainHandlerFunc)(CpuPageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData);
typedef void (*gpuDomainHandlerFunc)(PageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData);
void setGpuDomainHandler(gpuDomainHandlerFunc gpuHandlerFuncPtr);
virtual void allowCPUMemoryAccess(void *ptr, size_t size) = 0;
virtual void protectCPUMemoryAccess(void *ptr, size_t size) = 0;
MOCKABLE_VIRTUAL void transferToCpu(void *ptr, size_t size, void *cmdQ);
protected:
virtual bool checkFaultHandlerFromPageFaultManager() = 0;
virtual void registerFaultHandler() = 0;
virtual void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) = 0;
virtual void allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) = 0;
virtual bool checkFaultHandlerFromPageFaultManager() = 0;
virtual void registerFaultHandler() = 0;
virtual bool verifyAndHandlePageFault(void *ptr, bool handlePageFault);
virtual void handlePageFault(void *ptr, PageFaultData &faultData);
MOCKABLE_VIRTUAL bool verifyAndHandlePageFault(void *ptr, bool handlePageFault);
MOCKABLE_VIRTUAL void transferToGpu(void *ptr, void *cmdQ);
MOCKABLE_VIRTUAL void setAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager);
MOCKABLE_VIRTUAL void setCpuAllocEvictable(bool evictable, void *ptr, SVMAllocsManager *unifiedMemoryManager);
MOCKABLE_VIRTUAL void allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData);
static void transferAndUnprotectMemory(CpuPageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData);
static void unprotectAndTransferMemory(CpuPageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData);
static void transferAndUnprotectMemory(PageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData);
static void unprotectAndTransferMemory(PageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData);
void selectGpuDomainHandler();
inline void migrateStorageToGpuDomain(void *ptr, PageFaultData &pageFaultData);
inline void migrateStorageToCpuDomain(void *ptr, PageFaultData &pageFaultData);
using gpuDomainHandlerType = decltype(&transferAndUnprotectMemory);
gpuDomainHandlerType gpuDomainHandler = &transferAndUnprotectMemory;
decltype(&transferAndUnprotectMemory) gpuDomainHandler = &transferAndUnprotectMemory;
std::unordered_map<void *, PageFaultData> memoryData;
SpinLock mtx;

2
shared/source/page_fault_manager/linux/CMakeLists.txt
View File

@@ -8,8 +8,6 @@ set(NEO_CORE_PAGE_FAULT_MANAGER_LINUX
${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
${CMAKE_CURRENT_SOURCE_DIR}/cpu_page_fault_manager_linux.cpp
${CMAKE_CURRENT_SOURCE_DIR}/cpu_page_fault_manager_linux.h
${CMAKE_CURRENT_SOURCE_DIR}/tbx_page_fault_manager_linux.cpp
${CMAKE_CURRENT_SOURCE_DIR}/tbx_page_fault_manager_linux.h
)
set_property(GLOBAL PROPERTY NEO_CORE_PAGE_FAULT_MANAGER_LINUX ${NEO_CORE_PAGE_FAULT_MANAGER_LINUX})

16
shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.cpp
View File

@@ -10,20 +10,15 @@
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/device/device.h"
#include "shared/source/execution_environment/root_device_environment.h"
#include "shared/source/helpers/debug_helpers.h"
#include "shared/source/memory_manager/memory_operations_handler.h"
#include "shared/source/page_fault_manager/linux/tbx_page_fault_manager_linux.h"
#include <algorithm>
#include <sys/mman.h>
namespace NEO {
std::unique_ptr<CpuPageFaultManager> CpuPageFaultManager::create() {
auto pageFaultManager = [&]() -> std::unique_ptr<CpuPageFaultManager> {
if (debugManager.isTbxMode()) {
return TbxPageFaultManager::create();
}
return std::make_unique<PageFaultManagerLinux>();
}();
std::unique_ptr<PageFaultManager> PageFaultManager::create() {
auto pageFaultManager = std::make_unique<PageFaultManagerLinux>();
pageFaultManager->selectGpuDomainHandler();
return pageFaultManager;
@@ -95,11 +90,6 @@ void PageFaultManagerLinux::protectCPUMemoryAccess(void *ptr, size_t size) {
UNRECOVERABLE_IF(retVal != 0);
}
void PageFaultManagerLinux::protectCpuMemoryFromWrites(void *ptr, size_t size) {
auto retVal = mprotect(ptr, size, PROT_READ);
UNRECOVERABLE_IF(retVal != 0);
}
void PageFaultManagerLinux::callPreviousHandler(int signal, siginfo_t *info, void *context) {
handlerIndex++;
UNRECOVERABLE_IF(handlerIndex < 0 && handlerIndex >= static_cast<int>(previousPageFaultHandlers.size()));

6
shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h
View File

@@ -14,7 +14,7 @@
#include <vector>
namespace NEO {
class PageFaultManagerLinux : public virtual CpuPageFaultManager {
class PageFaultManagerLinux : public PageFaultManager {
public:
PageFaultManagerLinux();
~PageFaultManagerLinux() override;
@@ -24,7 +24,6 @@ class PageFaultManagerLinux : public virtual CpuPageFaultManager {
protected:
void allowCPUMemoryAccess(void *ptr, size_t size) override;
void protectCPUMemoryAccess(void *ptr, size_t size) override;
void protectCpuMemoryFromWrites(void *ptr, size_t size) override;
void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override;
void allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override;
@@ -42,7 +41,4 @@ class PageFaultManagerLinux : public virtual CpuPageFaultManager {
bool evictMemoryAfterCopy = false;
int handlerIndex = 0;
};
class CpuPageFaultManagerLinux final : public PageFaultManagerLinux {};
} // namespace NEO

18
shared/source/page_fault_manager/linux/tbx_page_fault_manager_linux.cpp
View File

@@ -1,18 +0,0 @@
/*
* Copyright (C) 2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/page_fault_manager/linux/tbx_page_fault_manager_linux.h"
namespace NEO {
std::unique_ptr<TbxPageFaultManager> TbxPageFaultManager::create() {
return std::make_unique<TbxPageFaultManagerLinux>();
}
void TbxPageFaultManagerLinux::handlePageFault(void *ptr, PageFaultData &faultData) {
TbxPageFaultManager::handlePageFault(ptr, faultData);
}
} // namespace NEO

21
shared/source/page_fault_manager/linux/tbx_page_fault_manager_linux.h
View File

@@ -1,21 +0,0 @@
/*
* Copyright (C) 2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h"
#include "shared/source/page_fault_manager/tbx_page_fault_manager.h"
namespace NEO {
class TbxPageFaultManagerLinux final : public TbxPageFaultManager, public PageFaultManagerLinux {
public:
protected:
void handlePageFault(void *ptr, PageFaultData &faultData) override;
};
} // namespace NEO

68
shared/source/page_fault_manager/tbx_page_fault_manager.cpp
View File

@@ -1,68 +0,0 @@
/*
* Copyright (C) 2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/page_fault_manager/tbx_page_fault_manager.h"
#include "shared/source/command_stream/command_stream_receiver.h"
#include "shared/source/memory_manager/graphics_allocation.h"
#include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
namespace NEO {
void TbxPageFaultManager::handlePageFault(void *ptr, PageFaultData &faultData) {
if (faultData.faultType == FaultMode::cpu) {
CpuPageFaultManager::handlePageFault(ptr, faultData);
return;
}
auto &graphicsAllocation = *faultData.gfxAllocation;
auto bank = faultData.bank;
auto hasBeenDownloaded = faultData.hasBeenDownloaded;
auto size = faultData.size;
auto csr = faultData.csr;
if (!hasBeenDownloaded) {
this->allowCPUMemoryAccess(ptr, size);
csr->downloadAllocation(graphicsAllocation);
this->protectCpuMemoryFromWrites(ptr, size);
faultData.hasBeenDownloaded = true;
} else {
graphicsAllocation.setTbxWritable(true, bank);
this->allowCPUMemoryAccess(ptr, size);
this->memoryData.erase(ptr);
}
}
void TbxPageFaultManager::removeAllocation(GraphicsAllocation *alloc) {
std::unique_lock<SpinLock> lock{mtx};
for (auto &data : memoryData) {
auto allocPtr = data.first;
auto faultData = data.second;
if (faultData.gfxAllocation == alloc) {
memoryData.erase(allocPtr);
this->allowCPUMemoryAccess(allocPtr, faultData.size);
return;
}
}
}
void TbxPageFaultManager::insertAllocation(CommandStreamReceiver *csr, GraphicsAllocation *alloc, uint32_t bank, void *ptr, size_t size) {
std::unique_lock<SpinLock> lock{mtx};
if (this->memoryData.find(ptr) == this->memoryData.end()) {
PageFaultData pageFaultData{};
pageFaultData.faultType = FaultMode::tbx;
pageFaultData.size = size;
pageFaultData.gfxAllocation = alloc;
pageFaultData.bank = bank;
pageFaultData.csr = csr;
memoryData[ptr] = pageFaultData;
}
auto &faultData = this->memoryData[ptr];
faultData.hasBeenDownloaded = false;
this->protectCPUMemoryAccess(ptr, size);
}
} // namespace NEO

32
shared/source/page_fault_manager/tbx_page_fault_manager.h
View File

@@ -1,32 +0,0 @@
/*
* Copyright (C) 2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
namespace NEO {
class CommandStreamReceiver;
class GraphicsAllocation;
class TbxPageFaultManager : public virtual CpuPageFaultManager {
public:
static std::unique_ptr<TbxPageFaultManager> create();
using CpuPageFaultManager::insertAllocation;
using CpuPageFaultManager::removeAllocation;
void insertAllocation(CommandStreamReceiver *csr, GraphicsAllocation *alloc, uint32_t bank, void *ptr, size_t size) override;
void removeAllocation(GraphicsAllocation *alloc) override;
using CpuPageFaultManager::checkFaultHandlerFromPageFaultManager;
using CpuPageFaultManager::verifyAndHandlePageFault;
protected:
void handlePageFault(void *ptr, PageFaultData &faultData) override;
};
} // namespace NEO

2
shared/source/page_fault_manager/windows/CMakeLists.txt
View File

@@ -8,8 +8,6 @@ set(NEO_CORE_PAGE_FAULT_MANAGER_WINDOWS
${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
${CMAKE_CURRENT_SOURCE_DIR}/cpu_page_fault_manager_windows.cpp
${CMAKE_CURRENT_SOURCE_DIR}/cpu_page_fault_manager_windows.h
${CMAKE_CURRENT_SOURCE_DIR}/tbx_page_fault_manager_windows.cpp
${CMAKE_CURRENT_SOURCE_DIR}/tbx_page_fault_manager_windows.h
)
if(WIN32)

22
shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.cpp
View File

@@ -8,27 +8,21 @@
#include "shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h"
#include "shared/source/command_stream/command_stream_receiver.h"
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/device/device.h"
#include "shared/source/helpers/debug_helpers.h"
#include "shared/source/memory_manager/unified_memory_manager.h"
#include "shared/source/os_interface/os_interface.h"
#include "shared/source/os_interface/windows/os_context_win.h"
#include "shared/source/page_fault_manager/windows/tbx_page_fault_manager_windows.h"
namespace NEO {
std::unique_ptr<CpuPageFaultManager> CpuPageFaultManager::create() {
auto pageFaultManager = [&]() -> std::unique_ptr<CpuPageFaultManager> {
if (debugManager.isTbxMode()) {
return TbxPageFaultManager::create();
}
return std::make_unique<CpuPageFaultManagerWindows>();
}();
std::unique_ptr<PageFaultManager> PageFaultManager::create() {
auto pageFaultManager = std::make_unique<PageFaultManagerWindows>();
pageFaultManager->selectGpuDomainHandler();
return pageFaultManager;
}
std::function<LONG(struct _EXCEPTION_POINTERS *exceptionInfo)> PageFaultManagerWindows::pageFaultHandler = nullptr;
std::function<LONG(struct _EXCEPTION_POINTERS *exceptionInfo)> PageFaultManagerWindows::pageFaultHandler;
PageFaultManagerWindows::PageFaultManagerWindows() {
PageFaultManagerWindows::registerFaultHandler();
@@ -73,16 +67,10 @@ void PageFaultManagerWindows::protectCPUMemoryAccess(void *ptr, size_t size) {
UNRECOVERABLE_IF(!retVal);
}
void PageFaultManagerWindows::protectCpuMemoryFromWrites(void *ptr, size_t size) {
DWORD previousState;
auto retVal = VirtualProtect(ptr, size, PAGE_READONLY, &previousState);
UNRECOVERABLE_IF(!retVal);
}
void PageFaultManagerWindows::evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) {}
void PageFaultManagerWindows::allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) {
NEO::SvmAllocationData *allocData = this->memoryData[ptr].unifiedMemoryManager->getSVMAlloc(ptr);
NEO::SvmAllocationData *allocData = memoryData[ptr].unifiedMemoryManager->getSVMAlloc(ptr);
UNRECOVERABLE_IF(allocData == nullptr);
if (osInterface) {

6
shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h
View File

@@ -13,8 +13,7 @@
#include <functional>
namespace NEO {
class PageFaultManagerWindows : public virtual CpuPageFaultManager {
class PageFaultManagerWindows : public PageFaultManager {
public:
PageFaultManagerWindows();
~PageFaultManagerWindows() override;
@@ -24,7 +23,6 @@ class PageFaultManagerWindows : public virtual CpuPageFaultManager {
protected:
void allowCPUMemoryAccess(void *ptr, size_t size) override;
void protectCPUMemoryAccess(void *ptr, size_t size) override;
void protectCpuMemoryFromWrites(void *ptr, size_t size) override;
void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override;
void allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override;
@@ -36,6 +34,4 @@ class PageFaultManagerWindows : public virtual CpuPageFaultManager {
PVOID previousHandler;
};
class CpuPageFaultManagerWindows final : public PageFaultManagerWindows {};
} // namespace NEO

19
shared/source/page_fault_manager/windows/tbx_page_fault_manager_windows.cpp
View File

@@ -1,19 +0,0 @@
/*
* Copyright (C) 2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/page_fault_manager/windows/tbx_page_fault_manager_windows.h"
namespace NEO {
std::unique_ptr<TbxPageFaultManager> TbxPageFaultManager::create() {
return std::make_unique<TbxPageFaultManagerWindows>();
}
void TbxPageFaultManagerWindows::handlePageFault(void *ptr, PageFaultData &faultData) {
TbxPageFaultManager::handlePageFault(ptr, faultData);
}
} // namespace NEO

24
shared/source/page_fault_manager/windows/tbx_page_fault_manager_windows.h
View File

@@ -1,24 +0,0 @@
/*
* Copyright (C) 2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "shared/source/page_fault_manager/tbx_page_fault_manager.h"
#include "shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h"
// Known false positive for valid virtual inheritance
#pragma warning(disable : 4250)
namespace NEO {
class TbxPageFaultManagerWindows final : public TbxPageFaultManager, public PageFaultManagerWindows {
public:
protected:
void handlePageFault(void *ptr, PageFaultData &faultData) override;
};
} // namespace NEO

3
shared/test/common/fixtures/tbx_command_stream_fixture.cpp
View File

@@ -10,7 +10,6 @@
#include "shared/source/command_stream/command_stream_receiver.h"
#include "shared/source/command_stream/tbx_command_stream_receiver_hw.h"
#include "shared/source/memory_manager/os_agnostic_memory_manager.h"
#include "shared/test/common/helpers/debug_manager_state_restore.h"
#include "shared/test/common/mocks/mock_device.h"
#include "gtest/gtest.h"
@@ -19,8 +18,6 @@ namespace NEO {
void TbxCommandStreamFixture::setUp(MockDevice *pDevice) {
// Create our TBX command stream receiver based on HW type
DebugManagerStateRestore dbgRestore;
debugManager.flags.SetCommandStreamReceiver.set(static_cast<int32_t>(CommandStreamReceiverType::tbx));
pCommandStreamReceiver = TbxCommandStreamReceiver::create("", false, *pDevice->executionEnvironment, pDevice->getRootDeviceIndex(), pDevice->getDeviceBitfield());
ASSERT_NE(nullptr, pCommandStreamReceiver);
memoryManager = new OsAgnosticMemoryManager(*pDevice->executionEnvironment);

1
shared/test/common/mocks/mock_allocation_properties.h
View File

@@ -19,6 +19,5 @@ struct MockAllocationProperties : public AllocationProperties {
MockAllocationProperties(uint32_t rootDeviceIndex, bool allocateMemory, size_t size, DeviceBitfield deviceBitfield) : AllocationProperties(rootDeviceIndex, allocateMemory, size, AllocationType::internalHostMemory, false, deviceBitfield) {}
MockAllocationProperties(uint32_t rootDeviceIndex, bool allocateMemory, size_t size, AllocationType allocationType) : AllocationProperties(rootDeviceIndex, allocateMemory, size, allocationType, false, mockDeviceBitfield) {}
MockAllocationProperties(uint32_t rootDeviceIndex, bool allocateMemory, size_t size, AllocationType allocationType, DeviceBitfield deviceBitfield) : AllocationProperties(rootDeviceIndex, allocateMemory, size, allocationType, false, deviceBitfield) {}
MockAllocationProperties(uint32_t rootDeviceIndex, AllocationType allocType, size_t size) : AllocationProperties(rootDeviceIndex, true, size, allocType, false, mockDeviceBitfield) {}
};
} // namespace NEO

41
shared/test/common/mocks/mock_cpu_page_fault_manager.h
View File

@@ -14,17 +14,16 @@
using namespace NEO;
template <class BaseFaultManager>
class MockPageFaultManagerImpl : public BaseFaultManager {
class MockPageFaultManager : public PageFaultManager {
public:
using BaseFaultManager::BaseFaultManager;
using BaseFaultManager::gpuDomainHandler;
using BaseFaultManager::memoryData;
using PageFaultData = typename BaseFaultManager::PageFaultData;
using BaseFaultManager::selectGpuDomainHandler;
using BaseFaultManager::transferAndUnprotectMemory;
using BaseFaultManager::unprotectAndTransferMemory;
using BaseFaultManager::verifyAndHandlePageFault;
using PageFaultManager::gpuDomainHandler;
using PageFaultManager::memoryData;
using PageFaultManager::PageFaultData;
using PageFaultManager::PageFaultManager;
using PageFaultManager::selectGpuDomainHandler;
using PageFaultManager::transferAndUnprotectMemory;
using PageFaultManager::unprotectAndTransferMemory;
using PageFaultManager::verifyAndHandlePageFault;
bool checkFaultHandlerFromPageFaultManager() override {
checkFaultHandlerCalled++;
@@ -43,9 +42,6 @@ class MockPageFaultManagerImpl : public BaseFaultManager {
protectedMemoryAccessAddress = ptr;
protectedSize = size;
}
void protectCpuMemoryFromWrites(void *ptr, size_t size) override {
protectWritesCalled++;
}
void transferToCpu(void *ptr, size_t size, void *cmdQ) override {
transferToCpuCalled++;
transferToCpuAddress = ptr;
@@ -66,19 +62,19 @@ class MockPageFaultManagerImpl : public BaseFaultManager {
allowCPUMemoryEvictionCalled++;
}
void baseAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
BaseFaultManager::setAubWritable(writable, ptr, unifiedMemoryManager);
PageFaultManager::setAubWritable(writable, ptr, unifiedMemoryManager);
}
void baseCpuTransfer(void *ptr, size_t size, void *cmdQ) {
BaseFaultManager::transferToCpu(ptr, size, cmdQ);
PageFaultManager::transferToCpu(ptr, size, cmdQ);
}
void baseGpuTransfer(void *ptr, void *cmdQ) {
BaseFaultManager::transferToGpu(ptr, cmdQ);
PageFaultManager::transferToGpu(ptr, cmdQ);
}
void baseCpuAllocEvictable(bool evictable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
BaseFaultManager::setCpuAllocEvictable(evictable, ptr, unifiedMemoryManager);
PageFaultManager::setCpuAllocEvictable(evictable, ptr, unifiedMemoryManager);
}
void baseAllowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
BaseFaultManager::allowCPUMemoryEviction(evict, ptr, pageFaultData);
PageFaultManager::allowCPUMemoryEviction(evict, ptr, pageFaultData);
}
void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override {}
@@ -89,22 +85,21 @@ class MockPageFaultManagerImpl : public BaseFaultManager {
}
void *getHwHandlerAddress() {
return reinterpret_cast<void *>(BaseFaultManager::transferAndUnprotectMemory);
return reinterpret_cast<void *>(PageFaultManager::transferAndUnprotectMemory);
}
void *getAubAndTbxHandlerAddress() {
return reinterpret_cast<void *>(BaseFaultManager::unprotectAndTransferMemory);
return reinterpret_cast<void *>(PageFaultManager::unprotectAndTransferMemory);
}
void moveAllocationToGpuDomain(void *ptr) override {
moveAllocationToGpuDomainCalled++;
BaseFaultManager::moveAllocationToGpuDomain(ptr);
PageFaultManager::moveAllocationToGpuDomain(ptr);
}
int checkFaultHandlerCalled = 0;
int registerFaultHandlerCalled = 0;
int allowMemoryAccessCalled = 0;
int protectMemoryCalled = 0;
int protectWritesCalled = 0;
int transferToCpuCalled = 0;
int transferToGpuCalled = 0;
int moveAllocationToGpuDomainCalled = 0;
@@ -125,8 +120,6 @@ class MockPageFaultManagerImpl : public BaseFaultManager {
EngineUsage engineUsage = EngineUsage::engineUsageCount;
};
class MockPageFaultManager : public MockPageFaultManagerImpl<CpuPageFaultManager> {};
template <class T>
class MockPageFaultManagerHandlerInvoke : public T {
public:

5
shared/test/common/mocks/mock_tbx_csr.h
View File

@@ -10,8 +10,6 @@
#include "shared/source/command_stream/submission_status.h"
#include "shared/source/command_stream/tbx_command_stream_receiver_hw.h"
#include "shared/source/execution_environment/execution_environment.h"
#include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
#include "shared/source/page_fault_manager/tbx_page_fault_manager.h"
namespace NEO {
@@ -20,8 +18,6 @@ class MockTbxCsr : public TbxCommandStreamReceiverHw<GfxFamily> {
public:
using TbxCommandStreamReceiverHw<GfxFamily>::writeMemory;
using TbxCommandStreamReceiverHw<GfxFamily>::allocationsForDownload;
using TbxCommandStreamReceiverHw<GfxFamily>::getParametersForMemory;
using TbxCommandStreamReceiverHw<GfxFamily>::getTbxPageFaultManager;
MockTbxCsr(ExecutionEnvironment &executionEnvironment, const DeviceBitfield deviceBitfield)
: TbxCommandStreamReceiverHw<GfxFamily>(executionEnvironment, 0, deviceBitfield) {
this->downloadAllocationImpl = [this](GraphicsAllocation &gfxAllocation) {
@@ -68,7 +64,6 @@ class MockTbxCsr : public TbxCommandStreamReceiverHw<GfxFamily> {
TbxCommandStreamReceiverHw<GfxFamily>::dumpAllocation(gfxAllocation);
dumpAllocationCalled = true;
}
bool initializeEngineCalled = false;
bool writeMemoryWithAubManagerCalled = false;
bool writeMemoryCalled = false;

198
shared/test/unit_test/command_stream/tbx_command_stream_tests.cpp
View File

@@ -13,11 +13,9 @@
#include "shared/source/helpers/engine_node_helper.h"
#include "shared/source/helpers/gfx_core_helper.h"
#include "shared/source/helpers/hardware_context_controller.h"
#include "shared/source/helpers/options.h"
#include "shared/source/helpers/ptr_math.h"
#include "shared/source/memory_manager/memory_banks.h"
#include "shared/source/memory_manager/os_agnostic_memory_manager.h"
#include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
#include "shared/test/common/fixtures/device_fixture.h"
#include "shared/test/common/fixtures/mock_aub_center_fixture.h"
#include "shared/test/common/fixtures/tbx_command_stream_fixture.h"
@@ -1283,199 +1281,3 @@ HWTEST_F(TbxCommandStreamTests, givenTimestampBufferAllocationWhenTbxWriteMemory
memoryManager->freeGraphicsMemory(timestampAllocation);
}
template <typename FamilyType, CommandStreamReceiverType csrType>
struct TbxPageFaultTestFixture {
class MockTbxCsrForPageFaultTests : public MockTbxCsr<FamilyType> {
public:
using MockTbxCsr<FamilyType>::MockTbxCsr;
CpuPageFaultManager *getTbxPageFaultManager() override {
return this->tbxFaultManager.get();
}
using MockTbxCsr<FamilyType>::isAllocTbxFaultable;
std::unique_ptr<TbxPageFaultManager> tbxFaultManager = TbxPageFaultManager::create();
};
static void runTest1(MockDevice *pDevice) {
DebugManagerStateRestore dbgRestore;
debugManager.flags.SetCommandStreamReceiver.set(static_cast<int32_t>(csrType));
std::unique_ptr<MockTbxCsrForPageFaultTests> tbxCsr(new MockTbxCsrForPageFaultTests(*pDevice->executionEnvironment, pDevice->getDeviceBitfield()));
tbxCsr->setupContext(*pDevice->getDefaultEngine().osContext);
EXPECT_TRUE(tbxCsr->tbxFaultManager->checkFaultHandlerFromPageFaultManager());
auto memoryManager = pDevice->getMemoryManager();
NEO::GraphicsAllocation *gfxAlloc1 = memoryManager->allocateGraphicsMemoryWithProperties(
{pDevice->getRootDeviceIndex(),
MemoryConstants::pageSize,
AllocationType::bufferHostMemory,
pDevice->getDeviceBitfield()});
uint64_t gpuAddress;
void *cpuAddress;
size_t size;
EXPECT_TRUE(tbxCsr->getParametersForMemory(*gfxAlloc1, gpuAddress, cpuAddress, size));
tbxCsr->writeMemory(*gfxAlloc1);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
// accessing outside address range does not affect inserted host allocs
auto ptrBelow = (void *)0x0;
EXPECT_FALSE(tbxCsr->tbxFaultManager->verifyAndHandlePageFault(ptrBelow, true));
auto ptrAbove = ptrOffset(cpuAddress, size + 1);
EXPECT_FALSE(tbxCsr->tbxFaultManager->verifyAndHandlePageFault(ptrAbove, true));
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
*reinterpret_cast<char *>(cpuAddress) = 1;
EXPECT_TRUE(tbxCsr->isTbxWritable(*gfxAlloc1));
EXPECT_TRUE(tbxCsr->makeCoherentCalled);
tbxCsr->makeCoherentCalled = false;
tbxCsr->writeMemory(*gfxAlloc1);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
// accessing address with offset that is still in alloc range should
// also make writable and download
reinterpret_cast<char *>(cpuAddress)[1] = 1;
EXPECT_TRUE(tbxCsr->isTbxWritable(*gfxAlloc1));
EXPECT_TRUE(tbxCsr->makeCoherentCalled);
tbxCsr->makeCoherentCalled = false;
// for coverage
tbxCsr->tbxFaultManager->removeAllocation(static_cast<GraphicsAllocation *>(nullptr));
tbxCsr->tbxFaultManager->removeAllocation(gfxAlloc1);
memoryManager->freeGraphicsMemory(gfxAlloc1);
}
static void runtTest2(MockDevice *pDevice) {
DebugManagerStateRestore dbgRestore;
debugManager.flags.SetCommandStreamReceiver.set(static_cast<int32_t>(csrType));
std::unique_ptr<MockTbxCsrForPageFaultTests> tbxCsr(new MockTbxCsrForPageFaultTests(*pDevice->executionEnvironment, pDevice->getDeviceBitfield()));
tbxCsr->setupContext(*pDevice->getDefaultEngine().osContext);
EXPECT_TRUE(tbxCsr->tbxFaultManager->checkFaultHandlerFromPageFaultManager());
auto memoryManager = pDevice->getMemoryManager();
NEO::GraphicsAllocation *gfxAlloc1 = memoryManager->allocateGraphicsMemoryWithProperties(
{pDevice->getRootDeviceIndex(),
MemoryConstants::pageSize,
AllocationType::bufferHostMemory,
pDevice->getDeviceBitfield()});
uint64_t gpuAddress;
void *cpuAddress;
size_t size;
EXPECT_TRUE(tbxCsr->getParametersForMemory(*gfxAlloc1, gpuAddress, cpuAddress, size));
tbxCsr->writeMemory(*gfxAlloc1);
tbxCsr->downloadAllocationTbx(*gfxAlloc1);
EXPECT_TRUE(!tbxCsr->isTbxWritable(*gfxAlloc1));
static_cast<float *>(cpuAddress)[0] = 1.0f;
memoryManager->freeGraphicsMemory(gfxAlloc1);
}
static void runtTest3(MockDevice *pDevice) {
DebugManagerStateRestore dbgRestore;
debugManager.flags.SetCommandStreamReceiver.set(static_cast<int32_t>(csrType));
std::unique_ptr<MockTbxCsrForPageFaultTests> tbxCsr(new MockTbxCsrForPageFaultTests(*pDevice->executionEnvironment, pDevice->getDeviceBitfield()));
tbxCsr->setupContext(*pDevice->getDefaultEngine().osContext);
EXPECT_TRUE(tbxCsr->tbxFaultManager->checkFaultHandlerFromPageFaultManager());
auto memoryManager = pDevice->getMemoryManager();
NEO::GraphicsAllocation *gfxAlloc1 = memoryManager->allocateGraphicsMemoryWithProperties(
{pDevice->getRootDeviceIndex(),
MemoryConstants::pageSize,
AllocationType::bufferHostMemory,
pDevice->getDeviceBitfield()});
auto cpuPtr = gfxAlloc1->getDriverAllocatedCpuPtr();
gfxAlloc1->setDriverAllocatedCpuPtr(nullptr);
EXPECT_FALSE(tbxCsr->isAllocTbxFaultable(gfxAlloc1));
gfxAlloc1->setDriverAllocatedCpuPtr(cpuPtr);
memoryManager->freeGraphicsMemory(gfxAlloc1);
}
static void runTest4(MockDevice *pDevice) {
DebugManagerStateRestore dbgRestore;
debugManager.flags.SetCommandStreamReceiver.set(static_cast<int32_t>(csrType));
std::unique_ptr<MockTbxCsrForPageFaultTests> tbxCsr(new MockTbxCsrForPageFaultTests(*pDevice->executionEnvironment, pDevice->getDeviceBitfield()));
tbxCsr->setupContext(*pDevice->getDefaultEngine().osContext);
EXPECT_TRUE(tbxCsr->tbxFaultManager->checkFaultHandlerFromPageFaultManager());
auto memoryManager = pDevice->getMemoryManager();
NEO::GraphicsAllocation *gfxAlloc1 = memoryManager->allocateGraphicsMemoryWithProperties(
{pDevice->getRootDeviceIndex(),
MemoryConstants::pageSize,
AllocationType::bufferHostMemory,
pDevice->getDeviceBitfield()});
uint64_t gpuAddress;
void *cpuAddress;
size_t size;
EXPECT_TRUE(tbxCsr->getParametersForMemory(*gfxAlloc1, gpuAddress, cpuAddress, size));
*reinterpret_cast<char *>(cpuAddress) = 1;
tbxCsr->writeMemory(*gfxAlloc1);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
auto readVal = *reinterpret_cast<char *>(cpuAddress);
EXPECT_EQ(1, readVal);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
EXPECT_TRUE(tbxCsr->makeCoherentCalled);
tbxCsr->makeCoherentCalled = false;
tbxCsr->writeMemory(*gfxAlloc1);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
readVal = *reinterpret_cast<char *>(cpuAddress);
EXPECT_EQ(1, readVal);
EXPECT_FALSE(tbxCsr->isTbxWritable(*gfxAlloc1));
EXPECT_TRUE(tbxCsr->makeCoherentCalled);
tbxCsr->makeCoherentCalled = false;
// for coverage
tbxCsr->tbxFaultManager->removeAllocation(static_cast<GraphicsAllocation *>(nullptr));
tbxCsr->tbxFaultManager->removeAllocation(gfxAlloc1);
memoryManager->freeGraphicsMemory(gfxAlloc1);
}
};
HWTEST_F(TbxCommandStreamTests, givenTbxModeWhenHostWritesHostAllocThenAllocShouldBeDownloadedAndWritable) {
TbxPageFaultTestFixture<FamilyType, CommandStreamReceiverType::tbx>::runTest1(pDevice);
}
HWTEST_F(TbxCommandStreamTests, givenTbxWithAubModeWhenHostWritesHostAllocThenAllocShouldBeDownloadedAndWritable) {
TbxPageFaultTestFixture<FamilyType, CommandStreamReceiverType::tbxWithAub>::runTest1(pDevice);
}
HWTEST_F(TbxCommandStreamTests, givenTbxWithModeWhenHostBufferNotWritableAndProtectedThenDownloadShouldNotCrash) {
TbxPageFaultTestFixture<FamilyType, CommandStreamReceiverType::tbx>::runtTest2(pDevice);
}
HWTEST_F(TbxCommandStreamTests, givenAllocationWithNoDriverAllocatedCpuPtrThenIsAllocTbxFaultableShouldReturnFalse) {
TbxPageFaultTestFixture<FamilyType, CommandStreamReceiverType::tbx>::runtTest3(pDevice);
}
HWTEST_F(TbxCommandStreamTests, givenTbxModeWhenHostReadsHostAllocThenAllocShouldBeDownloadedButNotWritable) {
TbxPageFaultTestFixture<FamilyType, CommandStreamReceiverType::tbx>::runTest4(pDevice);
}

18
shared/test/unit_test/debug_settings/debug_settings_manager_tests.cpp
View File

@@ -452,21 +452,3 @@ TEST(DurationLogTest, givenDurationGetTimeStringThenTimeStringIsCorrect) {
EXPECT_TRUE(std::isdigit(c) || c == '[' || c == ']' || c == '.' || c == ' ');
}
}
TEST(DebugSettingsManager, GivenTbxOrTbxWithAubCsrTypeWhenCallingIsTbxModeThenReturnTrue) {
DebugManagerStateRestore restorer;
NEO::debugManager.flags.SetCommandStreamReceiver.set(2);
EXPECT_TRUE(NEO::debugManager.isTbxMode());
NEO::debugManager.flags.SetCommandStreamReceiver.set(4);
EXPECT_TRUE(NEO::debugManager.isTbxMode());
}
TEST(DebugSettingsManager, GivenHardwareOrHardwareWithAubCsrTypeWhenCallingIsTbxModeThenReturnFalse) {
DebugManagerStateRestore restorer;
NEO::debugManager.flags.SetCommandStreamReceiver.set(1);
EXPECT_FALSE(NEO::debugManager.isTbxMode());
NEO::debugManager.flags.SetCommandStreamReceiver.set(3);
EXPECT_FALSE(NEO::debugManager.isTbxMode());
}

87
shared/test/unit_test/page_fault_manager/cpu_page_fault_manager_tests.cpp
View File

@@ -5,8 +5,6 @@
*
*/
#include "shared/source/helpers/options.h"
#include "shared/source/page_fault_manager/tbx_page_fault_manager.h"
#include "shared/test/common/fixtures/cpu_page_fault_manager_tests_fixture.h"
#include "shared/test/common/helpers/debug_manager_state_restore.h"
#include "shared/test/common/mocks/mock_graphics_allocation.h"
@@ -23,7 +21,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocsWhenInsertingAllocsThenAllo
EXPECT_EQ(pageFaultManager->memoryData.size(), 1u);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].size, 10u);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].cmdQ, cmdQ);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].domain, CpuPageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].domain, PageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].unifiedMemoryManager, unifiedMemoryManager.get());
EXPECT_EQ(pageFaultManager->allowMemoryAccessCalled, 0);
EXPECT_EQ(pageFaultManager->protectMemoryCalled, 0);
@@ -35,11 +33,11 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocsWhenInsertingAllocsThenAllo
EXPECT_EQ(pageFaultManager->memoryData.size(), 2u);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].size, 10u);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].cmdQ, cmdQ);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].domain, CpuPageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].domain, PageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].unifiedMemoryManager, unifiedMemoryManager.get());
EXPECT_EQ(pageFaultManager->memoryData[alloc2].size, 20u);
EXPECT_EQ(pageFaultManager->memoryData[alloc2].cmdQ, cmdQ);
EXPECT_EQ(pageFaultManager->memoryData[alloc2].domain, CpuPageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData[alloc2].domain, PageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData[alloc2].unifiedMemoryManager, unifiedMemoryManager.get());
EXPECT_EQ(pageFaultManager->allowMemoryAccessCalled, 0);
EXPECT_EQ(pageFaultManager->protectMemoryCalled, 0);
@@ -64,7 +62,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenRemovingProtectedAllocTh
EXPECT_EQ(pageFaultManager->memoryData[alloc1].size, 10u);
EXPECT_EQ(pageFaultManager->memoryData[alloc1].unifiedMemoryManager, unifiedMemoryManager.get());
pageFaultManager->memoryData[alloc1].domain = CpuPageFaultManager::AllocationDomain::gpu;
pageFaultManager->memoryData[alloc1].domain = PageFaultManager::AllocationDomain::gpu;
pageFaultManager->removeAllocation(alloc1);
EXPECT_EQ(pageFaultManager->allowMemoryAccessCalled, 1);
@@ -93,7 +91,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenRemovingAllocsThenNonGpu
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 1u);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs[0], alloc2);
pageFaultManager->memoryData.at(alloc2).domain = CpuPageFaultManager::AllocationDomain::gpu;
pageFaultManager->memoryData.at(alloc2).domain = PageFaultManager::AllocationDomain::gpu;
pageFaultManager->removeAllocation(alloc2);
EXPECT_EQ(pageFaultManager->allowMemoryAccessCalled, 1);
EXPECT_EQ(pageFaultManager->allowedMemoryAccessAddress, alloc2);
@@ -111,7 +109,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenAllocNotPresentInNonGpuD
pageFaultManager->moveAllocationToGpuDomain(alloc1);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 0u);
pageFaultManager->memoryData.at(alloc1).domain = CpuPageFaultManager::AllocationDomain::cpu;
pageFaultManager->memoryData.at(alloc1).domain = PageFaultManager::AllocationDomain::cpu;
pageFaultManager->removeAllocation(alloc1);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 0u);
}
@@ -129,7 +127,7 @@ TEST_F(PageFaultManagerTest, givenNonGpuAllocsContainerWhenMovingToGpuDomainMult
pageFaultManager->moveAllocationToGpuDomain(alloc1);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 1u);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs[0], alloc2);
pageFaultManager->memoryData.at(alloc1).domain = CpuPageFaultManager::AllocationDomain::none;
pageFaultManager->memoryData.at(alloc1).domain = PageFaultManager::AllocationDomain::none;
pageFaultManager->moveAllocationToGpuDomain(alloc1);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 1u);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs[0], alloc2);
@@ -148,7 +146,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocsWhenMovingToGpuDomainAllocs
pageFaultManager->insertAllocation(alloc3, 30, unifiedMemoryManager.get(), cmdQ, {});
EXPECT_EQ(pageFaultManager->transferToCpuCalled, 0);
EXPECT_EQ(pageFaultManager->memoryData.size(), 3u);
pageFaultManager->memoryData.at(alloc3).domain = CpuPageFaultManager::AllocationDomain::gpu;
pageFaultManager->memoryData.at(alloc3).domain = PageFaultManager::AllocationDomain::gpu;
pageFaultManager->moveAllocationsWithinUMAllocsManagerToGpuDomain(unifiedMemoryManager.get());
@@ -168,9 +166,9 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenMovingAllocsOfGivenUMMan
pageFaultManager->insertAllocation(alloc1, 10u, unifiedMemoryManager.get(), nullptr, {});
pageFaultManager->insertAllocation(alloc2, 20u, unifiedMemoryManager.get(), nullptr, {});
pageFaultManager->memoryData.at(alloc2).domain = CpuPageFaultManager::AllocationDomain::gpu;
pageFaultManager->memoryData.at(alloc2).domain = PageFaultManager::AllocationDomain::gpu;
pageFaultManager->moveAllocationsWithinUMAllocsManagerToGpuDomain(unifiedMemoryManager.get());
EXPECT_EQ(pageFaultManager->memoryData.at(alloc1).domain, CpuPageFaultManager::AllocationDomain::gpu);
EXPECT_EQ(pageFaultManager->memoryData.at(alloc1).domain, PageFaultManager::AllocationDomain::gpu);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 0u);
}
@@ -189,7 +187,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocsWhenMovingToGpuDomainWithPr
pageFaultManager->insertAllocation(alloc3, 30, unifiedMemoryManager.get(), cmdQ, {});
EXPECT_EQ(pageFaultManager->transferToCpuCalled, 0);
EXPECT_EQ(pageFaultManager->memoryData.size(), 3u);
pageFaultManager->memoryData.at(alloc3).domain = CpuPageFaultManager::AllocationDomain::gpu;
pageFaultManager->memoryData.at(alloc3).domain = PageFaultManager::AllocationDomain::gpu;
testing::internal::CaptureStdout(); // start capturing
@@ -221,8 +219,8 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocsWhenMovingToGpuDomainAllocs
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 2u);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs[0], alloc1);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs[1], alloc2);
pageFaultManager->memoryData.at(alloc1).domain = CpuPageFaultManager::AllocationDomain::cpu;
pageFaultManager->memoryData.at(alloc2).domain = CpuPageFaultManager::AllocationDomain::none;
pageFaultManager->memoryData.at(alloc1).domain = PageFaultManager::AllocationDomain::cpu;
pageFaultManager->memoryData.at(alloc2).domain = PageFaultManager::AllocationDomain::none;
pageFaultManager->moveAllocationsWithinUMAllocsManagerToGpuDomain(unifiedMemoryManager.get());
@@ -368,13 +366,13 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenMovingToGpuDomainThenUpd
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 2u);
pageFaultManager->moveAllocationToGpuDomain(alloc1);
EXPECT_EQ(pageFaultManager->memoryData.at(alloc1).domain, CpuPageFaultManager::AllocationDomain::gpu);
EXPECT_EQ(pageFaultManager->memoryData.at(alloc1).domain, PageFaultManager::AllocationDomain::gpu);
const auto &allocsVec = unifiedMemoryManager->nonGpuDomainAllocs;
EXPECT_EQ(std::find(allocsVec.cbegin(), allocsVec.cend(), alloc1), allocsVec.cend());
EXPECT_EQ(allocsVec.size(), 1u);
EXPECT_EQ(allocsVec[0], alloc2);
pageFaultManager->memoryData.at(alloc2).domain = CpuPageFaultManager::AllocationDomain::gpu;
pageFaultManager->memoryData.at(alloc2).domain = PageFaultManager::AllocationDomain::gpu;
pageFaultManager->moveAllocationToGpuDomain(alloc2);
EXPECT_NE(std::find(allocsVec.cbegin(), allocsVec.cend(), alloc2), allocsVec.cend());
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 1u);
@@ -415,7 +413,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocInGpuDomainWhenMovingToGpuDo
pageFaultManager->insertAllocation(alloc, 10, unifiedMemoryManager.get(), cmdQ, {});
EXPECT_EQ(pageFaultManager->transferToCpuCalled, 0);
EXPECT_EQ(pageFaultManager->memoryData.size(), 1u);
pageFaultManager->memoryData.at(alloc).domain = CpuPageFaultManager::AllocationDomain::gpu;
pageFaultManager->memoryData.at(alloc).domain = PageFaultManager::AllocationDomain::gpu;
pageFaultManager->moveAllocationToGpuDomain(alloc);
EXPECT_EQ(pageFaultManager->allowMemoryAccessCalled, 0);
@@ -541,15 +539,15 @@ TEST_F(PageFaultManagerTest, givenAllocsFromCpuDomainWhenVerifyingPageFaultThenD
pageFaultManager->moveAllocationsWithinUMAllocsManagerToGpuDomain(unifiedMemoryManager.get());
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 0u);
pageFaultManager->memoryData.at(alloc2).domain = CpuPageFaultManager::AllocationDomain::none;
pageFaultManager->memoryData.at(alloc2).domain = PageFaultManager::AllocationDomain::none;
pageFaultManager->verifyAndHandlePageFault(alloc2, true);
EXPECT_EQ(pageFaultManager->memoryData.at(alloc2).domain, CpuPageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData.at(alloc2).domain, PageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 0u);
pageFaultManager->gpuDomainHandler = &MockPageFaultManager::unprotectAndTransferMemory;
pageFaultManager->memoryData.at(alloc1).domain = CpuPageFaultManager::AllocationDomain::none;
pageFaultManager->memoryData.at(alloc1).domain = PageFaultManager::AllocationDomain::none;
pageFaultManager->verifyAndHandlePageFault(alloc1, true);
EXPECT_EQ(pageFaultManager->memoryData.at(alloc1).domain, CpuPageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(pageFaultManager->memoryData.at(alloc1).domain, PageFaultManager::AllocationDomain::cpu);
EXPECT_EQ(unifiedMemoryManager->nonGpuDomainAllocs.size(), 0u);
}
@@ -904,7 +902,7 @@ struct PageFaultManagerTestWithDebugFlag : public ::testing::TestWithParam<uint3
TEST_P(PageFaultManagerTestWithDebugFlag, givenDebugFlagWhenInsertingAllocationThenItOverridesHints) {
DebugManagerStateRestore restore;
debugManager.flags.UsmInitialPlacement.set(GetParam()); // Should be ignored by the driver, when passing hints
const auto expectedDomain = GetParam() == 1 ? CpuPageFaultManager::AllocationDomain::none : CpuPageFaultManager::AllocationDomain::cpu;
const auto expectedDomain = GetParam() == 1 ? PageFaultManager::AllocationDomain::none : PageFaultManager::AllocationDomain::cpu;
void *allocs[] = {
reinterpret_cast<void *>(0x1),
@@ -955,59 +953,24 @@ TEST_F(PageFaultManagerTest, givenNoUsmInitialPlacementFlagsWHenInsertingUsmAllo
memoryProperties.allocFlags.usmInitialPlacementCpu = 0;
memoryProperties.allocFlags.usmInitialPlacementGpu = 0;
pageFaultManager->insertAllocation(allocs[0], 10, unifiedMemoryManager.get(), cmdQ, memoryProperties);
EXPECT_EQ(CpuPageFaultManager::AllocationDomain::cpu, pageFaultManager->memoryData.at(allocs[0]).domain);
EXPECT_EQ(PageFaultManager::AllocationDomain::cpu, pageFaultManager->memoryData.at(allocs[0]).domain);
EXPECT_EQ(allocs[0], unifiedMemoryManager->nonGpuDomainAllocs[0]);
memoryProperties.allocFlags.usmInitialPlacementCpu = 0;
memoryProperties.allocFlags.usmInitialPlacementGpu = 1;
pageFaultManager->insertAllocation(allocs[1], 10, unifiedMemoryManager.get(), cmdQ, memoryProperties);
EXPECT_EQ(CpuPageFaultManager::AllocationDomain::none, pageFaultManager->memoryData.at(allocs[1]).domain);
EXPECT_EQ(PageFaultManager::AllocationDomain::none, pageFaultManager->memoryData.at(allocs[1]).domain);
EXPECT_EQ(allocs[1], unifiedMemoryManager->nonGpuDomainAllocs[1]);
memoryProperties.allocFlags.usmInitialPlacementCpu = 1;
memoryProperties.allocFlags.usmInitialPlacementGpu = 0;
pageFaultManager->insertAllocation(allocs[2], 10, unifiedMemoryManager.get(), cmdQ, memoryProperties);
EXPECT_EQ(CpuPageFaultManager::AllocationDomain::cpu, pageFaultManager->memoryData.at(allocs[2]).domain);
EXPECT_EQ(PageFaultManager::AllocationDomain::cpu, pageFaultManager->memoryData.at(allocs[2]).domain);
EXPECT_EQ(allocs[2], unifiedMemoryManager->nonGpuDomainAllocs[2]);
memoryProperties.allocFlags.usmInitialPlacementCpu = 1;
memoryProperties.allocFlags.usmInitialPlacementGpu = 1;
pageFaultManager->insertAllocation(allocs[3], 10, unifiedMemoryManager.get(), cmdQ, memoryProperties);
EXPECT_EQ(CpuPageFaultManager::AllocationDomain::cpu, pageFaultManager->memoryData.at(allocs[3]).domain);
EXPECT_EQ(PageFaultManager::AllocationDomain::cpu, pageFaultManager->memoryData.at(allocs[3]).domain);
EXPECT_EQ(allocs[3], unifiedMemoryManager->nonGpuDomainAllocs[3]);
}
TEST_F(PageFaultManagerTest, givenTbxModeWhenAllocateSharedMemoryThenTbxFaultManagerShouldBehaveLikeCpuFaultManager) {
auto memoryManager2 = std::make_unique<MockMemoryManager>(executionEnvironment);
auto unifiedMemoryManager2 = std::make_unique<SVMAllocsManager>(memoryManager2.get(), false);
auto pageFaultManager2 = std::make_unique<MockPageFaultManagerImpl<TbxPageFaultManager>>();
void *cmdQ = reinterpret_cast<void *>(0xFFFF);
RootDeviceIndicesContainer rootDeviceIndices = {mockRootDeviceIndex};
std::map<uint32_t, DeviceBitfield> deviceBitfields{{mockRootDeviceIndex, mockDeviceBitfield}};
auto properties = SVMAllocsManager::UnifiedMemoryProperties(InternalMemoryType::sharedUnifiedMemory, 1, rootDeviceIndices, deviceBitfields);
void *ptr = unifiedMemoryManager2->createSharedUnifiedMemoryAllocation(10, properties, cmdQ);
pageFaultManager2->insertAllocation(ptr, 10, unifiedMemoryManager2.get(), cmdQ, {});
EXPECT_TRUE(pageFaultManager2->verifyAndHandlePageFault(ptr, true));
EXPECT_EQ(pageFaultManager2->protectWritesCalled, 0);
unifiedMemoryManager2->freeSVMAlloc(ptr);
}
TEST_F(PageFaultManagerTest, givenHardwareModeWhenCallTbxInsertOrRemoveApiThenNothing) {
auto pageFaultManager2 = std::make_unique<MockPageFaultManagerImpl<CpuPageFaultManager>>();
auto ptr = reinterpret_cast<void *>(0XFFFF);
auto gfxAlloc = reinterpret_cast<GraphicsAllocation *>(0xFFFF);
auto csr = reinterpret_cast<CommandStreamReceiver *>(0xFFFF);
pageFaultManager2->insertAllocation(csr, gfxAlloc, 0, ptr, 0);
EXPECT_EQ(pageFaultManager2->memoryData.find(ptr), pageFaultManager2->memoryData.end());
pageFaultManager2->memoryData[ptr] = {};
pageFaultManager2->removeAllocation(gfxAlloc);
EXPECT_FALSE(pageFaultManager2->memoryData.find(ptr) == pageFaultManager2->memoryData.end());
pageFaultManager2->memoryData.erase(ptr);
}

2
shared/test/unit_test/page_fault_manager/windows/cpu_page_fault_manager_windows_tests.cpp
View File

@@ -156,4 +156,4 @@ TEST_F(PageFaultManagerTest,
EXPECT_EQ(0u, csr->getEvictionAllocations().size());
unifiedMemoryManager->freeSVMAlloc(ptr);
}
}

6
shared/test/unit_test/ult_specific_config.cpp
View File

@@ -29,11 +29,11 @@ const char *neoMockSettingsFileName = "neo_mock.config";
bool CompressionSelector::preferCompressedAllocation(const AllocationProperties &properties) {
return false;
}
void CpuPageFaultManager::transferToCpu(void *ptr, size_t size, void *cmdQ) {
void PageFaultManager::transferToCpu(void *ptr, size_t size, void *cmdQ) {
}
void CpuPageFaultManager::transferToGpu(void *ptr, void *cmdQ) {
void PageFaultManager::transferToGpu(void *ptr, void *cmdQ) {
}
void CpuPageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
void PageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
}
void RootDeviceEnvironment::initApiGfxCoreHelper() {