From 80dc4fb43a4252a348956357bb6954b99481940a Mon Sep 17 00:00:00 2001 From: "Chandio, Bibrak Qamar" Date: Tue, 31 Dec 2024 13:48:34 +0000 Subject: [PATCH] fix: Set vmbind user fence when makeMemoryResident Related-To: NEO-11977, GSD-10293 Signed-off-by: Chandio, Bibrak Qamar --- .../core/source/context/context_imp.cpp | 10 +- level_zero/core/source/module/module_imp.cpp | 2 +- opencl/source/mem_obj/buffer.cpp | 2 +- .../windows/wddm_residency_handler_tests.cpp | 16 +- shared/source/debugger/debugger_l0.cpp | 8 +- .../direct_submission_hw.inl | 4 +- .../source/helpers/bindless_heaps_helper.cpp | 4 +- .../memory_operations_handler.h | 6 +- .../aub_memory_operations_handler.cpp | 10 +- .../aub_memory_operations_handler.h | 6 +- .../os_interface/linux/drm_allocation.cpp | 20 +- .../os_interface/linux/drm_allocation.h | 8 +- .../os_interface/linux/drm_buffer_object.cpp | 14 +- .../os_interface/linux/drm_buffer_object.h | 2 +- .../os_interface/linux/drm_command_stream.inl | 10 +- .../os_interface/linux/drm_memory_manager.cpp | 6 +- .../drm_memory_operations_handler_bind.cpp | 19 +- .../drm_memory_operations_handler_bind.h | 6 +- .../drm_memory_operations_handler_default.cpp | 10 +- .../drm_memory_operations_handler_default.h | 6 +- ..._memory_operations_handler_with_aub_dump.h | 16 +- shared/source/os_interface/linux/drm_neo.cpp | 54 ++- shared/source/os_interface/linux/drm_neo.h | 3 +- .../os_interface/linux/os_context_linux.cpp | 24 +- .../os_interface/linux/os_context_linux.h | 4 +- .../wddm_memory_operations_handler.cpp | 6 +- .../windows/wddm_memory_operations_handler.h | 8 +- ..._memory_operations_handler_with_aub_dump.h | 14 +- .../wddm_residency_allocations_container.cpp | 8 +- .../wddm_residency_allocations_container.h | 4 +- .../common/mocks/linux/mock_drm_allocation.h | 8 +- .../mock_aub_memory_operations_handler.h | 10 +- .../mocks/mock_memory_operations_handler.h | 14 +- .../linux/drm_command_stream_fixture.h | 6 +- .../aub_command_stream_receiver_1_tests.cpp | 2 +- .../tbx_command_stream_tests.cpp | 2 +- .../memory_manager/memory_manager_tests.cpp | 2 +- .../aub_memory_operations_handler_tests.cpp | 44 +-- .../os_interface/linux/drm_bind_tests.cpp | 41 ++- .../linux/drm_buffer_object_tests.cpp | 328 ++++++++++++++++-- .../linux/drm_command_stream_tests.cpp | 22 +- .../linux/drm_command_stream_tests_1.cpp | 14 +- .../linux/drm_debug_prelim_tests.cpp | 10 +- .../linux/drm_memory_manager_tests.cpp | 6 +- ...m_memory_operations_handler_bind_tests.cpp | 96 ++--- ...emory_operations_handler_default_tests.cpp | 18 +- ...operations_handler_with_aub_dump_tests.cpp | 8 +- .../linux/drm_query_prelim_tests.cpp | 8 +- .../linux/drm_vm_bind_prelim_tests.cpp | 16 +- .../linux/os_context_linux_tests.cpp | 48 ++- ...operations_handler_with_aub_dump_tests.cpp | 16 +- 51 files changed, 725 insertions(+), 304 deletions(-) diff --git a/level_zero/core/source/context/context_imp.cpp b/level_zero/core/source/context/context_imp.cpp index 4ff1ca9973..805a090695 100644 --- a/level_zero/core/source/context/context_imp.cpp +++ b/level_zero/core/source/context/context_imp.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2024 Intel Corporation + * Copyright (C) 2020-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -513,7 +513,7 @@ ze_result_t ContextImp::makeMemoryResident(ze_device_handle_t hDevice, void *ptr } NEO::MemoryOperationsHandler *memoryOperationsIface = neoDevice->getRootDeviceEnvironment().memoryOperationsInterface.get(); - auto success = memoryOperationsIface->makeResident(neoDevice, ArrayRef(&allocation, 1), true); + auto success = memoryOperationsIface->makeResident(neoDevice, ArrayRef(&allocation, 1), true, true); ze_result_t res = changeMemoryOperationStatusToL0ResultType(success); if (ZE_RESULT_SUCCESS == res) { @@ -557,9 +557,9 @@ ze_result_t ContextImp::makeImageResident(ze_device_handle_t hDevice, ze_image_h NEO::Device *neoDevice = L0::Device::fromHandle(hDevice)->getNEODevice(); NEO::MemoryOperationsHandler *memoryOperationsIface = neoDevice->getRootDeviceEnvironment().memoryOperationsInterface.get(); - auto success = memoryOperationsIface->makeResident(neoDevice, ArrayRef(&alloc, 1), true); + auto success = memoryOperationsIface->makeResident(neoDevice, ArrayRef(&alloc, 1), true, false); if (implicitArgsAlloc) { - memoryOperationsIface->makeResident(neoDevice, ArrayRef(&implicitArgsAlloc, 1), true); + memoryOperationsIface->makeResident(neoDevice, ArrayRef(&implicitArgsAlloc, 1), true, false); } return changeMemoryOperationStatusToL0ResultType(success); } @@ -1312,7 +1312,7 @@ ze_result_t ContextImp::mapVirtualMem(const void *ptr, virtualMemoryReservation->mappedAllocations.insert(std::pair(const_cast(ptr), mappedRange)); this->driverHandle->getSvmAllocsManager()->insertSVMAlloc(allocData); NEO::MemoryOperationsHandler *memoryOperationsIface = allocationNode->device->getRootDeviceEnvironment().memoryOperationsInterface.get(); - auto success = memoryOperationsIface->makeResident(allocationNode->device, ArrayRef(&allocationNode->allocation, 1), false); + auto success = memoryOperationsIface->makeResident(allocationNode->device, ArrayRef(&allocationNode->allocation, 1), false, false); ze_result_t res = changeMemoryOperationStatusToL0ResultType(success); return res; } else { diff --git a/level_zero/core/source/module/module_imp.cpp b/level_zero/core/source/module/module_imp.cpp index d5906025af..9f1effdedc 100644 --- a/level_zero/core/source/module/module_imp.cpp +++ b/level_zero/core/source/module/module_imp.cpp @@ -1048,7 +1048,7 @@ void ModuleImp::copyPatchedSegments(const NEO::Linker::PatchableSegments &isaSeg NEO::MemoryOperationsHandler *memoryOperationsIface = rootDeviceEnvironment.memoryOperationsInterface.get(); auto allocation = kernelImmData->getIsaGraphicsAllocation(); if (memoryOperationsIface) { - memoryOperationsIface->makeResident(neoDevice, ArrayRef(&allocation, 1), false); + memoryOperationsIface->makeResident(neoDevice, ArrayRef(&allocation, 1), false, false); } } } diff --git a/opencl/source/mem_obj/buffer.cpp b/opencl/source/mem_obj/buffer.cpp index 6dd9d464f0..91add875e7 100644 --- a/opencl/source/mem_obj/buffer.cpp +++ b/opencl/source/mem_obj/buffer.cpp @@ -575,7 +575,7 @@ Buffer *Buffer::create(Context *context, auto device = context->getDevice(deviceNum); auto graphicsAllocation = pBuffer->getGraphicsAllocation(device->getRootDeviceIndex()); auto rootDeviceEnvironment = pBuffer->executionEnvironment->rootDeviceEnvironments[device->getRootDeviceIndex()].get(); - rootDeviceEnvironment->memoryOperationsInterface->makeResident(&device->getDevice(), ArrayRef(&graphicsAllocation, 1), false); + rootDeviceEnvironment->memoryOperationsInterface->makeResident(&device->getDevice(), ArrayRef(&graphicsAllocation, 1), false, false); } } diff --git a/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp index 86b0aad6c6..1d7f59b372 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 Intel Corporation + * Copyright (C) 2019-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -58,7 +58,7 @@ struct WddmMemoryOperationsHandlerTest : public WddmTest { TEST_F(WddmMemoryOperationsHandlerTest, givenRegularAllocationWhenMakingResidentAllocationThenMakeResidentIsCalledAndAllocationIsMarkedAsExplicitlyResident) { wddmAllocation->setExplicitlyMadeResident(false); - EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmAllocation), MemoryOperationsStatus::success); EXPECT_TRUE(wddmAllocation->isExplicitlyMadeResident()); } @@ -67,7 +67,7 @@ TEST_F(WddmMemoryOperationsHandlerTest, givenFragmentedAllocationWhenMakingResid allocationPtr = wddmFragmentedAllocation.get(); allocationPtr->setExplicitlyMadeResident(false); - EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success); EXPECT_TRUE(allocationPtr->isExplicitlyMadeResident()); } @@ -78,7 +78,7 @@ TEST_F(WddmMemoryOperationsHandlerTest, givenVariousAllocationsWhenMakingResiden allocation->setExplicitlyMadeResident(false); } - EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(allocationData), false), MemoryOperationsStatus::success); + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(allocationData), false, false), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmAllocation), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success); @@ -91,7 +91,7 @@ TEST_F(WddmMemoryOperationsHandlerTest, givenRegularAllocationWhenEvictingReside wddm->callBaseEvict = true; allocationPtr->setExplicitlyMadeResident(false); - EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false, false), MemoryOperationsStatus::success); EXPECT_TRUE(allocationPtr->isExplicitlyMadeResident()); EXPECT_EQ(wddmMemoryOperationsHandler->evict(nullptr, *wddmAllocation), MemoryOperationsStatus::success); @@ -105,7 +105,7 @@ TEST_F(WddmMemoryOperationsHandlerTest, givenFragmentedAllocationWhenEvictingRes allocationPtr = wddmFragmentedAllocation.get(); allocationPtr->setExplicitlyMadeResident(false); - EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false, false), MemoryOperationsStatus::success); EXPECT_TRUE(allocationPtr->isExplicitlyMadeResident()); EXPECT_EQ(wddmMemoryOperationsHandler->evict(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success); @@ -119,7 +119,7 @@ TEST_F(WddmMemoryOperationsHandlerTest, givenVariousAllocationsWhenEvictingResid allocation->setExplicitlyMadeResident(false); } wddm->evictResult.called = 0; - EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(allocationData), false), MemoryOperationsStatus::success); + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(allocationData), false, false), MemoryOperationsStatus::success); EXPECT_TRUE(wddmAllocation->isExplicitlyMadeResident()); EXPECT_TRUE(wddmFragmentedAllocation->isExplicitlyMadeResident()); @@ -143,7 +143,7 @@ TEST_F(WddmMemoryOperationsHandlerTest, givenVariousAllocationsWhenFreeResidentA allocation->setExplicitlyMadeResident(false); } wddm->evictResult.called = 0; - EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(allocationData), false), MemoryOperationsStatus::success); + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(allocationData), false, false), MemoryOperationsStatus::success); EXPECT_TRUE(wddmAllocation->isExplicitlyMadeResident()); EXPECT_TRUE(wddmFragmentedAllocation->isExplicitlyMadeResident()); diff --git a/shared/source/debugger/debugger_l0.cpp b/shared/source/debugger/debugger_l0.cpp index 1261ac32ee..e9c216c327 100644 --- a/shared/source/debugger/debugger_l0.cpp +++ b/shared/source/debugger/debugger_l0.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2024 Intel Corporation + * Copyright (C) 2020-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -98,11 +98,11 @@ void DebuggerL0::initialize() { NEO::MemoryOperationsHandler *memoryOperationsIface = rootDeviceEnvironment.memoryOperationsInterface.get(); if (memoryOperationsIface) { - memoryOperationsIface->makeResident(device, ArrayRef(&moduleDebugArea, 1), false); + memoryOperationsIface->makeResident(device, ArrayRef(&moduleDebugArea, 1), false, false); auto numSubDevices = device->getNumSubDevices(); for (uint32_t i = 0; i < numSubDevices; i++) { auto subDevice = device->getSubDevice(i); - memoryOperationsIface->makeResident(subDevice, ArrayRef(&moduleDebugArea, 1), false); + memoryOperationsIface->makeResident(subDevice, ArrayRef(&moduleDebugArea, 1), false, false); } } @@ -142,7 +142,7 @@ void DebuggerL0::notifyModuleLoadAllocations(Device *device, const StackVecgetRootDeviceEnvironment().memoryOperationsInterface.get(); if (memoryOperationsIface) { for (auto gfxAlloc : allocs) { - memoryOperationsIface->makeResident(device, ArrayRef(&gfxAlloc, 1), false); + memoryOperationsIface->makeResident(device, ArrayRef(&gfxAlloc, 1), false, false); } } } diff --git a/shared/source/direct_submission/direct_submission_hw.inl b/shared/source/direct_submission/direct_submission_hw.inl index 0d0092cee3..1934b9040f 100644 --- a/shared/source/direct_submission/direct_submission_hw.inl +++ b/shared/source/direct_submission/direct_submission_hw.inl @@ -420,7 +420,7 @@ bool DirectSubmissionHw::allocateResources() { template bool DirectSubmissionHw::makeResourcesResident(DirectSubmissionAllocations &allocations) { - auto ret = memoryOperationHandler->makeResidentWithinOsContext(&this->osContext, ArrayRef(allocations), false) == MemoryOperationsStatus::success; + auto ret = memoryOperationHandler->makeResidentWithinOsContext(&this->osContext, ArrayRef(allocations), false, false) == MemoryOperationsStatus::success; return ret; } @@ -1140,7 +1140,7 @@ inline GraphicsAllocation *DirectSubmissionHw::switchRing nextAllocation = memoryManager->allocateGraphicsMemoryWithProperties(commandStreamAllocationProperties); this->currentRingBuffer = static_cast(this->ringBuffers.size()); this->ringBuffers.emplace_back(0ull, nextAllocation); - auto ret = memoryOperationHandler->makeResidentWithinOsContext(&this->osContext, ArrayRef(&nextAllocation, 1u), false) == MemoryOperationsStatus::success; + auto ret = memoryOperationHandler->makeResidentWithinOsContext(&this->osContext, ArrayRef(&nextAllocation, 1u), false, false) == MemoryOperationsStatus::success; UNRECOVERABLE_IF(!ret); } } diff --git a/shared/source/helpers/bindless_heaps_helper.cpp b/shared/source/helpers/bindless_heaps_helper.cpp index 726b0640b8..5f61d32ded 100644 --- a/shared/source/helpers/bindless_heaps_helper.cpp +++ b/shared/source/helpers/bindless_heaps_helper.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2024 Intel Corporation + * Copyright (C) 2020-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -153,7 +153,7 @@ GraphicsAllocation *BindlessHeapsHelper::getHeapAllocation(size_t heapSize, size GraphicsAllocation *allocation = memManager->allocateGraphicsMemoryWithProperties(properties); MemoryOperationsHandler *memoryOperationsIface = rootDevice->getRootDeviceEnvironmentRef().memoryOperationsInterface.get(); - auto result = memoryOperationsIface->makeResident(rootDevice, ArrayRef(&allocation, 1), false); + auto result = memoryOperationsIface->makeResident(rootDevice, ArrayRef(&allocation, 1), false, false); if (result != NEO::MemoryOperationsStatus::success) { memManager->freeGraphicsMemory(allocation); return nullptr; diff --git a/shared/source/memory_manager/memory_operations_handler.h b/shared/source/memory_manager/memory_operations_handler.h index 97407621f3..c4fa190ff1 100644 --- a/shared/source/memory_manager/memory_operations_handler.h +++ b/shared/source/memory_manager/memory_operations_handler.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 Intel Corporation + * Copyright (C) 2019-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -20,13 +20,13 @@ class MemoryOperationsHandler { MemoryOperationsHandler() = default; virtual ~MemoryOperationsHandler() = default; - virtual MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded) = 0; + virtual MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) = 0; virtual MemoryOperationsStatus lock(Device *device, ArrayRef gfxAllocations) = 0; virtual MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) = 0; virtual MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) = 0; virtual MemoryOperationsStatus free(Device *device, GraphicsAllocation &gfxAllocation) { return MemoryOperationsStatus::success; } - virtual MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) = 0; + virtual MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence) = 0; virtual MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) = 0; virtual void processFlushResidency(CommandStreamReceiver *csr) {} }; diff --git a/shared/source/os_interface/aub_memory_operations_handler.cpp b/shared/source/os_interface/aub_memory_operations_handler.cpp index abfec8f6d4..8a593095da 100644 --- a/shared/source/os_interface/aub_memory_operations_handler.cpp +++ b/shared/source/os_interface/aub_memory_operations_handler.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 Intel Corporation + * Copyright (C) 2019-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -26,7 +26,7 @@ AubMemoryOperationsHandler::AubMemoryOperationsHandler(aub_stream::AubManager *a this->aubManager = aubManager; } -MemoryOperationsStatus AubMemoryOperationsHandler::makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded) { +MemoryOperationsStatus AubMemoryOperationsHandler::makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) { if (!aubManager) { return MemoryOperationsStatus::deviceUninitialized; } @@ -76,7 +76,7 @@ MemoryOperationsStatus AubMemoryOperationsHandler::makeResident(Device *device, } MemoryOperationsStatus AubMemoryOperationsHandler::lock(Device *device, ArrayRef gfxAllocations) { - return makeResident(device, gfxAllocations, false); + return makeResident(device, gfxAllocations, false, false); } MemoryOperationsStatus AubMemoryOperationsHandler::evict(Device *device, GraphicsAllocation &gfxAllocation) { @@ -98,8 +98,8 @@ MemoryOperationsStatus AubMemoryOperationsHandler::free(Device *device, Graphics return MemoryOperationsStatus::success; } -MemoryOperationsStatus AubMemoryOperationsHandler::makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) { - return makeResident(nullptr, gfxAllocations, false); +MemoryOperationsStatus AubMemoryOperationsHandler::makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence) { + return makeResident(nullptr, gfxAllocations, false, forcePagingFence); } MemoryOperationsStatus AubMemoryOperationsHandler::evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) { diff --git a/shared/source/os_interface/aub_memory_operations_handler.h b/shared/source/os_interface/aub_memory_operations_handler.h index a4f56d844d..0524b78309 100644 --- a/shared/source/os_interface/aub_memory_operations_handler.h +++ b/shared/source/os_interface/aub_memory_operations_handler.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 Intel Corporation + * Copyright (C) 2019-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -22,13 +22,13 @@ class AubMemoryOperationsHandler : public MemoryOperationsHandler { AubMemoryOperationsHandler(aub_stream::AubManager *aubManager); ~AubMemoryOperationsHandler() override = default; - MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded) override; + MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) override; MemoryOperationsStatus lock(Device *device, ArrayRef gfxAllocations) override; MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override; MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override; MemoryOperationsStatus free(Device *device, GraphicsAllocation &gfxAllocation) override; - MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) override; + MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence) override; MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override; void processFlushResidency(CommandStreamReceiver *csr) override; diff --git a/shared/source/os_interface/linux/drm_allocation.cpp b/shared/source/os_interface/linux/drm_allocation.cpp index ca748db46b..1092f9fc6a 100644 --- a/shared/source/os_interface/linux/drm_allocation.cpp +++ b/shared/source/os_interface/linux/drm_allocation.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2024 Intel Corporation + * Copyright (C) 2020-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -241,11 +241,11 @@ bool DrmAllocation::prefetchBOWithChunking(Drm *drm) { return success; } -int DrmAllocation::makeBOsResident(OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind) { +int DrmAllocation::makeBOsResident(OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind, const bool forcePagingFence) { if (this->fragmentsStorage.fragmentCount) { for (unsigned int f = 0; f < this->fragmentsStorage.fragmentCount; f++) { if (!this->fragmentsStorage.fragmentStorageData[f].residency->resident[osContext->getContextId()]) { - int retVal = bindBO(static_cast(this->fragmentsStorage.fragmentStorageData[f].osHandleStorage)->bo, osContext, vmHandleId, bufferObjects, bind); + int retVal = bindBO(static_cast(this->fragmentsStorage.fragmentStorageData[f].osHandleStorage)->bo, osContext, vmHandleId, bufferObjects, bind, forcePagingFence); if (retVal) { return retVal; } @@ -253,7 +253,7 @@ int DrmAllocation::makeBOsResident(OsContext *osContext, uint32_t vmHandleId, st } } } else { - int retVal = bindBOs(osContext, vmHandleId, bufferObjects, bind); + int retVal = bindBOs(osContext, vmHandleId, bufferObjects, bind, forcePagingFence); if (retVal) { return retVal; } @@ -262,7 +262,7 @@ int DrmAllocation::makeBOsResident(OsContext *osContext, uint32_t vmHandleId, st return 0; } -int DrmAllocation::bindBO(BufferObject *bo, OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind) { +int DrmAllocation::bindBO(BufferObject *bo, OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind, const bool forcePagingFence) { auto retVal = 0; if (bo) { bo->requireExplicitResidency(bo->peekDrm()->hasPageFaultSupport() && !shouldAllocationPageFault(bo->peekDrm())); @@ -279,7 +279,7 @@ int DrmAllocation::bindBO(BufferObject *bo, OsContext *osContext, uint32_t vmHan } else { if (bind) { - retVal = bo->bind(osContext, vmHandleId); + retVal = bo->bind(osContext, vmHandleId, forcePagingFence); } else { retVal = bo->unbind(osContext, vmHandleId); } @@ -288,19 +288,19 @@ int DrmAllocation::bindBO(BufferObject *bo, OsContext *osContext, uint32_t vmHan return retVal; } -int DrmAllocation::bindBOs(OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind) { +int DrmAllocation::bindBOs(OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind, const bool forcePagingFence) { int retVal = 0; if (this->storageInfo.getNumBanks() > 1) { auto &bos = this->getBOs(); if (this->storageInfo.tileInstanced) { auto bo = bos[vmHandleId]; - retVal = bindBO(bo, osContext, vmHandleId, bufferObjects, bind); + retVal = bindBO(bo, osContext, vmHandleId, bufferObjects, bind, forcePagingFence); if (retVal) { return retVal; } } else { for (auto bo : bos) { - retVal = bindBO(bo, osContext, vmHandleId, bufferObjects, bind); + retVal = bindBO(bo, osContext, vmHandleId, bufferObjects, bind, forcePagingFence); if (retVal) { return retVal; } @@ -308,7 +308,7 @@ int DrmAllocation::bindBOs(OsContext *osContext, uint32_t vmHandleId, std::vecto } } else { auto bo = this->getBO(); - retVal = bindBO(bo, osContext, vmHandleId, bufferObjects, bind); + retVal = bindBO(bo, osContext, vmHandleId, bufferObjects, bind, forcePagingFence); if (retVal) { return retVal; } diff --git a/shared/source/os_interface/linux/drm_allocation.h b/shared/source/os_interface/linux/drm_allocation.h index 432062f94e..1a58bf2cc1 100644 --- a/shared/source/os_interface/linux/drm_allocation.h +++ b/shared/source/os_interface/linux/drm_allocation.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2024 Intel Corporation + * Copyright (C) 2018-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -130,9 +130,9 @@ class DrmAllocation : public GraphicsAllocation { } bool prefetchBOWithChunking(Drm *drm); - MOCKABLE_VIRTUAL int makeBOsResident(OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind); - MOCKABLE_VIRTUAL int bindBO(BufferObject *bo, OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind); - MOCKABLE_VIRTUAL int bindBOs(OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind); + MOCKABLE_VIRTUAL int makeBOsResident(OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind, const bool forcePagingFence); + MOCKABLE_VIRTUAL int bindBO(BufferObject *bo, OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind, const bool forcePagingFence); + MOCKABLE_VIRTUAL int bindBOs(OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind, const bool forcePagingFence); MOCKABLE_VIRTUAL bool prefetchBO(BufferObject *bo, uint32_t vmHandleId, uint32_t subDeviceId); MOCKABLE_VIRTUAL void registerBOBindExtHandle(Drm *drm); void addRegisteredBoBindHandle(uint32_t handle) { registeredBoBindHandles.push_back(handle); } diff --git a/shared/source/os_interface/linux/drm_buffer_object.cpp b/shared/source/os_interface/linux/drm_buffer_object.cpp index 2532d9c3e1..656efbc495 100644 --- a/shared/source/os_interface/linux/drm_buffer_object.cpp +++ b/shared/source/os_interface/linux/drm_buffer_object.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2024 Intel Corporation + * Copyright (C) 2018-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -262,11 +262,11 @@ void BufferObject::printBOBindingResult(OsContext *osContext, uint32_t vmHandleI } } -int BufferObject::bind(OsContext *osContext, uint32_t vmHandleId) { +int BufferObject::bind(OsContext *osContext, uint32_t vmHandleId, const bool forcePagingFence) { int retVal = 0; auto contextId = getOsContextId(osContext); if (!this->bindInfo[contextId][vmHandleId]) { - retVal = this->drm->bindBufferObject(osContext, vmHandleId, this); + retVal = this->drm->bindBufferObject(osContext, vmHandleId, this, forcePagingFence); if (debugManager.flags.PrintBOBindingResult.get()) { printBOBindingResult(osContext, vmHandleId, true, retVal); } @@ -307,13 +307,13 @@ void BufferObject::printExecutionBuffer(ExecBuffer &execbuf, const size_t &resid printf("%s\n", logger.str().c_str()); } -int bindBOsWithinContext(BufferObject *const boToPin[], size_t numberOfBos, OsContext *osContext, uint32_t vmHandleId) { +int bindBOsWithinContext(BufferObject *const boToPin[], size_t numberOfBos, OsContext *osContext, uint32_t vmHandleId, const bool forcePagingFence) { auto retVal = 0; for (auto drmIterator = 0u; drmIterator < osContext->getDeviceBitfield().size(); drmIterator++) { if (osContext->getDeviceBitfield().test(drmIterator)) { for (size_t i = 0; i < numberOfBos; i++) { - retVal |= boToPin[i]->bind(osContext, drmIterator); + retVal |= boToPin[i]->bind(osContext, drmIterator, forcePagingFence); } } } @@ -325,7 +325,7 @@ int BufferObject::pin(BufferObject *const boToPin[], size_t numberOfBos, OsConte auto retVal = 0; if (this->drm->isVmBindAvailable()) { - retVal = bindBOsWithinContext(boToPin, numberOfBos, osContext, vmHandleId); + retVal = bindBOsWithinContext(boToPin, numberOfBos, osContext, vmHandleId, false); } else { StackVec execObject(numberOfBos + 1); retVal = this->exec(4u, 0u, 0u, false, osContext, vmHandleId, drmContextId, boToPin, numberOfBos, &execObject[0], 0, 0); @@ -339,7 +339,7 @@ int BufferObject::validateHostPtr(BufferObject *const boToPin[], size_t numberOf if (this->drm->isVmBindAvailable()) { for (size_t i = 0; i < numberOfBos; i++) { - retVal = boToPin[i]->bind(osContext, vmHandleId); + retVal = boToPin[i]->bind(osContext, vmHandleId, false); if (retVal) { break; } diff --git a/shared/source/os_interface/linux/drm_buffer_object.h b/shared/source/os_interface/linux/drm_buffer_object.h index 013186e5e8..046e03ce53 100644 --- a/shared/source/os_interface/linux/drm_buffer_object.h +++ b/shared/source/os_interface/linux/drm_buffer_object.h @@ -114,7 +114,7 @@ class BufferObject { MOCKABLE_VIRTUAL int exec(uint32_t used, size_t startOffset, unsigned int flags, bool requiresCoherency, OsContext *osContext, uint32_t vmHandleId, uint32_t drmContextId, BufferObject *const residency[], size_t residencyCount, ExecObject *execObjectsStorage, uint64_t completionGpuAddress, TaskCountType completionValue); - int bind(OsContext *osContext, uint32_t vmHandleId); + int bind(OsContext *osContext, uint32_t vmHandleId, const bool forcePagingFence); int unbind(OsContext *osContext, uint32_t vmHandleId); void printExecutionBuffer(ExecBuffer &execbuf, const size_t &residencyCount, ExecObject *execObjectsStorage, BufferObject *const residency[]); diff --git a/shared/source/os_interface/linux/drm_command_stream.inl b/shared/source/os_interface/linux/drm_command_stream.inl index a8c9d7c1df..65d32cb0e3 100644 --- a/shared/source/os_interface/linux/drm_command_stream.inl +++ b/shared/source/os_interface/linux/drm_command_stream.inl @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2024 Intel Corporation + * Copyright (C) 2018-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -190,13 +190,13 @@ SubmissionStatus DrmCommandStreamReceiver::printBOsForSubmit(Residenc if (osContext->getDeviceBitfield().test(drmIterator)) { for (auto gfxAllocation = allocationsForResidency.begin(); gfxAllocation != allocationsForResidency.end(); gfxAllocation++) { auto drmAllocation = static_cast(*gfxAllocation); - auto retCode = drmAllocation->makeBOsResident(osContext, drmIterator, &bosForSubmit, true); + auto retCode = drmAllocation->makeBOsResident(osContext, drmIterator, &bosForSubmit, true, false); if (retCode) { return Drm::getSubmissionStatusFromReturnCode(retCode); } } auto drmCmdBufferAllocation = static_cast(&cmdBufferAllocation); - auto retCode = drmCmdBufferAllocation->makeBOsResident(osContext, drmIterator, &bosForSubmit, true); + auto retCode = drmCmdBufferAllocation->makeBOsResident(osContext, drmIterator, &bosForSubmit, true, false); if (retCode) { return Drm::getSubmissionStatusFromReturnCode(retCode); } @@ -260,7 +260,7 @@ SubmissionStatus DrmCommandStreamReceiver::processResidency(Residency int ret = 0; for (auto &alloc : inputAllocationsForResidency) { auto drmAlloc = static_cast(alloc); - ret = drmAlloc->makeBOsResident(osContext, handleId, &this->residency, false); + ret = drmAlloc->makeBOsResident(osContext, handleId, &this->residency, false, false); if (ret != 0) { break; } @@ -389,4 +389,4 @@ template bool DrmCommandStreamReceiver::isKmdWaitOnTaskCountAllowed() const { return this->isDirectSubmissionEnabled(); } -} // namespace NEO +} // namespace NEO \ No newline at end of file diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index 87b485a5cf..cfe5384988 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -306,7 +306,7 @@ bool DrmMemoryManager::setMemPrefetch(GraphicsAllocation *gfxAllocation, SubDevi for (auto subDeviceId : subDeviceIds) { auto vmHandleId = subDeviceId; - auto retVal = drmAllocation->bindBOs(osContextLinux, vmHandleId, nullptr, true); + auto retVal = drmAllocation->bindBOs(osContextLinux, vmHandleId, nullptr, true, false); if (retVal != 0) { DEBUG_BREAK_IF(true); return false; @@ -736,7 +736,7 @@ bool DrmMemoryManager::mapPhysicalHostMemoryToVirtualMemory(RootDeviceIndicesCon bo->setMmapOffset(mmapOffset); bo->setAddress(gpuRange); - bo->bind(getDefaultOsContext(drmPhysicalAllocation->getRootDeviceIndex()), 0); + bo->bind(getDefaultOsContext(drmPhysicalAllocation->getRootDeviceIndex()), 0, false); auto drmAllocation = new DrmAllocation(drmPhysicalAllocation->getRootDeviceIndex(), 1u, AllocationType::bufferHostMemory, bo, addrToPtr(bo->peekAddress()), bo->peekSize(), static_cast(internalHandle), memoryPool, getGmmHelper(drmPhysicalAllocation->getRootDeviceIndex())->canonize(bo->peekAddress())); @@ -1669,7 +1669,7 @@ bool DrmMemoryManager::makeAllocationResident(GraphicsAllocation *allocation) { auto drmAllocation = static_cast(allocation); auto rootDeviceIndex = allocation->getRootDeviceIndex(); for (uint32_t i = 0; getDrm(rootDeviceIndex).getVirtualMemoryAddressSpace(i) > 0u; i++) { - if (drmAllocation->makeBOsResident(getDefaultOsContext(rootDeviceIndex), i, nullptr, true)) { + if (drmAllocation->makeBOsResident(getDefaultOsContext(rootDeviceIndex), i, nullptr, true, false)) { return false; } getDrm(rootDeviceIndex).waitForBind(i); diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler_bind.cpp b/shared/source/os_interface/linux/drm_memory_operations_handler_bind.cpp index c52809b106..74cbe41709 100644 --- a/shared/source/os_interface/linux/drm_memory_operations_handler_bind.cpp +++ b/shared/source/os_interface/linux/drm_memory_operations_handler_bind.cpp @@ -26,12 +26,12 @@ DrmMemoryOperationsHandlerBind::DrmMemoryOperationsHandlerBind(const RootDeviceE DrmMemoryOperationsHandlerBind::~DrmMemoryOperationsHandlerBind() = default; -MemoryOperationsStatus DrmMemoryOperationsHandlerBind::makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded) { +MemoryOperationsStatus DrmMemoryOperationsHandlerBind::makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) { auto &engines = device->getAllEngines(); MemoryOperationsStatus result = MemoryOperationsStatus::success; for (const auto &engine : engines) { engine.commandStreamReceiver->initializeResources(false, device->getPreemptionMode()); - result = this->makeResidentWithinOsContext(engine.osContext, gfxAllocations, false); + result = this->makeResidentWithinOsContext(engine.osContext, gfxAllocations, false, forcePagingFence); if (result != MemoryOperationsStatus::success) { break; } @@ -43,10 +43,10 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerBind::lock(Device *device, Arra for (auto gfxAllocation = gfxAllocations.begin(); gfxAllocation != gfxAllocations.end(); gfxAllocation++) { (*gfxAllocation)->setLockedMemory(true); } - return makeResident(device, gfxAllocations, false); + return makeResident(device, gfxAllocations, false, false); } -MemoryOperationsStatus DrmMemoryOperationsHandlerBind::makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) { +MemoryOperationsStatus DrmMemoryOperationsHandlerBind::makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence) { auto deviceBitfield = osContext->getDeviceBitfield(); std::lock_guard lock(mutex); @@ -68,7 +68,8 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerBind::makeResidentWithinOsConte if (!bo->getBindInfo()[bo->getOsContextId(osContext)][drmIterator]) { bo->requireExplicitLockedMemory(drmAllocation->isLockedMemory()); bo->requireImmediateBinding(true); - int result = drmAllocation->makeBOsResident(osContext, drmIterator, nullptr, true); + + int result = drmAllocation->makeBOsResident(osContext, drmIterator, nullptr, true, forcePagingFence); if (result) { return MemoryOperationsStatus::outOfMemory; } @@ -108,7 +109,7 @@ int DrmMemoryOperationsHandlerBind::evictImpl(OsContext *osContext, GraphicsAllo auto drmAllocation = static_cast(&gfxAllocation); for (auto drmIterator = 0u; drmIterator < deviceBitfield.size(); drmIterator++) { if (deviceBitfield.test(drmIterator)) { - int retVal = drmAllocation->makeBOsResident(osContext, drmIterator, nullptr, false); + int retVal = drmAllocation->makeBOsResident(osContext, drmIterator, nullptr, false, false); if (retVal) { return retVal; } @@ -143,11 +144,11 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerBind::mergeWithResidencyContain auto memoryManager = static_cast(this->rootDeviceEnvironment.executionEnvironment.memoryManager.get()); auto allocLock = memoryManager->acquireAllocLock(); - this->makeResidentWithinOsContext(osContext, ArrayRef(memoryManager->getSysMemAllocs()), true); - this->makeResidentWithinOsContext(osContext, ArrayRef(memoryManager->getLocalMemAllocs(this->rootDeviceIndex)), true); + this->makeResidentWithinOsContext(osContext, ArrayRef(memoryManager->getSysMemAllocs()), true, false); + this->makeResidentWithinOsContext(osContext, ArrayRef(memoryManager->getLocalMemAllocs(this->rootDeviceIndex)), true, false); } - auto retVal = this->makeResidentWithinOsContext(osContext, ArrayRef(residencyContainer), true); + auto retVal = this->makeResidentWithinOsContext(osContext, ArrayRef(residencyContainer), true, false); if (retVal != MemoryOperationsStatus::success) { return retVal; } diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler_bind.h b/shared/source/os_interface/linux/drm_memory_operations_handler_bind.h index e259cd5a72..38c7fe828f 100644 --- a/shared/source/os_interface/linux/drm_memory_operations_handler_bind.h +++ b/shared/source/os_interface/linux/drm_memory_operations_handler_bind.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2024 Intel Corporation + * Copyright (C) 2020-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -16,8 +16,8 @@ class DrmMemoryOperationsHandlerBind : public DrmMemoryOperationsHandler { DrmMemoryOperationsHandlerBind(const RootDeviceEnvironment &rootDeviceEnvironment, uint32_t rootDeviceIndex); ~DrmMemoryOperationsHandlerBind() override; - MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) override; - MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded) override; + MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence) override; + MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) override; MemoryOperationsStatus lock(Device *device, ArrayRef gfxAllocations) override; MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override; MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override; diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler_default.cpp b/shared/source/os_interface/linux/drm_memory_operations_handler_default.cpp index c68ef079d4..6ba5fc7cbe 100644 --- a/shared/source/os_interface/linux/drm_memory_operations_handler_default.cpp +++ b/shared/source/os_interface/linux/drm_memory_operations_handler_default.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2024 Intel Corporation + * Copyright (C) 2020-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -23,15 +23,15 @@ DrmMemoryOperationsHandlerDefault::DrmMemoryOperationsHandlerDefault(uint32_t ro ; DrmMemoryOperationsHandlerDefault::~DrmMemoryOperationsHandlerDefault() = default; -MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) { +MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence) { std::lock_guard lock(mutex); this->residency.insert(gfxAllocations.begin(), gfxAllocations.end()); return MemoryOperationsStatus::success; } -MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded) { +MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) { OsContext *osContext = nullptr; - auto ret = this->makeResidentWithinOsContext(osContext, gfxAllocations, false); + auto ret = this->makeResidentWithinOsContext(osContext, gfxAllocations, false, forcePagingFence); if (!isDummyExecNeeded || ret != MemoryOperationsStatus::success) { return ret; } @@ -53,7 +53,7 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::lock(Device *device, A bo->requireExplicitLockedMemory(true); } } - return this->makeResidentWithinOsContext(osContext, gfxAllocations, false); + return this->makeResidentWithinOsContext(osContext, gfxAllocations, false, false); } MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) { diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler_default.h b/shared/source/os_interface/linux/drm_memory_operations_handler_default.h index 165f37b52a..8b20e785b3 100644 --- a/shared/source/os_interface/linux/drm_memory_operations_handler_default.h +++ b/shared/source/os_interface/linux/drm_memory_operations_handler_default.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2024 Intel Corporation + * Copyright (C) 2020-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -19,8 +19,8 @@ class DrmMemoryOperationsHandlerDefault : public DrmMemoryOperationsHandler { DrmMemoryOperationsHandlerDefault(const RootDeviceEnvironment &rootDeviceEnvironment, uint32_t rootDeviceIndex) : DrmMemoryOperationsHandlerDefault(rootDeviceIndex) {} ~DrmMemoryOperationsHandlerDefault() override; - MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) override; - MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded) override; + MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence) override; + MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) override; MemoryOperationsStatus lock(Device *device, ArrayRef gfxAllocations) override; MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override; MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override; diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler_with_aub_dump.h b/shared/source/os_interface/linux/drm_memory_operations_handler_with_aub_dump.h index a85a1fe0fc..b09d0fe521 100644 --- a/shared/source/os_interface/linux/drm_memory_operations_handler_with_aub_dump.h +++ b/shared/source/os_interface/linux/drm_memory_operations_handler_with_aub_dump.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023-2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -36,13 +36,13 @@ class DrmMemoryOperationsHandlerWithAubDump : public BaseOperationsHandler { ~DrmMemoryOperationsHandlerWithAubDump() override = default; - MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded) override { - aubMemoryOperationsHandler->makeResident(device, gfxAllocations, isDummyExecNeeded); - return BaseOperationsHandler::makeResident(device, gfxAllocations, isDummyExecNeeded); + MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) override { + aubMemoryOperationsHandler->makeResident(device, gfxAllocations, isDummyExecNeeded, forcePagingFence); + return BaseOperationsHandler::makeResident(device, gfxAllocations, isDummyExecNeeded, forcePagingFence); } MemoryOperationsStatus lock(Device *device, ArrayRef gfxAllocations) override { - aubMemoryOperationsHandler->makeResident(device, gfxAllocations, false); + aubMemoryOperationsHandler->makeResident(device, gfxAllocations, false, false); return BaseOperationsHandler::lock(device, gfxAllocations); } @@ -61,9 +61,9 @@ class DrmMemoryOperationsHandlerWithAubDump : public BaseOperationsHandler { return BaseOperationsHandler::isResident(device, gfxAllocation); } - MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) override { - aubMemoryOperationsHandler->makeResidentWithinOsContext(osContext, gfxAllocations, evictable); - return BaseOperationsHandler::makeResidentWithinOsContext(osContext, gfxAllocations, evictable); + MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence) override { + aubMemoryOperationsHandler->makeResidentWithinOsContext(osContext, gfxAllocations, evictable, forcePagingFence); + return BaseOperationsHandler::makeResidentWithinOsContext(osContext, gfxAllocations, evictable, forcePagingFence); } MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override { diff --git a/shared/source/os_interface/linux/drm_neo.cpp b/shared/source/os_interface/linux/drm_neo.cpp index 25a63350e3..4715d33453 100644 --- a/shared/source/os_interface/linux/drm_neo.cpp +++ b/shared/source/os_interface/linux/drm_neo.cpp @@ -1278,6 +1278,15 @@ void Drm::waitForBind(uint32_t vmHandleId) { waitUserFence(0u, fenceAddress, fenceValue, ValueWidth::u64, -1, ioctlHelper->getWaitUserFenceSoftFlag(), false, NEO::InterruptId::notUsed, nullptr); } +void Drm::waitForBindGivenFenceVal(uint32_t vmHandleId, uint64_t fenceValToWait) { + if (*ioctlHelper->getPagingFenceAddress(vmHandleId, nullptr) >= fenceValToWait) { + return; + } + + auto fenceAddress = castToUint64(ioctlHelper->getPagingFenceAddress(vmHandleId, nullptr)); + waitUserFence(0u, fenceAddress, fenceValToWait, ValueWidth::u64, -1, ioctlHelper->getWaitUserFenceSoftFlag(), false, NEO::InterruptId::notUsed, nullptr); +} + bool Drm::isSetPairAvailable() { if (debugManager.flags.EnableSetPair.get() == 1) { std::call_once(checkSetPairOnce, [this]() { @@ -1406,7 +1415,7 @@ void programUserFence(Drm *drm, OsContext *osContext, BufferObject *bo, VmBindEx ioctlHelper->fillVmBindExtUserFence(vmBindExtUserFence, address, value, nextExtension); } -int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleId, BufferObject *bo, bool bind) { +int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleId, BufferObject *bo, bool bind, const bool forcePagingFence) { auto vmId = drm->getVirtualMemoryAddressSpace(vmHandleId); auto ioctlHelper = drm->getIoctlHelper(); @@ -1418,6 +1427,9 @@ int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleI vmId = osContextLinux->getDrmVmIds()[vmHandleId]; } + // Use only when debugger is disabled + const bool guaranteePagingFence = forcePagingFence && !drm->getRootDeviceEnvironment().executionEnvironment.isDebuggingEnabled(); + std::unique_ptr extensions; if (bind) { bool allowUUIDsForDebug = !osContext->isInternalEngine() && !EngineHelpers::isBcs(osContext->getEngineType()); @@ -1429,7 +1441,7 @@ int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleI bool bindMakeResident = false; bool readOnlyResource = bo->isReadOnlyGpuResource(); - if (drm->useVMBindImmediate()) { + if (drm->useVMBindImmediate() || guaranteePagingFence) { bindMakeResident = bo->isExplicitResidencyRequired(); bindImmediate = true; } @@ -1479,11 +1491,12 @@ int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleI } std::unique_lock lock; - VmBindExtUserFenceT vmBindExtUserFence{}; bool incrementFenceValue = false; + if (ioctlHelper->isWaitBeforeBindRequired(bind)) { - if (drm->useVMBindImmediate()) { + + if (drm->useVMBindImmediate() || guaranteePagingFence) { lock = drm->lockBindFenceMutex(); auto nextExtension = vmBind.extensions; incrementFenceValue = true; @@ -1491,6 +1504,7 @@ int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleI ioctlHelper->setVmBindUserFence(vmBind, vmBindExtUserFence); } } + if (bind) { ret = ioctlHelper->vmBind(vmBind); if (ret) { @@ -1505,38 +1519,46 @@ int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleI break; } } - bool waitOnUserFenceAfterBindAndUnbind = false; - if (debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.get() != -1) { - waitOnUserFenceAfterBindAndUnbind = !!debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.get(); - } - if (ioctlHelper->isWaitBeforeBindRequired(bind) && waitOnUserFenceAfterBindAndUnbind && drm->useVMBindImmediate()) { - auto osContextLinux = static_cast(osContext); - osContextLinux->waitForPagingFence(); - } + if (incrementFenceValue) { + uint64_t fenceValToWait = 0; + if (drm->isPerContextVMRequired()) { auto osContextLinux = static_cast(osContext); + fenceValToWait = osContextLinux->getNextFenceVal(vmHandleId); osContextLinux->incFenceVal(vmHandleId); } else { + fenceValToWait = drm->getNextFenceVal(vmHandleId); drm->incFenceVal(vmHandleId); } + + lock.unlock(); + + bool waitOnUserFenceAfterBindAndUnbind = false; + if (debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.get() != -1) { + waitOnUserFenceAfterBindAndUnbind = !!debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.get(); + } + if ((ioctlHelper->isWaitBeforeBindRequired(bind) && waitOnUserFenceAfterBindAndUnbind && drm->useVMBindImmediate()) || guaranteePagingFence) { + auto osContextLinux = static_cast(osContext); + osContextLinux->waitForPagingFenceGivenFenceVal(fenceValToWait); + } } } return ret; } -int Drm::bindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo) { - auto ret = changeBufferObjectBinding(this, osContext, vmHandleId, bo, true); +int Drm::bindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo, const bool forcePagingFence) { + auto ret = changeBufferObjectBinding(this, osContext, vmHandleId, bo, true, forcePagingFence); if (ret != 0) { static_cast(this->rootDeviceEnvironment.memoryOperationsInterface.get())->evictUnusedAllocations(false, false); - ret = changeBufferObjectBinding(this, osContext, vmHandleId, bo, true); + ret = changeBufferObjectBinding(this, osContext, vmHandleId, bo, true, forcePagingFence); } return ret; } int Drm::unbindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo) { - return changeBufferObjectBinding(this, osContext, vmHandleId, bo, false); + return changeBufferObjectBinding(this, osContext, vmHandleId, bo, false, false); } int Drm::createDrmVirtualMemory(uint32_t &drmVmId) { diff --git a/shared/source/os_interface/linux/drm_neo.h b/shared/source/os_interface/linux/drm_neo.h index 4bfac6cb51..a1107160b3 100644 --- a/shared/source/os_interface/linux/drm_neo.h +++ b/shared/source/os_interface/linux/drm_neo.h @@ -121,7 +121,7 @@ class Drm : public DriverModel { bool createVirtualMemoryAddressSpace(uint32_t vmCount); void destroyVirtualMemoryAddressSpace(); uint32_t getVirtualMemoryAddressSpace(uint32_t vmId) const; - MOCKABLE_VIRTUAL int bindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo); + MOCKABLE_VIRTUAL int bindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo, const bool forcePagingFence); MOCKABLE_VIRTUAL int unbindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo); int setupHardwareInfo(const DeviceDescriptor *, bool); void setupSystemInfo(HardwareInfo *hwInfo, SystemInfo *sysInfo); @@ -218,6 +218,7 @@ class Drm : public DriverModel { std::string getDeviceNode() { return hwDeviceId->getDeviceNode(); } void waitForBind(uint32_t vmHandleId); + void waitForBindGivenFenceVal(uint32_t vmHandleId, uint64_t fenceValToWait); uint64_t getNextFenceVal(uint32_t vmHandleId) { return fenceVal[vmHandleId] + 1; } void incFenceVal(uint32_t vmHandleId) { fenceVal[vmHandleId]++; } uint64_t *getFenceAddr(uint32_t vmHandleId) { return &pagingFence[vmHandleId]; } diff --git a/shared/source/os_interface/linux/os_context_linux.cpp b/shared/source/os_interface/linux/os_context_linux.cpp index 4066eb4441..0d6a8337cc 100644 --- a/shared/source/os_interface/linux/os_context_linux.cpp +++ b/shared/source/os_interface/linux/os_context_linux.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2024 Intel Corporation + * Copyright (C) 2018-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -124,6 +124,28 @@ void OsContextLinux::waitForBind(uint32_t drmIterator) { } } +void OsContextLinux::waitForPagingFenceGivenFenceVal(uint64_t fenceValToWait) { + for (auto drmIterator = 0u; drmIterator < this->deviceBitfield.size(); drmIterator++) { + if (this->deviceBitfield.test(drmIterator)) { + this->waitForBindGivenFenceVal(drmIterator, fenceValToWait); + } + } +} + +void OsContextLinux::waitForBindGivenFenceVal(uint32_t drmIterator, uint64_t fenceValToWait) { + if (drm.isPerContextVMRequired()) { + if (pagingFence[drmIterator] >= fenceValToWait) { + return; + } + + auto fenceAddress = castToUint64(&this->pagingFence[drmIterator]); + drm.waitUserFence(0u, fenceAddress, fenceValToWait, Drm::ValueWidth::u64, -1, drm.getIoctlHelper()->getWaitUserFenceSoftFlag(), false, NEO::InterruptId::notUsed, nullptr); + + } else { + drm.waitForBindGivenFenceVal(drmIterator, fenceValToWait); + } +} + void OsContextLinux::reInitializeContext() {} uint64_t OsContextLinux::getOfflineDumpContextId(uint32_t deviceIndex) const { diff --git a/shared/source/os_interface/linux/os_context_linux.h b/shared/source/os_interface/linux/os_context_linux.h index 8c10b359a4..0d6c038cc5 100644 --- a/shared/source/os_interface/linux/os_context_linux.h +++ b/shared/source/os_interface/linux/os_context_linux.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2024 Intel Corporation + * Copyright (C) 2018-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -30,6 +30,7 @@ class OsContextLinux : public OsContext { bool isDirectSubmissionSupported() const override; Drm &getDrm() const; virtual void waitForPagingFence(); + virtual void waitForPagingFenceGivenFenceVal(uint64_t fenceValToWait); static OsContext *create(OSInterface *osInterface, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor); void reInitializeContext() override; void setHangDetected() { @@ -45,6 +46,7 @@ class OsContextLinux : public OsContext { void incFenceVal(uint32_t deviceIndex) { fenceVal[deviceIndex]++; } uint64_t *getFenceAddr(uint32_t deviceIndex) { return &pagingFence[deviceIndex]; } void waitForBind(uint32_t drmIterator); + void waitForBindGivenFenceVal(uint32_t drmIterator, uint64_t fenceValToWait); protected: bool initializeContext(bool allocateInterrupt) override; diff --git a/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp b/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp index 972be56c3c..cc8e40bca6 100644 --- a/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp +++ b/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 Intel Corporation + * Copyright (C) 2019-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -20,7 +20,7 @@ WddmMemoryOperationsHandler::WddmMemoryOperationsHandler(Wddm *wddm) : wddm(wddm WddmMemoryOperationsHandler::~WddmMemoryOperationsHandler() = default; -MemoryOperationsStatus WddmMemoryOperationsHandler::makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded) { +MemoryOperationsStatus WddmMemoryOperationsHandler::makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) { uint32_t totalHandlesCount = 0; constexpr uint32_t stackAllocations = 64; constexpr uint32_t stackHandlesCount = NEO::maxFragmentsCount * EngineLimits::maxHandleCount * stackAllocations; @@ -45,7 +45,7 @@ MemoryOperationsStatus WddmMemoryOperationsHandler::makeResident(Device *device, totalHandlesCount += wddmAllocation->getNumGmms(); } } - return residentAllocations->makeResidentResources(handlesForResidency.begin(), totalHandlesCount, totalSize); + return residentAllocations->makeResidentResources(handlesForResidency.begin(), totalHandlesCount, totalSize, forcePagingFence); } MemoryOperationsStatus WddmMemoryOperationsHandler::evict(Device *device, GraphicsAllocation &gfxAllocation) { diff --git a/shared/source/os_interface/windows/wddm_memory_operations_handler.h b/shared/source/os_interface/windows/wddm_memory_operations_handler.h index d44caa244e..de36bdba1c 100644 --- a/shared/source/os_interface/windows/wddm_memory_operations_handler.h +++ b/shared/source/os_interface/windows/wddm_memory_operations_handler.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 Intel Corporation + * Copyright (C) 2019-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -23,15 +23,15 @@ class WddmMemoryOperationsHandler : public MemoryOperationsHandler { static std::unique_ptr create(Wddm *wddm, RootDeviceEnvironment *rootDeviceEnvironment, bool withAubDump); - MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded) override; + MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) override; MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override; MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override; MemoryOperationsStatus lock(Device *device, ArrayRef gfxAllocations) override { return MemoryOperationsStatus::unsupported; } - MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) override { - return makeResident(nullptr, gfxAllocations, false); + MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence) override { + return makeResident(nullptr, gfxAllocations, false, forcePagingFence); } MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override { return evict(nullptr, gfxAllocation); diff --git a/shared/source/os_interface/windows/wddm_memory_operations_handler_with_aub_dump.h b/shared/source/os_interface/windows/wddm_memory_operations_handler_with_aub_dump.h index ed0866bdeb..4214083d75 100644 --- a/shared/source/os_interface/windows/wddm_memory_operations_handler_with_aub_dump.h +++ b/shared/source/os_interface/windows/wddm_memory_operations_handler_with_aub_dump.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023-2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -36,9 +36,9 @@ class WddmMemoryOperationsHandlerWithAubDump : public BaseOperationsHandler { ~WddmMemoryOperationsHandlerWithAubDump() override = default; - MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded) override { - aubMemoryOperationsHandler->makeResident(device, gfxAllocations, isDummyExecNeeded); - return BaseOperationsHandler::makeResident(device, gfxAllocations, isDummyExecNeeded); + MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) override { + aubMemoryOperationsHandler->makeResident(device, gfxAllocations, isDummyExecNeeded, forcePagingFence); + return BaseOperationsHandler::makeResident(device, gfxAllocations, isDummyExecNeeded, forcePagingFence); } MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override { @@ -56,9 +56,9 @@ class WddmMemoryOperationsHandlerWithAubDump : public BaseOperationsHandler { return BaseOperationsHandler::isResident(device, gfxAllocation); } - MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) override { - aubMemoryOperationsHandler->makeResidentWithinOsContext(osContext, gfxAllocations, evictable); - return BaseOperationsHandler::makeResidentWithinOsContext(osContext, gfxAllocations, evictable); + MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence) override { + aubMemoryOperationsHandler->makeResidentWithinOsContext(osContext, gfxAllocations, evictable, forcePagingFence); + return BaseOperationsHandler::makeResidentWithinOsContext(osContext, gfxAllocations, evictable, forcePagingFence); } MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override { diff --git a/shared/source/os_interface/windows/wddm_residency_allocations_container.cpp b/shared/source/os_interface/windows/wddm_residency_allocations_container.cpp index 712d518882..42b3c46c31 100644 --- a/shared/source/os_interface/windows/wddm_residency_allocations_container.cpp +++ b/shared/source/os_interface/windows/wddm_residency_allocations_container.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 Intel Corporation + * Copyright (C) 2019-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -61,10 +61,10 @@ MemoryOperationsStatus WddmResidentAllocationsContainer::evictResources(const D3 } MemoryOperationsStatus WddmResidentAllocationsContainer::makeResidentResource(const D3DKMT_HANDLE &handle, size_t size) { - return makeResidentResources(&handle, 1u, size); + return makeResidentResources(&handle, 1u, size, false); } -MemoryOperationsStatus WddmResidentAllocationsContainer::makeResidentResources(const D3DKMT_HANDLE *handles, const uint32_t count, size_t size) { +MemoryOperationsStatus WddmResidentAllocationsContainer::makeResidentResources(const D3DKMT_HANDLE *handles, const uint32_t count, size_t size, const bool forcePagingFence) { while (!wddm->makeResident(handles, count, false, nullptr, size)) { if (evictAllResources() == MemoryOperationsStatus::success) { continue; @@ -80,7 +80,7 @@ MemoryOperationsStatus WddmResidentAllocationsContainer::makeResidentResources(c resourceHandles.push_back(handles[i]); } lock.unlock(); - wddm->waitOnPagingFenceFromCpu(false); + wddm->waitOnPagingFenceFromCpu(forcePagingFence); return MemoryOperationsStatus::success; } diff --git a/shared/source/os_interface/windows/wddm_residency_allocations_container.h b/shared/source/os_interface/windows/wddm_residency_allocations_container.h index 10082027e4..8fe0db35af 100644 --- a/shared/source/os_interface/windows/wddm_residency_allocations_container.h +++ b/shared/source/os_interface/windows/wddm_residency_allocations_container.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2022 Intel Corporation + * Copyright (C) 2019-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -26,7 +26,7 @@ class WddmResidentAllocationsContainer { MOCKABLE_VIRTUAL MemoryOperationsStatus evictResource(const D3DKMT_HANDLE &handle); MemoryOperationsStatus evictResources(const D3DKMT_HANDLE *handles, const uint32_t count); MOCKABLE_VIRTUAL MemoryOperationsStatus makeResidentResource(const D3DKMT_HANDLE &handle, size_t size); - MemoryOperationsStatus makeResidentResources(const D3DKMT_HANDLE *handles, const uint32_t count, size_t size); + MemoryOperationsStatus makeResidentResources(const D3DKMT_HANDLE *handles, const uint32_t count, size_t size, const bool forcePagingFence); MOCKABLE_VIRTUAL void removeResource(const D3DKMT_HANDLE &handle); protected: diff --git a/shared/test/common/mocks/linux/mock_drm_allocation.h b/shared/test/common/mocks/linux/mock_drm_allocation.h index caf61c2461..c0080cea0c 100644 --- a/shared/test/common/mocks/linux/mock_drm_allocation.h +++ b/shared/test/common/mocks/linux/mock_drm_allocation.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 Intel Corporation + * Copyright (C) 2019-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -88,9 +88,9 @@ class MockDrmAllocation : public DrmAllocation { DrmAllocation::markForCapture(); } - int bindBOs(OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind) override { + int bindBOs(OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind, const bool forcePagingFence) override { bindBOsCalled = true; - DrmAllocation::bindBOs(osContext, vmHandleId, bufferObjects, bind); + DrmAllocation::bindBOs(osContext, vmHandleId, bufferObjects, bind, forcePagingFence); return bindBOsRetValue; } @@ -101,7 +101,7 @@ class MockDrmAllocation : public DrmAllocation { return DrmAllocation::prefetchBO(bo, vmHandleId, subDeviceId); } - ADDMETHOD_NOBASE(makeBOsResident, int, 0, (OsContext * osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind)); + ADDMETHOD_NOBASE(makeBOsResident, int, 0, (OsContext * osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind, const bool forcePagingFence)); bool registerBOBindExtHandleCalled = false; bool markedForCapture = false; diff --git a/shared/test/common/mocks/mock_aub_memory_operations_handler.h b/shared/test/common/mocks/mock_aub_memory_operations_handler.h index ff7340bf50..8cbb9105d7 100644 --- a/shared/test/common/mocks/mock_aub_memory_operations_handler.h +++ b/shared/test/common/mocks/mock_aub_memory_operations_handler.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023-2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -21,9 +21,9 @@ struct MockAubMemoryOperationsHandler : public AubMemoryOperationsHandler { using AubMemoryOperationsHandler::getMemoryBanksBitfield; using AubMemoryOperationsHandler::residentAllocations; - MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded) override { + MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) override { makeResidentCalled = true; - return AubMemoryOperationsHandler::makeResident(device, gfxAllocations, isDummyExecNeeded); + return AubMemoryOperationsHandler::makeResident(device, gfxAllocations, isDummyExecNeeded, forcePagingFence); } MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override { @@ -36,9 +36,9 @@ struct MockAubMemoryOperationsHandler : public AubMemoryOperationsHandler { return AubMemoryOperationsHandler::isResident(device, gfxAllocation); } - MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) override { + MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence) override { makeResidentWithinOsContextCalled = true; - return AubMemoryOperationsHandler::makeResidentWithinOsContext(osContext, gfxAllocations, evictable); + return AubMemoryOperationsHandler::makeResidentWithinOsContext(osContext, gfxAllocations, evictable, forcePagingFence); } MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override { diff --git a/shared/test/common/mocks/mock_memory_operations_handler.h b/shared/test/common/mocks/mock_memory_operations_handler.h index ea736bb95f..b20e999584 100644 --- a/shared/test/common/mocks/mock_memory_operations_handler.h +++ b/shared/test/common/mocks/mock_memory_operations_handler.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 Intel Corporation + * Copyright (C) 2019-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -20,22 +20,22 @@ class GraphicsAllocation; class MockMemoryOperationsHandler : public MemoryOperationsHandler { public: MockMemoryOperationsHandler() {} - MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded) override { return MemoryOperationsStatus::unsupported; } + MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) override { return MemoryOperationsStatus::unsupported; } MemoryOperationsStatus lock(Device *device, ArrayRef gfxAllocations) override { return MemoryOperationsStatus::unsupported; } MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override { return MemoryOperationsStatus::unsupported; } MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override { return MemoryOperationsStatus::unsupported; } - MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) override { return MemoryOperationsStatus::unsupported; } + MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence) override { return MemoryOperationsStatus::unsupported; } MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override { return MemoryOperationsStatus::unsupported; } }; class MockMemoryOperationsHandlerTests : public MemoryOperationsHandler { public: MockMemoryOperationsHandlerTests() {} - ADDMETHOD_NOBASE(makeResident, MemoryOperationsStatus, MemoryOperationsStatus::unsupported, (Device * device, ArrayRef gfxAllocations, bool isDummyExecNeeded)); + ADDMETHOD_NOBASE(makeResident, MemoryOperationsStatus, MemoryOperationsStatus::unsupported, (Device * device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence)); ADDMETHOD_NOBASE(lock, MemoryOperationsStatus, MemoryOperationsStatus::unsupported, (Device * device, ArrayRef gfxAllocations)); ADDMETHOD_NOBASE(evict, MemoryOperationsStatus, MemoryOperationsStatus::unsupported, (Device * device, GraphicsAllocation &gfxAllocation)); ADDMETHOD_NOBASE(isResident, MemoryOperationsStatus, MemoryOperationsStatus::unsupported, (Device * device, GraphicsAllocation &gfxAllocation)); - ADDMETHOD_NOBASE(makeResidentWithinOsContext, MemoryOperationsStatus, MemoryOperationsStatus::unsupported, (OsContext * osContext, ArrayRef gfxAllocations, bool evictable)); + ADDMETHOD_NOBASE(makeResidentWithinOsContext, MemoryOperationsStatus, MemoryOperationsStatus::unsupported, (OsContext * osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence)); ADDMETHOD_NOBASE(evictWithinOsContext, MemoryOperationsStatus, MemoryOperationsStatus::unsupported, (OsContext * osContext, GraphicsAllocation &gfxAllocation)); }; @@ -43,7 +43,7 @@ class MockMemoryOperations : public MemoryOperationsHandler { public: MockMemoryOperations() {} - MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded) override { + MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) override { makeResidentCalledCount++; if (captureGfxAllocationsForMakeResident) { for (auto &gfxAllocation : gfxAllocations) { @@ -85,7 +85,7 @@ class MockMemoryOperations : public MemoryOperationsHandler { return MemoryOperationsStatus::success; } - MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) override { + MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence) override { makeResidentCalledCount++; if (osContext) { makeResidentContextId = osContext->getContextId(); diff --git a/shared/test/common/os_interface/linux/drm_command_stream_fixture.h b/shared/test/common/os_interface/linux/drm_command_stream_fixture.h index 6fdc9d13a7..2ae714a23c 100644 --- a/shared/test/common/os_interface/linux/drm_command_stream_fixture.h +++ b/shared/test/common/os_interface/linux/drm_command_stream_fixture.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 Intel Corporation + * Copyright (C) 2019-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -157,7 +157,7 @@ class DrmCommandStreamEnhancedTemplate : public ::testing::Test { template void makeResidentBufferObjects(OsContext *osContext, DrmAllocation *drmAllocation) { - drmAllocation->bindBOs(osContext, 0u, &static_cast *>(csr)->residency, false); + drmAllocation->bindBOs(osContext, 0u, &static_cast *>(csr)->residency, false, false); } template @@ -237,7 +237,7 @@ class DrmCommandStreamEnhancedWithFailingExecTemplate : public ::testing::Test { template void makeResidentBufferObjects(OsContext *osContext, DrmAllocation *drmAllocation) { - drmAllocation->bindBOs(osContext, 0u, &static_cast *>(csr)->residency, false); + drmAllocation->bindBOs(osContext, 0u, &static_cast *>(csr)->residency, false, false); } template diff --git a/shared/test/unit_test/command_stream/aub_command_stream_receiver_1_tests.cpp b/shared/test/unit_test/command_stream/aub_command_stream_receiver_1_tests.cpp index f35b02f3b7..4e06a2f824 100644 --- a/shared/test/unit_test/command_stream/aub_command_stream_receiver_1_tests.cpp +++ b/shared/test/unit_test/command_stream/aub_command_stream_receiver_1_tests.cpp @@ -363,7 +363,7 @@ HWTEST_F(AubCommandStreamReceiverTests, givenAubCsrAndResidentAllocationWhenProc MockGraphicsAllocation allocation2(reinterpret_cast(0x5000), 0x5000, 0x1000); GraphicsAllocation *allocPtr = &allocation2; - memoryOperationsHandler->makeResident(pDevice, ArrayRef(&allocPtr, 1), false); + memoryOperationsHandler->makeResident(pDevice, ArrayRef(&allocPtr, 1), false, false); EXPECT_TRUE(mockManager->writeMemory2Called); mockManager->storeAllocationParams = true; diff --git a/shared/test/unit_test/command_stream/tbx_command_stream_tests.cpp b/shared/test/unit_test/command_stream/tbx_command_stream_tests.cpp index fff56dab11..898416563b 100644 --- a/shared/test/unit_test/command_stream/tbx_command_stream_tests.cpp +++ b/shared/test/unit_test/command_stream/tbx_command_stream_tests.cpp @@ -379,7 +379,7 @@ HWTEST_F(TbxCommandSteamSimpleTest, givenTbxCsrAndResidentAllocationWhenProcessR MockGraphicsAllocation allocation2(reinterpret_cast(0x5000), 0x5000, 0x1000); GraphicsAllocation *allocPtr = &allocation2; - memoryOperationsHandler->makeResident(pDevice, ArrayRef(&allocPtr, 1), false); + memoryOperationsHandler->makeResident(pDevice, ArrayRef(&allocPtr, 1), false, false); EXPECT_TRUE(mockManager->writeMemory2Called); mockManager->storeAllocationParams = true; diff --git a/shared/test/unit_test/memory_manager/memory_manager_tests.cpp b/shared/test/unit_test/memory_manager/memory_manager_tests.cpp index efab076d3f..f659e417ab 100644 --- a/shared/test/unit_test/memory_manager/memory_manager_tests.cpp +++ b/shared/test/unit_test/memory_manager/memory_manager_tests.cpp @@ -3313,7 +3313,7 @@ TEST(MemoryManagerTest, givenMemoryAllocationWhenFreedThenFreeCalledOnMemoryOper auto memoryAllocation = memoryManager.allocateGraphicsMemoryWithProperties(allocationProperties); EXPECT_NE(nullptr, memoryAllocation); - memoryOperationsHandler->makeResident(device.get(), ArrayRef(&memoryAllocation, 1), false); + memoryOperationsHandler->makeResident(device.get(), ArrayRef(&memoryAllocation, 1), false, false); EXPECT_EQ(1u, memoryOperationsHandler->residentAllocations.size()); diff --git a/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.cpp b/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.cpp index c7bacedb88..dd28b98339 100644 --- a/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.cpp +++ b/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 Intel Corporation + * Copyright (C) 2019-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenNullPtrAsAubManagerWhenMakeResidentCalledThenFalseReturned) { getMemoryOperationsHandler()->setAubManager(nullptr); auto memoryOperationsInterface = getMemoryOperationsHandler(); - auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::deviceUninitialized); } @@ -31,7 +31,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenAubManagerWhenMakeResidentCalledThe MockAubManager aubManager; getMemoryOperationsHandler()->setAubManager(&aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); - auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_TRUE(aubManager.writeMemory2Called); @@ -41,7 +41,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenAubManagerWhenMakeResidentCalledThe aubManager.writeMemory2Called = false; - result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_TRUE(aubManager.writeMemory2Called); @@ -80,7 +80,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenAubManagerAndAllocationOfOneTimeAub MockAubManager aubManager; getMemoryOperationsHandler()->setAubManager(&aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); - auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_TRUE(aubManager.writeMemory2Called); @@ -90,7 +90,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenAubManagerAndAllocationOfOneTimeAub aubManager.writeMemory2Called = false; - result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_FALSE(aubManager.writeMemory2Called); @@ -106,7 +106,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenAubManagerWhenMakeResidentCalledOnW allocPtr->setWriteMemoryOnly(true); - auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_TRUE(aubManager.writeMemory2Called); @@ -115,7 +115,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenAubManagerWhenMakeResidentCalledOnW EXPECT_EQ(0u, memoryOperationsInterface->residentAllocations.size()); aubManager.writeMemory2Called = false; - result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_TRUE(aubManager.writeMemory2Called); @@ -140,7 +140,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenAubManagerAndAllocationInLocalMemor MockAubManager aubManager; getMemoryOperationsHandler()->setAubManager(&aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); - auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_FALSE(aubManager.writeMemory2Called); EXPECT_EQ(1u, csr->writeMemoryAubCalled); @@ -163,7 +163,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenAubManagerWhenMakeResidentCalledThe MockAubManager aubManager; getMemoryOperationsHandler()->setAubManager(&aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); - auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_EQ(1u, csr->initializeEngineCalled); @@ -185,7 +185,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenAubManagerWhenMakeResidentCalledWit MockAubManager aubManager; getMemoryOperationsHandler()->setAubManager(&aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); - auto result = memoryOperationsInterface->makeResident(nullptr, ArrayRef(&allocPtr, 1), false); + auto result = memoryOperationsInterface->makeResident(nullptr, ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_EQ(0u, csr->initializeEngineCalled); @@ -204,7 +204,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenAubManagerWhenMakeResidentCalledOnC gmm.setCompressionEnabled(true); allocPtr->setDefaultGmm(&gmm); - auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_TRUE(aubManager.writeMemory2Called); @@ -228,7 +228,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenAubManagerWhenMakeResidentCalledOnU gmm.setCompressionEnabled(false); allocPtr->setDefaultGmm(&gmm); - auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_TRUE(aubManager.writeMemory2Called); @@ -241,7 +241,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenAllocationWhenMakeResidentCalledThe MockAubManager aubManager; getMemoryOperationsHandler()->setAubManager(&aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); - memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(aubManager.hintToWriteMemory, AubMemDump::DataTypeHintValues::TraceNotype); } TEST_F(AubMemoryOperationsHandlerTests, givenNonResidentAllocationWhenIsResidentCalledThenFalseReturned) { @@ -255,7 +255,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenResidentAllocationWhenIsResidentCal MockAubManager aubManager; getMemoryOperationsHandler()->setAubManager(&aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); - memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); auto result = memoryOperationsInterface->isResident(nullptr, allocation); EXPECT_EQ(result, MemoryOperationsStatus::success); } @@ -270,7 +270,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenResidentAllocationWhenEvictCalledTh MockAubManager aubManager; getMemoryOperationsHandler()->setAubManager(&aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); - memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); auto result = memoryOperationsInterface->evict(nullptr, allocation); EXPECT_EQ(result, MemoryOperationsStatus::success); } @@ -320,7 +320,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenNonLocalMemoryAllocationWithStorage auto memoryOperationsInterface = getMemoryOperationsHandler(); allocPtr = &allocation; - memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_TRUE(aubManager.writeMemory2Called); EXPECT_EQ(0u, aubManager.storedAllocationParams[0].memoryBanks); @@ -510,7 +510,7 @@ HWTEST_F(AubMemoryOperationsHandlerTests, givenCloningOfPageTablesWhenProcessing allocPtr->storageInfo.cloningOfPageTables = true; - auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_TRUE(aubManager->writeMemory2Called); @@ -540,7 +540,7 @@ HWTEST_F(AubMemoryOperationsHandlerTests, givenNonLocalGfxAllocationWriteableWhe allocPtr = &allocation; allocPtr->storageInfo.cloningOfPageTables = false; - auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_TRUE(aubManager->writeMemory2Called); @@ -575,7 +575,7 @@ HWTEST_F(AubMemoryOperationsHandlerTests, givenLocalGfxAllocationWriteableWhenPr auto backupDefaultCsr = std::make_unique>(&device->getDefaultEngine().commandStreamReceiver); device->getDefaultEngine().commandStreamReceiver = csr.get(); - auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_FALSE(aubManager->writeMemory2Called); @@ -611,7 +611,7 @@ HWTEST_F(AubMemoryOperationsHandlerTests, givenGfxAllocationWriteableWithGmmPres executionEnvironment->rootDeviceEnvironments[0]->aubCenter.reset(mockAubCenter.release()); - auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); aubManager->writeMemory2Called = false; @@ -644,7 +644,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenDeviceNullWhenProcessingFlushReside memoryOperationsInterface->setAubWritable(true, allocation, device.get()); EXPECT_TRUE(memoryOperationsInterface->isAubWritable(allocation, device.get())); - auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false); + auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(result, MemoryOperationsStatus::success); EXPECT_TRUE(aubManager.writeMemory2Called); diff --git a/shared/test/unit_test/os_interface/linux/drm_bind_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_bind_tests.cpp index 92d1424d5a..faa714b78d 100644 --- a/shared/test/unit_test/os_interface/linux/drm_bind_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_bind_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022-2024 Intel Corporation + * Copyright (C) 2022-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -55,6 +55,45 @@ TEST(DrmBindTest, givenBindNotCompleteWhenWaitForBindThenWaitUserFenceIoctlIsCal EXPECT_EQ(-1, drm.waitUserFenceParams[0].timeout); } +TEST(DrmBindTest, givenBindAlreadyCompleteWhenwaitForBindGivenFenceValThenWaitUserFenceIoctlIsNotCalled) { + auto executionEnvironment = std::make_unique(); + DrmMock drm{*executionEnvironment->rootDeviceEnvironments[0]}; + OsContextLinux osContext(drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); + osContext.ensureContextInitialized(false); + + drm.pagingFence[0] = 31u; + uint64_t fenceValToWait = 31u; + + drm.waitForBindGivenFenceVal(0u, fenceValToWait); + + EXPECT_EQ(0u, drm.waitUserFenceParams.size()); + + drm.pagingFence[0] = 49u; + + drm.waitForBindGivenFenceVal(0u, fenceValToWait); + + EXPECT_EQ(0u, drm.waitUserFenceParams.size()); +} + +TEST(DrmBindTest, givenBindNotCompleteWhenwaitForBindGivenFenceValThenWaitUserFenceIoctlIsCalled) { + auto executionEnvironment = std::make_unique(); + DrmMock drm{*executionEnvironment->rootDeviceEnvironments[0]}; + OsContextLinux osContext(drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); + osContext.ensureContextInitialized(false); + + drm.pagingFence[0] = 26u; + uint64_t fenceValToWait = 31u; + + drm.waitForBindGivenFenceVal(0u, fenceValToWait); + + EXPECT_EQ(1u, drm.waitUserFenceParams.size()); + EXPECT_EQ(0u, drm.waitUserFenceParams[0].ctxId); + EXPECT_EQ(castToUint64(&drm.pagingFence[0]), drm.waitUserFenceParams[0].address); + EXPECT_EQ(drm.ioctlHelper->getWaitUserFenceSoftFlag(), drm.waitUserFenceParams[0].flags); + EXPECT_EQ(fenceValToWait, drm.waitUserFenceParams[0].value); + EXPECT_EQ(-1, drm.waitUserFenceParams[0].timeout); +} + TEST(DrmBindTest, whenCheckingVmBindAvailabilityThenIoctlHelperSupportIsUsed) { auto executionEnvironment = std::make_unique(); DrmMock drm{*executionEnvironment->rootDeviceEnvironments[0]}; diff --git a/shared/test/unit_test/os_interface/linux/drm_buffer_object_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_buffer_object_tests.cpp index 08b161ee44..4e50d1f3a6 100644 --- a/shared/test/unit_test/os_interface/linux/drm_buffer_object_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_buffer_object_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2024 Intel Corporation + * Copyright (C) 2018-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -450,7 +450,7 @@ TEST(DrmBufferObject, givenPerContextVmRequiredWhenBoBoundAndUnboundThenCorrectB auto osContext = engines[contextId].osContext; osContext->ensureContextInitialized(false); - bo.bind(osContext, 0); + bo.bind(osContext, 0, false); EXPECT_TRUE(bo.bindInfo[contextId][0]); bo.unbind(osContext, 0); @@ -461,7 +461,7 @@ TEST(DrmBufferObject, givenPrintBOBindingResultWhenBOBindAndUnbindSucceedsThenPr struct DrmMockToSucceedBindBufferObject : public DrmMock { DrmMockToSucceedBindBufferObject(RootDeviceEnvironment &rootDeviceEnvironment) : DrmMock(rootDeviceEnvironment) {} - int bindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo) override { return 0; } + int bindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo, const bool forcePagingFence) override { return 0; } int unbindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo) override { return 0; } }; @@ -496,7 +496,7 @@ TEST(DrmBufferObject, givenPrintBOBindingResultWhenBOBindAndUnbindSucceedsThenPr testing::internal::CaptureStdout(); - bo.bind(osContext, 0); + bo.bind(osContext, 0, false); EXPECT_TRUE(bo.bindInfo[contextId][0]); std::string bindOutput = testing::internal::GetCapturedStdout(); @@ -518,7 +518,7 @@ TEST(DrmBufferObject, givenPrintBOBindingResultWhenBOBindAndUnbindFailsThenPrint struct DrmMockToFailBindBufferObject : public DrmMock { DrmMockToFailBindBufferObject(RootDeviceEnvironment &rootDeviceEnvironment) : DrmMock(rootDeviceEnvironment) {} - int bindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo) override { return -1; } + int bindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo, const bool forcePagingFence) override { return -1; } int unbindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo) override { return -1; } int getErrno() override { return EINVAL; } }; @@ -554,7 +554,7 @@ TEST(DrmBufferObject, givenPrintBOBindingResultWhenBOBindAndUnbindFailsThenPrint testing::internal::CaptureStderr(); - bo.bind(osContext, 0); + bo.bind(osContext, 0, false); EXPECT_FALSE(bo.bindInfo[contextId][0]); std::string bindOutput = testing::internal::GetCapturedStderr(); @@ -600,7 +600,7 @@ TEST(DrmBufferObject, givenDrmWhenBindOperationFailsThenFenceValueNotGrow) { auto contextId = osContextCount / 2; auto osContext = engines[contextId].osContext; MockBufferObject bo(device->getRootDeviceIndex(), drm, 3, 0, 0, osContextCount); - drm->bindBufferObject(osContext, 0, &bo); + drm->bindBufferObject(osContext, 0, &bo, false); EXPECT_EQ(drm->fenceVal[0], initFenceValue); } @@ -631,11 +631,299 @@ TEST(DrmBufferObject, givenDrmWhenBindOperationSucceedsThenFenceValueGrow) { auto contextId = osContextCount / 2; auto osContext = engines[contextId].osContext; MockBufferObject bo(device->getRootDeviceIndex(), drm, 3, 0, 0, osContextCount); - drm->bindBufferObject(osContext, 0, &bo); + drm->bindBufferObject(osContext, 0, &bo, false); EXPECT_EQ(drm->fenceVal[0], initFenceValue + 1); } +TEST(DrmBufferObject, givenDrmWhenBindOperationSucceedsWithForcePagingFenceThenFenceValueGrow) { + + DebugManagerStateRestore restorer; + debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.set(1); + + class MockOsContextLinuxUnbind : public OsContextLinux { + public: + using OsContextLinux::drmContextIds; + using OsContextLinux::fenceVal; + using OsContextLinux::pagingFence; + + MockOsContextLinuxUnbind(Drm &drm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor) + : OsContextLinux(drm, rootDeviceIndex, contextId, engineDescriptor) {} + + void waitForPagingFenceGivenFenceVal(uint64_t fenceValToWait) override { + waitForPagingFenceGivenFenceValCalled = true; + } + + bool waitForPagingFenceGivenFenceValCalled = false; + }; + + auto executionEnvironment = new ExecutionEnvironment; + executionEnvironment->setDebuggingMode(NEO::DebuggingMode::online); + executionEnvironment->prepareRootDeviceEnvironments(1); + executionEnvironment->rootDeviceEnvironments[0]->setHwInfoAndInitHelpers(defaultHwInfo.get()); + executionEnvironment->rootDeviceEnvironments[0]->initGmm(); + executionEnvironment->rootDeviceEnvironments[0]->osInterface = std::make_unique(); + + auto drm = new DrmMock(*executionEnvironment->rootDeviceEnvironments[0]); + drm->requirePerContextVM = false; + drm->isVMBindImmediateSupported = true; + auto ioctlHelper = std::make_unique(*drm); + ioctlHelper->isWaitBeforeBindRequiredResult = true; + drm->ioctlHelper.reset(ioctlHelper.release()); + + auto osContext = new MockOsContextLinuxUnbind(*drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); + osContext->ensureContextInitialized(false); + + executionEnvironment->rootDeviceEnvironments[0]->osInterface->setDriverModel(std::unique_ptr(drm)); + executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(*drm, 0u, false); + // Make sure to disable the debugging mode + executionEnvironment->setDebuggingMode(DebuggingMode::disabled); + uint64_t initFenceValue = 10u; + drm->fenceVal[0] = initFenceValue; + std::unique_ptr device(MockDevice::createWithExecutionEnvironment(defaultHwInfo.get(), executionEnvironment, 0)); + + auto &engines = device->getExecutionEnvironment()->memoryManager->getRegisteredEngines(device->getRootDeviceIndex()); + auto osContextCount = engines.size(); + + MockBufferObject bo(device->getRootDeviceIndex(), drm, 3, 0, 0, osContextCount); + drm->bindBufferObject(osContext, 0, &bo, true); + + EXPECT_EQ(drm->fenceVal[0], initFenceValue + 1); + EXPECT_TRUE(osContext->waitForPagingFenceGivenFenceValCalled); + delete osContext; +} + +TEST(DrmBufferObject, givenDrmWhenBindOperationSucceedsWithForcePagingFenceNotWaitOnUserFenceAfterBindAndUnbindThenFenceValueGrow) { + + DebugManagerStateRestore restorer; + debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.set(0); + + class MockOsContextLinuxUnbind : public OsContextLinux { + public: + using OsContextLinux::drmContextIds; + using OsContextLinux::fenceVal; + using OsContextLinux::pagingFence; + + MockOsContextLinuxUnbind(Drm &drm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor) + : OsContextLinux(drm, rootDeviceIndex, contextId, engineDescriptor) {} + + void waitForPagingFenceGivenFenceVal(uint64_t fenceValToWait) override { + waitForPagingFenceGivenFenceValCalled = true; + } + + bool waitForPagingFenceGivenFenceValCalled = false; + }; + + auto executionEnvironment = new ExecutionEnvironment; + executionEnvironment->setDebuggingMode(NEO::DebuggingMode::online); + executionEnvironment->prepareRootDeviceEnvironments(1); + executionEnvironment->rootDeviceEnvironments[0]->setHwInfoAndInitHelpers(defaultHwInfo.get()); + executionEnvironment->rootDeviceEnvironments[0]->initGmm(); + executionEnvironment->rootDeviceEnvironments[0]->osInterface = std::make_unique(); + + auto drm = new DrmMock(*executionEnvironment->rootDeviceEnvironments[0]); + drm->requirePerContextVM = false; + drm->isVMBindImmediateSupported = true; + auto ioctlHelper = std::make_unique(*drm); + ioctlHelper->isWaitBeforeBindRequiredResult = true; + drm->ioctlHelper.reset(ioctlHelper.release()); + + auto osContext = new MockOsContextLinuxUnbind(*drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); + osContext->ensureContextInitialized(false); + + executionEnvironment->rootDeviceEnvironments[0]->osInterface->setDriverModel(std::unique_ptr(drm)); + executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(*drm, 0u, false); + // Make sure to disable the debugging mode + executionEnvironment->setDebuggingMode(DebuggingMode::disabled); + uint64_t initFenceValue = 10u; + drm->fenceVal[0] = initFenceValue; + std::unique_ptr device(MockDevice::createWithExecutionEnvironment(defaultHwInfo.get(), executionEnvironment, 0)); + + auto &engines = device->getExecutionEnvironment()->memoryManager->getRegisteredEngines(device->getRootDeviceIndex()); + auto osContextCount = engines.size(); + + MockBufferObject bo(device->getRootDeviceIndex(), drm, 3, 0, 0, osContextCount); + drm->bindBufferObject(osContext, 0, &bo, true); + + EXPECT_EQ(drm->fenceVal[0], initFenceValue + 1); + EXPECT_TRUE(osContext->waitForPagingFenceGivenFenceValCalled); + delete osContext; +} + +TEST(DrmBufferObject, givenDrmWhenBindOperationSucceedsWithForcePagingFenceWithWaitOnUserFenceAfterBindAndUnbindAndNotUseVMBindImmediateThenFenceValueGrow) { + + DebugManagerStateRestore restorer; + debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.set(1); + + class MockOsContextLinuxUnbind : public OsContextLinux { + public: + using OsContextLinux::drmContextIds; + using OsContextLinux::fenceVal; + using OsContextLinux::pagingFence; + + MockOsContextLinuxUnbind(Drm &drm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor) + : OsContextLinux(drm, rootDeviceIndex, contextId, engineDescriptor) {} + + void waitForPagingFenceGivenFenceVal(uint64_t fenceValToWait) override { + waitForPagingFenceGivenFenceValCalled = true; + } + + bool waitForPagingFenceGivenFenceValCalled = false; + }; + + auto executionEnvironment = new ExecutionEnvironment; + executionEnvironment->setDebuggingMode(NEO::DebuggingMode::online); + executionEnvironment->prepareRootDeviceEnvironments(1); + executionEnvironment->rootDeviceEnvironments[0]->setHwInfoAndInitHelpers(defaultHwInfo.get()); + executionEnvironment->rootDeviceEnvironments[0]->initGmm(); + executionEnvironment->rootDeviceEnvironments[0]->osInterface = std::make_unique(); + + auto drm = new DrmMock(*executionEnvironment->rootDeviceEnvironments[0]); + drm->requirePerContextVM = false; + // Making the useVMBindImmediate() false + drm->isVMBindImmediateSupported = false; + auto ioctlHelper = std::make_unique(*drm); + ioctlHelper->isWaitBeforeBindRequiredResult = true; + drm->ioctlHelper.reset(ioctlHelper.release()); + + auto osContext = new MockOsContextLinuxUnbind(*drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); + osContext->ensureContextInitialized(false); + + executionEnvironment->rootDeviceEnvironments[0]->osInterface->setDriverModel(std::unique_ptr(drm)); + executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(*drm, 0u, false); + // Make sure to disable the debugging mode + executionEnvironment->setDebuggingMode(DebuggingMode::disabled); + uint64_t initFenceValue = 10u; + drm->fenceVal[0] = initFenceValue; + std::unique_ptr device(MockDevice::createWithExecutionEnvironment(defaultHwInfo.get(), executionEnvironment, 0)); + + auto &engines = device->getExecutionEnvironment()->memoryManager->getRegisteredEngines(device->getRootDeviceIndex()); + auto osContextCount = engines.size(); + + MockBufferObject bo(device->getRootDeviceIndex(), drm, 3, 0, 0, osContextCount); + drm->bindBufferObject(osContext, 0, &bo, true); + + EXPECT_EQ(drm->fenceVal[0], initFenceValue + 1); + EXPECT_TRUE(osContext->waitForPagingFenceGivenFenceValCalled); + delete osContext; +} + +TEST(DrmBufferObject, givenDrmWhenBindOperationSucceedsWithForcePagingFenceWithDebuggingEnabledWithWaitOnUserFenceAfterBindAndUnbindAndNotUseVMBindImmediateThenFenceValueDoesNotGrow) { + + DebugManagerStateRestore restorer; + debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.set(1); + + class MockOsContextLinuxUnbind : public OsContextLinux { + public: + using OsContextLinux::drmContextIds; + using OsContextLinux::fenceVal; + using OsContextLinux::pagingFence; + + MockOsContextLinuxUnbind(Drm &drm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor) + : OsContextLinux(drm, rootDeviceIndex, contextId, engineDescriptor) {} + + void waitForPagingFenceGivenFenceVal(uint64_t fenceValToWait) override { + waitForPagingFenceGivenFenceValCalled = true; + } + + bool waitForPagingFenceGivenFenceValCalled = false; + }; + + auto executionEnvironment = new ExecutionEnvironment; + executionEnvironment->setDebuggingMode(NEO::DebuggingMode::online); + executionEnvironment->prepareRootDeviceEnvironments(1); + executionEnvironment->rootDeviceEnvironments[0]->setHwInfoAndInitHelpers(defaultHwInfo.get()); + executionEnvironment->rootDeviceEnvironments[0]->initGmm(); + executionEnvironment->rootDeviceEnvironments[0]->osInterface = std::make_unique(); + + auto drm = new DrmMock(*executionEnvironment->rootDeviceEnvironments[0]); + drm->requirePerContextVM = false; + // Making the useVMBindImmediate() false + drm->isVMBindImmediateSupported = false; + auto ioctlHelper = std::make_unique(*drm); + ioctlHelper->isWaitBeforeBindRequiredResult = true; + drm->ioctlHelper.reset(ioctlHelper.release()); + + auto osContext = new MockOsContextLinuxUnbind(*drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); + osContext->ensureContextInitialized(false); + + executionEnvironment->rootDeviceEnvironments[0]->osInterface->setDriverModel(std::unique_ptr(drm)); + executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(*drm, 0u, false); + // Make sure to enable the debugging mode + executionEnvironment->setDebuggingMode(DebuggingMode::online); + uint64_t initFenceValue = 10u; + drm->fenceVal[0] = initFenceValue; + std::unique_ptr device(MockDevice::createWithExecutionEnvironment(defaultHwInfo.get(), executionEnvironment, 0)); + + auto &engines = device->getExecutionEnvironment()->memoryManager->getRegisteredEngines(device->getRootDeviceIndex()); + auto osContextCount = engines.size(); + + MockBufferObject bo(device->getRootDeviceIndex(), drm, 3, 0, 0, osContextCount); + drm->bindBufferObject(osContext, 0, &bo, true); + + EXPECT_EQ(drm->fenceVal[0], initFenceValue); + EXPECT_FALSE(osContext->waitForPagingFenceGivenFenceValCalled); + delete osContext; +} + +TEST(DrmBufferObject, givenDrmWhenBindOperationSucceedsWithForcePagingFenceNotWaitOnUserFenceAfterBindAndUnbindAndNotUseVMBindImmediateThenFenceValueGrow) { + + DebugManagerStateRestore restorer; + debugManager.flags.EnableWaitOnUserFenceAfterBindAndUnbind.set(0); + + class MockOsContextLinuxUnbind : public OsContextLinux { + public: + using OsContextLinux::drmContextIds; + using OsContextLinux::fenceVal; + using OsContextLinux::pagingFence; + + MockOsContextLinuxUnbind(Drm &drm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor) + : OsContextLinux(drm, rootDeviceIndex, contextId, engineDescriptor) {} + + void waitForPagingFenceGivenFenceVal(uint64_t fenceValToWait) override { + waitForPagingFenceGivenFenceValCalled = true; + } + + bool waitForPagingFenceGivenFenceValCalled = false; + }; + + auto executionEnvironment = new ExecutionEnvironment; + executionEnvironment->setDebuggingMode(NEO::DebuggingMode::online); + executionEnvironment->prepareRootDeviceEnvironments(1); + executionEnvironment->rootDeviceEnvironments[0]->setHwInfoAndInitHelpers(defaultHwInfo.get()); + executionEnvironment->rootDeviceEnvironments[0]->initGmm(); + executionEnvironment->rootDeviceEnvironments[0]->osInterface = std::make_unique(); + + auto drm = new DrmMock(*executionEnvironment->rootDeviceEnvironments[0]); + drm->requirePerContextVM = false; + // Making the useVMBindImmediate() false + drm->isVMBindImmediateSupported = false; + auto ioctlHelper = std::make_unique(*drm); + ioctlHelper->isWaitBeforeBindRequiredResult = true; + drm->ioctlHelper.reset(ioctlHelper.release()); + + auto osContext = new MockOsContextLinuxUnbind(*drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); + osContext->ensureContextInitialized(false); + + executionEnvironment->rootDeviceEnvironments[0]->osInterface->setDriverModel(std::unique_ptr(drm)); + executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(*drm, 0u, false); + // Make sure to disable the debugging mode + executionEnvironment->setDebuggingMode(DebuggingMode::disabled); + uint64_t initFenceValue = 10u; + drm->fenceVal[0] = initFenceValue; + std::unique_ptr device(MockDevice::createWithExecutionEnvironment(defaultHwInfo.get(), executionEnvironment, 0)); + + auto &engines = device->getExecutionEnvironment()->memoryManager->getRegisteredEngines(device->getRootDeviceIndex()); + auto osContextCount = engines.size(); + + MockBufferObject bo(device->getRootDeviceIndex(), drm, 3, 0, 0, osContextCount); + drm->bindBufferObject(osContext, 0, &bo, true); + + EXPECT_EQ(drm->fenceVal[0], initFenceValue + 1); + EXPECT_TRUE(osContext->waitForPagingFenceGivenFenceValCalled); + delete osContext; +} + TEST(DrmBufferObject, givenDrmWhenUnBindOperationFailsThenFenceValueNotGrow) { auto executionEnvironment = new ExecutionEnvironment; executionEnvironment->setDebuggingMode(NEO::DebuggingMode::online); @@ -748,11 +1036,11 @@ TEST(DrmBufferObject, givenDrmWhenUnBindOperationSucceedsAndForceFenceWaitThenFe MockOsContextLinuxUnbind(Drm &drm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor) : OsContextLinux(drm, rootDeviceIndex, contextId, engineDescriptor) {} - void waitForPagingFence() override { - waitForPagingFenceCalled = true; + void waitForPagingFenceGivenFenceVal(uint64_t fenceValToWait) override { + waitForPagingFenceGivenFenceValCalled = true; } - bool waitForPagingFenceCalled = false; + bool waitForPagingFenceGivenFenceValCalled = false; }; auto executionEnvironment = new ExecutionEnvironment; @@ -785,7 +1073,7 @@ TEST(DrmBufferObject, givenDrmWhenUnBindOperationSucceedsAndForceFenceWaitThenFe drm->unbindBufferObject(static_cast(osContext), 0, &bo); EXPECT_EQ(drm->fenceVal[0], initFenceValue + 1); - EXPECT_TRUE(osContext->waitForPagingFenceCalled); + EXPECT_TRUE(osContext->waitForPagingFenceGivenFenceValCalled); delete osContext; } @@ -803,11 +1091,11 @@ TEST(DrmBufferObject, givenDrmWhenUnBindOperationSucceedsWaitBeforeBindFalseAndF MockOsContextLinuxUnbind(Drm &drm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor) : OsContextLinux(drm, rootDeviceIndex, contextId, engineDescriptor) {} - void waitForPagingFence() override { - waitForPagingFenceCalled = true; + void waitForPagingFenceGivenFenceVal(uint64_t fenceValToWait) override { + waitForPagingFenceGivenFenceValCalled = true; } - bool waitForPagingFenceCalled = false; + bool waitForPagingFenceGivenFenceValCalled = false; }; auto executionEnvironment = new ExecutionEnvironment; @@ -840,7 +1128,7 @@ TEST(DrmBufferObject, givenDrmWhenUnBindOperationSucceedsWaitBeforeBindFalseAndF drm->unbindBufferObject(static_cast(osContext), 0, &bo); EXPECT_EQ(drm->fenceVal[0], initFenceValue); - EXPECT_FALSE(osContext->waitForPagingFenceCalled); + EXPECT_FALSE(osContext->waitForPagingFenceGivenFenceValCalled); delete osContext; } @@ -858,11 +1146,11 @@ TEST(DrmBufferObject, givenDrmWhenUnBindOperationSucceedsWaitBeforeBindTrueAndFo MockOsContextLinuxUnbind(Drm &drm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor) : OsContextLinux(drm, rootDeviceIndex, contextId, engineDescriptor) {} - void waitForPagingFence() override { - waitForPagingFenceCalled = true; + void waitForPagingFenceGivenFenceVal(uint64_t fenceValToWait) override { + waitForPagingFenceGivenFenceValCalled = true; } - bool waitForPagingFenceCalled = false; + bool waitForPagingFenceGivenFenceValCalled = false; }; auto executionEnvironment = new ExecutionEnvironment; @@ -895,7 +1183,7 @@ TEST(DrmBufferObject, givenDrmWhenUnBindOperationSucceedsWaitBeforeBindTrueAndFo drm->unbindBufferObject(static_cast(osContext), 0, &bo); EXPECT_EQ(drm->fenceVal[0], initFenceValue); - EXPECT_FALSE(osContext->waitForPagingFenceCalled); + EXPECT_FALSE(osContext->waitForPagingFenceGivenFenceValCalled); delete osContext; } diff --git a/shared/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp index 0ef7667bcb..78e745d44e 100644 --- a/shared/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022-2024 Intel Corporation + * Copyright (C) 2022-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -835,7 +835,7 @@ struct MockDrmAllocationBindBO : public DrmAllocation { } ADDMETHOD_NOBASE(bindBO, int, 0, - (BufferObject * bo, OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind)); + (BufferObject * bo, OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind, const bool forcePagingFence)); }; struct MockDrmAllocationBindBOs : public DrmAllocation { @@ -844,7 +844,7 @@ struct MockDrmAllocationBindBOs : public DrmAllocation { } ADDMETHOD_NOBASE(bindBOs, int, 0, - (OsContext * osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind)); + (OsContext * osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind, const bool forcePagingFence)); }; HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenBindBOsFailsThenMakeBOsResidentReturnsError) { @@ -855,7 +855,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenBindBOsFailsThenMakeBOsRes auto allocation = new MockDrmAllocationBindBOs(0, AllocationType::unknown, bos, nullptr, 0u, size, MemoryPool::localMemory); allocation->bindBOsResult = -1; - auto res = allocation->makeBOsResident(&csr->getOsContext(), 0, nullptr, true); + auto res = allocation->makeBOsResident(&csr->getOsContext(), 0, nullptr, true, false); EXPECT_NE(res, 0); EXPECT_EQ(allocation->fragmentsStorage.fragmentCount, 0u); EXPECT_GT(allocation->bindBOsCalled, 0u); @@ -884,7 +884,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenFragmentStorageAndBindBOFa memcpy(&prevStorage, &allocation->fragmentsStorage, sizeof(OsHandleStorage)); memcpy(&allocation->fragmentsStorage, &storage, sizeof(OsHandleStorage)); - auto res = allocation->makeBOsResident(&csr->getOsContext(), 0, nullptr, true); + auto res = allocation->makeBOsResident(&csr->getOsContext(), 0, nullptr, true, false); EXPECT_NE(res, 0); EXPECT_EQ(allocation->fragmentsStorage.fragmentCount, 1u); EXPECT_GT(allocation->bindBOCalled, 0u); @@ -902,7 +902,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenBindBOFailsThenBindBOsRetu auto allocation = new MockDrmAllocationBindBO(0, AllocationType::unknown, bos, nullptr, 0u, size, MemoryPool::localMemory); allocation->bindBOResult = -1; - auto res = allocation->bindBOs(&csr->getOsContext(), 0u, &static_cast *>(csr)->residency, false); + auto res = allocation->bindBOs(&csr->getOsContext(), 0u, &static_cast *>(csr)->residency, false, false); EXPECT_NE(res, 0); EXPECT_GT(allocation->bindBOCalled, 0u); @@ -920,7 +920,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenBindBOFailsWithMultipleMem allocation->storageInfo.memoryBanks = 0b11; EXPECT_EQ(allocation->storageInfo.getNumBanks(), 2u); - auto res = allocation->bindBOs(&csr->getOsContext(), 0u, &static_cast *>(csr)->residency, false); + auto res = allocation->bindBOs(&csr->getOsContext(), 0u, &static_cast *>(csr)->residency, false, false); EXPECT_NE(res, 0); EXPECT_GT(allocation->bindBOCalled, 0u); @@ -939,7 +939,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenBindBOFailsWithMultipleMem allocation->storageInfo.memoryBanks = 0b11; EXPECT_EQ(allocation->storageInfo.getNumBanks(), 2u); - auto res = allocation->bindBOs(&csr->getOsContext(), 0u, &static_cast *>(csr)->residency, false); + auto res = allocation->bindBOs(&csr->getOsContext(), 0u, &static_cast *>(csr)->residency, false, false); EXPECT_NE(res, 0); EXPECT_GT(allocation->bindBOCalled, 0u); @@ -1417,7 +1417,7 @@ struct MockMergeResidencyContainerMemoryOperationsHandler : public DrmMemoryOper (OsContext * osContext, ResidencyContainer &residencyContainer)); ADDMETHOD_NOBASE(makeResidentWithinOsContext, NEO::MemoryOperationsStatus, NEO::MemoryOperationsStatus::success, - (OsContext * osContext, ArrayRef gfxAllocations, bool evictable)); + (OsContext * osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence)); }; HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenMergeWithResidencyContainerFailsThenFlushReturnsError) { @@ -1534,7 +1534,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenAllocsInMemoryOperationHan PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo))); auto allocation = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); - executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocation, 1), false); + executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocation, 1), false, false); auto commandBuffer = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); LinearStream cs(commandBuffer); @@ -1558,7 +1558,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenAllocInMemoryOperationsInt BatchBuffer batchBuffer = BatchBufferHelper::createDefaultBatchBuffer(cs.getGraphicsAllocation(), &cs, cs.getUsed()); auto allocation = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); - executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocation, 1), false); + executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocation, 1), false, false); csr->flush(batchBuffer, csr->getResidencyAllocations()); diff --git a/shared/test/unit_test/os_interface/linux/drm_command_stream_tests_1.cpp b/shared/test/unit_test/os_interface/linux/drm_command_stream_tests_1.cpp index b98aa41aa7..1cf25921db 100644 --- a/shared/test/unit_test/os_interface/linux/drm_command_stream_tests_1.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_command_stream_tests_1.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2024 Intel Corporation + * Copyright (C) 2018-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -678,7 +678,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenProcessResidencyFailingOnO BatchBuffer batchBuffer = BatchBufferHelper::createDefaultBatchBuffer(cs.getGraphicsAllocation(), &cs, cs.getUsed()); auto allocation = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); - executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocation, 1), false); + executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocation, 1), false, false); auto testedCsr = static_cast *>(csr); testedCsr->processResidencyCallBase = false; @@ -700,7 +700,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenProcessResidencyFailingOnO BatchBuffer batchBuffer = BatchBufferHelper::createDefaultBatchBuffer(cs.getGraphicsAllocation(), &cs, cs.getUsed()); auto allocation = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); - executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocation, 1), false); + executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocation, 1), false, false); auto testedCsr = static_cast *>(csr); testedCsr->processResidencyCallBase = false; @@ -722,7 +722,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenFailingExecWhenFlushingThe BatchBuffer batchBuffer = BatchBufferHelper::createDefaultBatchBuffer(cs.getGraphicsAllocation(), &cs, cs.getUsed()); auto allocation = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); - executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocation, 1), false); + executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocation, 1), false, false); auto testedCsr = static_cast *>(csr); testedCsr->processResidencyCallBase = true; @@ -1085,9 +1085,9 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenPrintBOsForSubmitWhenPrint std::string output = testing::internal::GetCapturedStdout(); std::vector bos; - allocation1.makeBOsResident(&csr->getOsContext(), 0, &bos, true); - allocation2.makeBOsResident(&csr->getOsContext(), 0, &bos, true); - cmdBuffer.makeBOsResident(&csr->getOsContext(), 0, &bos, true); + allocation1.makeBOsResident(&csr->getOsContext(), 0, &bos, true, false); + allocation2.makeBOsResident(&csr->getOsContext(), 0, &bos, true, false); + cmdBuffer.makeBOsResident(&csr->getOsContext(), 0, &bos, true, false); std::stringstream expected; expected << "Buffer object for submit\n"; diff --git a/shared/test/unit_test/os_interface/linux/drm_debug_prelim_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_debug_prelim_tests.cpp index e1f979b757..ac13677304 100644 --- a/shared/test/unit_test/os_interface/linux/drm_debug_prelim_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_debug_prelim_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022-2024 Intel Corporation + * Copyright (C) 2022-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -351,7 +351,7 @@ TEST_F(DrmDebugPrelimTest, givenAddedBindExtHandlesInBoWhenBindingWithinDefaultE OsContextLinux osContext(drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); osContext.ensureContextInitialized(false); - bo.bind(&osContext, 0); + bo.bind(&osContext, 0, false); EXPECT_NE(0u, drm.context.receivedVmBind->extensions); @@ -371,7 +371,7 @@ TEST_F(DrmDebugPrelimTest, givenAddedBindExtHandlesInBoWhenBindingWithinInternal OsContextLinux osContext(drm, 0, 0u, {{aub_stream::EngineType::ENGINE_RCS, EngineUsage::internal}, 1 /*deviceBitfield*/, PreemptionMode::Disabled, true /* isRootDevice*/}); osContext.ensureContextInitialized(false); - bo.bind(&osContext, 0); + bo.bind(&osContext, 0, false); EXPECT_FALSE(drm.context.receivedVmBindUuidExt[0]); } @@ -387,7 +387,7 @@ TEST_F(DrmDebugPrelimTest, givenAddedBindExtHandlesInBoWhenBindingWithinCopyEngi OsContextLinux osContext(drm, 0, 0u, {{aub_stream::EngineType::ENGINE_BCS, EngineUsage::regular}, 1 /*deviceBitfield*/, PreemptionMode::Disabled, true /* isRootDevice*/}); osContext.ensureContextInitialized(false); - bo.bind(&osContext, 0); + bo.bind(&osContext, 0, false); EXPECT_FALSE(drm.context.receivedVmBindUuidExt[0]); } @@ -402,7 +402,7 @@ HWTEST_F(DrmDebugPrelimTest, givenAddedBindExtHandlesInBoWhenUnbindingThenExtens OsContextLinux osContext(drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); osContext.ensureContextInitialized(false); - bo.bind(&osContext, 0); + bo.bind(&osContext, 0, false); EXPECT_NE(0u, drm.context.receivedVmBind.value().extensions); bo.unbind(&osContext, 0); diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index ecc81e26f5..f209c90e01 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -5936,7 +5936,7 @@ TEST_F(DrmMemoryManagerTest, givenPageFaultIsUnSupportedWhenCallingBindBoOnBuffe allocation.bufferObjects[0] = &bo; std::vector bufferObjects; - allocation.bindBO(&bo, &osContext, vmHandleId, &bufferObjects, true); + allocation.bindBO(&bo, &osContext, vmHandleId, &bufferObjects, true, false); EXPECT_FALSE(allocation.shouldAllocationPageFault(&drm)); EXPECT_FALSE(bo.isExplicitResidencyRequired()); @@ -5965,7 +5965,7 @@ TEST_F(DrmMemoryManagerTest, givenPageFaultIsSupportedAndKmdMigrationEnabledForB debugManager.flags.UseKmdMigrationForBuffers.set(useKmdMigrationForBuffers); std::vector bufferObjects; - allocation.bindBO(&bo, &osContext, vmHandleId, &bufferObjects, true); + allocation.bindBO(&bo, &osContext, vmHandleId, &bufferObjects, true, false); if (useKmdMigrationForBuffers > 0) { EXPECT_TRUE(allocation.shouldAllocationPageFault(&drm)); @@ -6004,7 +6004,7 @@ TEST_F(DrmMemoryManagerTest, givenPageFaultIsSupportedWhenCallingBindBoOnAllocat allocation.shouldPageFault = shouldAllocationPageFault; std::vector bufferObjects; - allocation.bindBO(&bo, &osContext, vmHandleId, &bufferObjects, true); + allocation.bindBO(&bo, &osContext, vmHandleId, &bufferObjects, true, false); EXPECT_EQ(shouldAllocationPageFault, allocation.shouldAllocationPageFault(&drm)); EXPECT_EQ(!shouldAllocationPageFault, bo.isExplicitResidencyRequired()); diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_operations_handler_bind_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_operations_handler_bind_tests.cpp index 29ab80f1ad..55a44c985c 100755 --- a/shared/test/unit_test/os_interface/linux/drm_memory_operations_handler_bind_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_operations_handler_bind_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022-2024 Intel Corporation + * Copyright (C) 2022-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -261,8 +261,8 @@ TEST_F(DrmMemoryOperationsHandlerBindMultiRootDeviceTest2, givenOperationHandler EXPECT_EQ(operationHandlerDefault->getRootDeviceIndex(), 0u); EXPECT_EQ(operationHandler->getRootDeviceIndex(), 1u); - operationHandlerDefault->makeResident(device, ArrayRef(&allocationDefault, 1), false); - operationHandler->makeResident(device, ArrayRef(&allocation, 1), false); + operationHandlerDefault->makeResident(device, ArrayRef(&allocationDefault, 1), false, false); + operationHandler->makeResident(device, ArrayRef(&allocation, 1), false, false); operationHandlerDefault->setRootDeviceIndex(1u); operationHandler->setRootDeviceIndex(0u); @@ -294,7 +294,7 @@ TEST_F(DrmMemoryOperationsHandlerBindMultiRootDeviceTest2, whenNoSpaceLeftOnDevi EXPECT_EQ(allocationDefault, registeredAllocations[1]); EXPECT_EQ(operationHandler->evictUnusedCalled, 0u); - auto res = operationHandler->makeResident(device, ArrayRef(&allocation, 1), false); + auto res = operationHandler->makeResident(device, ArrayRef(&allocation, 1), false, false); EXPECT_EQ(MemoryOperationsStatus::outOfMemory, res); EXPECT_EQ(operationHandler->evictUnusedCalled, 1u); @@ -309,17 +309,17 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenObjectAlwaysResidentAndNotUsedWh for (auto &engine : device->getAllEngines()) { *engine.commandStreamReceiver->getTagAddress() = 10; allocation->updateTaskCount(GraphicsAllocation::objectNotUsed, engine.osContext->getContextId()); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); } for (auto &engine : device->getSubDevice(0u)->getAllEngines()) { *engine.commandStreamReceiver->getTagAddress() = 10; allocation->updateResidencyTaskCount(GraphicsAllocation::objectAlwaysResident, engine.osContext->getContextId()); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); } for (auto &engine : device->getSubDevice(1u)->getAllEngines()) { *engine.commandStreamReceiver->getTagAddress() = 10; allocation->updateTaskCount(GraphicsAllocation::objectNotUsed, engine.osContext->getContextId()); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); } EXPECT_EQ(mock->context.vmBindCalled, 2u); @@ -375,17 +375,17 @@ HWTEST_F(DrmMemoryOperationsHandlerBindTest, whenEvictUnusedResourcesWithWaitFor for (auto &engine : device->getAllEngines()) { *engine.commandStreamReceiver->getTagAddress() = 10; allocation->updateTaskCount(8u, engine.osContext->getContextId()); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); } for (auto &engine : device->getSubDevice(0u)->getAllEngines()) { *engine.commandStreamReceiver->getTagAddress() = 10; allocation->updateTaskCount(8u, engine.osContext->getContextId()); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); } for (auto &engine : device->getSubDevice(1u)->getAllEngines()) { *engine.commandStreamReceiver->getTagAddress() = 10; allocation->updateTaskCount(8u, engine.osContext->getContextId()); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); } *device->getSubDevice(1u)->getDefaultEngine().commandStreamReceiver->getTagAddress() = 5; @@ -420,17 +420,17 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, whenRunningOutOfMemoryThenUnusedAlloc for (auto &engine : device->getAllEngines()) { *engine.commandStreamReceiver->getTagAddress() = 10; allocation->updateTaskCount(8u, engine.osContext->getContextId()); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); } for (auto &engine : device->getSubDevice(0u)->getAllEngines()) { *engine.commandStreamReceiver->getTagAddress() = 10; allocation->updateTaskCount(8u, engine.osContext->getContextId()); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); } for (auto &engine : device->getSubDevice(1u)->getAllEngines()) { *engine.commandStreamReceiver->getTagAddress() = 10; allocation->updateTaskCount(8u, engine.osContext->getContextId()); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); } *device->getSubDevice(1u)->getDefaultEngine().commandStreamReceiver->getTagAddress() = 5; @@ -450,17 +450,17 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenUsedAllocationInBothSubdevicesWh for (auto &engine : device->getAllEngines()) { *engine.commandStreamReceiver->getTagAddress() = 5; allocation->updateTaskCount(8u, engine.osContext->getContextId()); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); } for (auto &engine : device->getSubDevice(0u)->getAllEngines()) { *engine.commandStreamReceiver->getTagAddress() = 5; allocation->updateTaskCount(8u, engine.osContext->getContextId()); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); } for (auto &engine : device->getSubDevice(1u)->getAllEngines()) { *engine.commandStreamReceiver->getTagAddress() = 5; allocation->updateTaskCount(8u, engine.osContext->getContextId()); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(engine.osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); } EXPECT_EQ(mock->context.vmBindCalled, 2u); @@ -473,12 +473,12 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenUsedAllocationInBothSubdevicesWh memoryManager->freeGraphicsMemory(allocation); } -TEST_F(DrmMemoryOperationsHandlerBindTest, givenResidencyWithinOsContextFailsThenThenMergeWithResidencyContainertReturnsError) { +TEST_F(DrmMemoryOperationsHandlerBindTest, givenResidencyWithinOsContextFailsThenMergeWithResidencyContainertReturnsError) { struct MockDrmMemoryOperationsHandlerBindResidencyFail : public DrmMemoryOperationsHandlerBind { MockDrmMemoryOperationsHandlerBindResidencyFail(RootDeviceEnvironment &rootDeviceEnvironment, uint32_t rootDeviceIndex) : DrmMemoryOperationsHandlerBind(rootDeviceEnvironment, rootDeviceIndex) {} - MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) override { + MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence) override { return NEO::MemoryOperationsStatus::failed; } }; @@ -539,7 +539,7 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenMakeBOsResidentFailsThenMakeResi : DrmAllocation(rootDeviceIndex, 1u /*num gmms*/, allocationType, bos, ptrIn, gpuAddress, sizeIn, pool) { } - int makeBOsResident(OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind) override { + int makeBOsResident(OsContext *osContext, uint32_t vmHandleId, std::vector *bufferObjects, bool bind, const bool forcePagingFence) override { return -1; } }; @@ -552,7 +552,7 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenMakeBOsResidentFailsThenMakeResi auto allocation = new MockDrmAllocationBOsResident(0, AllocationType::unknown, bos, nullptr, 0u, size, MemoryPool::localMemory); auto graphicsAllocation = static_cast(allocation); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&graphicsAllocation, 1), false), MemoryOperationsStatus::outOfMemory); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&graphicsAllocation, 1), false, false), MemoryOperationsStatus::outOfMemory); delete allocation; } @@ -581,19 +581,19 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, allocation->storageInfo.subDeviceBitfield = 0b0011; auto graphicsAllocation = static_cast(allocation); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&graphicsAllocation, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&graphicsAllocation, 1), false, false), MemoryOperationsStatus::success); delete allocation; } TEST_F(DrmMemoryOperationsHandlerBindTest, givenDrmMemoryOperationBindWhenMakeResidentWithinOsContextEvictableAllocationThenAllocationIsNotMarkedAsAlwaysResident) { auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{device->getRootDeviceIndex(), MemoryConstants::pageSize}); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&allocation, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&allocation, 1), false, false), MemoryOperationsStatus::success); EXPECT_TRUE(allocation->isAlwaysResident(device->getDefaultEngine().osContext->getContextId())); EXPECT_EQ(operationHandler->evict(device, *allocation), MemoryOperationsStatus::success); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); EXPECT_FALSE(allocation->isAlwaysResident(device->getDefaultEngine().osContext->getContextId())); memoryManager->freeGraphicsMemory(allocation); @@ -603,12 +603,12 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenDrmMemoryOperationBindWhenMakeRe auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{device->getRootDeviceIndex(), MemoryConstants::pageSize}); allocation->storageInfo.isChunked = true; - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&allocation, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&allocation, 1), false, false), MemoryOperationsStatus::success); EXPECT_TRUE(allocation->isAlwaysResident(device->getDefaultEngine().osContext->getContextId())); EXPECT_EQ(operationHandler->evict(device, *allocation), MemoryOperationsStatus::success); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); EXPECT_FALSE(allocation->isAlwaysResident(device->getDefaultEngine().osContext->getContextId())); memoryManager->freeGraphicsMemory(allocation); @@ -619,12 +619,12 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenDrmMemoryOperationBindWhenMakeRe allocation->storageInfo.isChunked = true; allocation->storageInfo.memoryBanks = 0x5; - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&allocation, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&allocation, 1), false, false), MemoryOperationsStatus::success); EXPECT_TRUE(allocation->isAlwaysResident(device->getDefaultEngine().osContext->getContextId())); EXPECT_EQ(operationHandler->evict(device, *allocation), MemoryOperationsStatus::success); - EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&allocation, 1), true), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResidentWithinOsContext(device->getDefaultEngine().osContext, ArrayRef(&allocation, 1), true, false), MemoryOperationsStatus::success); EXPECT_FALSE(allocation->isAlwaysResident(device->getDefaultEngine().osContext->getContextId())); memoryManager->freeGraphicsMemory(allocation); @@ -634,7 +634,7 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenDrmMemoryOperationBindWhenChangi auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{device->getRootDeviceIndex(), MemoryConstants::pageSize}); EXPECT_EQ(operationHandler->isResident(device, *allocation), MemoryOperationsStatus::memoryNotFound); - EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&allocation, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&allocation, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(operationHandler->isResident(device, *allocation), MemoryOperationsStatus::success); EXPECT_EQ(operationHandler->evict(device, *allocation), MemoryOperationsStatus::success); EXPECT_EQ(operationHandler->isResident(device, *allocation), MemoryOperationsStatus::memoryNotFound); @@ -649,7 +649,7 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenDeviceWithMultipleSubdevicesWhen EXPECT_EQ(operationHandler->isResident(device->getSubDevice(0u), *allocation), MemoryOperationsStatus::memoryNotFound); EXPECT_EQ(operationHandler->isResident(device->getSubDevice(1u), *allocation), MemoryOperationsStatus::memoryNotFound); - auto retVal = operationHandler->makeResident(device->getSubDevice(1u), ArrayRef(&allocation, 1), false); + auto retVal = operationHandler->makeResident(device->getSubDevice(1u), ArrayRef(&allocation, 1), false, false); EXPECT_EQ(retVal, MemoryOperationsStatus::success); EXPECT_EQ(operationHandler->isResident(device, *allocation), MemoryOperationsStatus::memoryNotFound); @@ -677,7 +677,7 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, whenIoctlFailDuringEvictingThenUnreco auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{device->getRootDeviceIndex(), MemoryConstants::pageSize}); EXPECT_EQ(operationHandler->isResident(device, *allocation), MemoryOperationsStatus::memoryNotFound); - EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&allocation, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&allocation, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(operationHandler->isResident(device, *allocation), MemoryOperationsStatus::success); mock->context.vmUnbindReturn = -1; @@ -691,8 +691,8 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, whenIoctlFailDuringEvictingThenUnreco TEST_F(DrmMemoryOperationsHandlerBindTest, whenMakeResidentTwiceThenAllocIsBoundOnlyOnce) { auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{device->getRootDeviceIndex(), MemoryConstants::pageSize}); - EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&allocation, 1), false), MemoryOperationsStatus::success); - EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&allocation, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&allocation, 1), false, false), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&allocation, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(operationHandler->isResident(device, *allocation), MemoryOperationsStatus::success); EXPECT_EQ(mock->context.vmBindCalled, 2u); @@ -712,7 +712,7 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenNoVmBindSupportInDrmWhenCheckFor mock->context.vmBindCalled = 0u; auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{device->getRootDeviceIndex(), MemoryConstants::pageSize}); - handler->makeResident(device, ArrayRef(&allocation, 1), false); + handler->makeResident(device, ArrayRef(&allocation, 1), false, false); EXPECT_FALSE(mock->context.vmBindCalled); memoryManager->freeGraphicsMemory(allocation); @@ -727,7 +727,7 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenVmBindSupportAndNoMultiTileWhenC mock->context.vmBindCalled = 0u; auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{device->getRootDeviceIndex(), MemoryConstants::pageSize}); - handler->makeResident(device, ArrayRef(&allocation, 1), false); + handler->makeResident(device, ArrayRef(&allocation, 1), false, false); EXPECT_FALSE(mock->context.vmBindCalled); memoryManager->freeGraphicsMemory(allocation); @@ -741,7 +741,7 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenDisabledVmBindWhenCreateDrmHandl mock->context.vmBindCalled = false; auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{device->getRootDeviceIndex(), MemoryConstants::pageSize}); - handler->makeResident(device, ArrayRef(&allocation, 1), false); + handler->makeResident(device, ArrayRef(&allocation, 1), false, false); EXPECT_FALSE(mock->context.vmBindCalled); memoryManager->freeGraphicsMemory(allocation); @@ -1072,9 +1072,9 @@ HWTEST_F(DrmMemoryOperationsHandlerBindTest, givenCsrTagAllocatorsWhenDestructin auto hwTimeStampsAlloc = csr->getEventTsAllocator()->getTag()->getBaseGraphicsAllocation()->getDefaultGraphicsAllocation(); auto hwPerfCounterAlloc = csr->getEventPerfCountAllocator(4)->getTag()->getBaseGraphicsAllocation()->getDefaultGraphicsAllocation(); - operationHandler->makeResident(device, ArrayRef(×tampStorageAlloc, 1), false); - operationHandler->makeResident(device, ArrayRef(&hwTimeStampsAlloc, 1), false); - operationHandler->makeResident(device, ArrayRef(&hwPerfCounterAlloc, 1), false); + operationHandler->makeResident(device, ArrayRef(×tampStorageAlloc, 1), false, false); + operationHandler->makeResident(device, ArrayRef(&hwTimeStampsAlloc, 1), false, false); + operationHandler->makeResident(device, ArrayRef(&hwPerfCounterAlloc, 1), false, false); csr.reset(); @@ -1113,7 +1113,7 @@ HWTEST_F(DrmMemoryOperationsHandlerBindTest, givenPatIndexProgrammingEnabledWhen bo.setPatIndex(mock->getPatIndex(allocation.getDefaultGmm(), allocation.getAllocationType(), CacheRegion::defaultRegion, CachePolicy::writeBack, (debugFlag == 1 && closSupported), true)); - operationHandler->makeResident(device, ArrayRef(&allocationPtr, 1), false); + operationHandler->makeResident(device, ArrayRef(&allocationPtr, 1), false, false); if (!patIndexProgrammingSupported) { EXPECT_FALSE(mock->context.receivedVmBindPatIndex); @@ -1184,7 +1184,7 @@ HWTEST_F(DrmMemoryOperationsHandlerBindTest, givenUncachedDebugFlagSetWhenVmBind mock->context.receivedVmUnbindPatIndex.reset(); auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{device->getRootDeviceIndex(), MemoryConstants::pageSize}); - operationHandler->makeResident(device, ArrayRef(&allocation, 1), false); + operationHandler->makeResident(device, ArrayRef(&allocation, 1), false, false); auto expectedIndex = productHelper.overridePatIndex(true, static_cast(MockGmmClientContextBase::MockPatIndex::uncached), allocation->getAllocationType()); @@ -1214,7 +1214,7 @@ HWTEST_F(DrmMemoryOperationsHandlerBindTest, givenDebugFlagSetWhenVmBindCalledTh GTEST_SKIP(); } - operationHandler->makeResident(device, ArrayRef(×tampStorageAlloc, 1), false); + operationHandler->makeResident(device, ArrayRef(×tampStorageAlloc, 1), false, false); EXPECT_EQ(1u, mock->context.receivedVmBindPatIndex.value()); @@ -1245,7 +1245,7 @@ HWTEST_F(DrmMemoryOperationsHandlerBindTest, givenDebugFlagSetWhenVmBindCalledTh GraphicsAllocation *allocPtr = &allocation; - operationHandler->makeResident(device, ArrayRef(&allocPtr, 1), false); + operationHandler->makeResident(device, ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(2u, mock->context.receivedVmBindPatIndex.value()); @@ -1276,7 +1276,7 @@ HWTEST_F(DrmMemoryOperationsHandlerBindTest, givenDebugFlagSetWhenVmBindCalledTh GraphicsAllocation *allocPtr = &allocation; - operationHandler->makeResident(device, ArrayRef(&allocPtr, 1), false); + operationHandler->makeResident(device, ArrayRef(&allocPtr, 1), false, false); EXPECT_EQ(3u, mock->context.receivedVmBindPatIndex.value()); @@ -1308,7 +1308,7 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenClosEnabledAndAllocationToBeCach EXPECT_TRUE(static_cast(allocation)->setCacheAdvice(mock, 32 * MemoryConstants::kiloByte, cacheRegion, false)); mock->context.receivedVmBindPatIndex.reset(); - operationHandler->makeResident(device, ArrayRef(&allocation, 1), false); + operationHandler->makeResident(device, ArrayRef(&allocation, 1), false, false); auto patIndex = productHelper.getPatIndex(cacheRegion, CachePolicy::writeBack); @@ -1375,7 +1375,7 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenPreviouslyLockedMemoryWhenCallin EXPECT_EQ(operationHandler->isResident(device, *mockDrmAllocation), MemoryOperationsStatus::memoryNotFound); EXPECT_FALSE(mockDrmAllocation->isLockedMemory()); - EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&mockDrmAllocation, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&mockDrmAllocation, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(operationHandler->isResident(device, *mockDrmAllocation), MemoryOperationsStatus::success); EXPECT_FALSE(mockDrmAllocation->isLockedMemory()); EXPECT_FALSE(mockBo.isExplicitLockedMemoryRequired()); @@ -1393,7 +1393,7 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenLockedAndResidentAllocationsWhen EXPECT_EQ(operationHandler->lock(device, ArrayRef(&allocation1, 1)), MemoryOperationsStatus::success); EXPECT_EQ(operationHandler->isResident(device, *allocation1), MemoryOperationsStatus::success); - EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&allocation2, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&allocation2, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(operationHandler->isResident(device, *allocation2), MemoryOperationsStatus::success); operationHandler->useBaseEvictUnused = true; @@ -1410,7 +1410,7 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenLockedAndResidentAllocationsWhen TEST_F(DrmMemoryOperationsHandlerBindTest, whenCallingMakeResidentThenVerifyImmediateBindingIsRequired) { auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{device->getRootDeviceIndex(), MemoryConstants::pageSize}); - EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&allocation, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&allocation, 1), false, false), MemoryOperationsStatus::success); auto bo = static_cast(allocation)->getBO(); EXPECT_TRUE(bo->isImmediateBindingRequired()); @@ -1420,7 +1420,7 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, whenCallingMakeResidentThenVerifyImme TEST_F(DrmMemoryOperationsHandlerBindTest, givenDrmMemoryOperationBindWhenCallingEvictAfterCallingResidentThenVerifyImmediateBindingIsNotRequired) { auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{device->getRootDeviceIndex(), MemoryConstants::pageSize}); - EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&allocation, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(operationHandler->makeResident(device, ArrayRef(&allocation, 1), false, false), MemoryOperationsStatus::success); auto bo = static_cast(allocation)->getBO(); EXPECT_TRUE(bo->isImmediateBindingRequired()); diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_operations_handler_default_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_operations_handler_default_tests.cpp index a8d7a9329d..abdf6ab699 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_operations_handler_default_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_operations_handler_default_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 Intel Corporation + * Copyright (C) 2019-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ struct MockDrmMemoryOperationsHandlerDefault : public DrmMemoryOperationsHandler using BaseClass = DrmMemoryOperationsHandlerDefault; using DrmMemoryOperationsHandlerDefault::DrmMemoryOperationsHandlerDefault; using DrmMemoryOperationsHandlerDefault::residency; - ADDMETHOD(makeResidentWithinOsContext, MemoryOperationsStatus, true, MemoryOperationsStatus::success, (OsContext * osContext, ArrayRef gfxAllocations, bool evictable), (osContext, gfxAllocations, evictable)); + ADDMETHOD(makeResidentWithinOsContext, MemoryOperationsStatus, true, MemoryOperationsStatus::success, (OsContext * osContext, ArrayRef gfxAllocations, bool evictable, const bool forcePagingFence), (osContext, gfxAllocations, evictable, forcePagingFence)); ADDMETHOD(flushDummyExec, MemoryOperationsStatus, true, MemoryOperationsStatus::success, (Device * device, ArrayRef gfxAllocations), (device, gfxAllocations)); ADDMETHOD(evictWithinOsContext, MemoryOperationsStatus, true, MemoryOperationsStatus::success, (OsContext * osContext, GraphicsAllocation &gfxAllocation), (osContext, gfxAllocation)); }; @@ -73,7 +73,7 @@ TEST_F(DrmMemoryOperationsHandlerBaseTest, whenMakingAllocationResidentThenAlloc initializeAllocation(1); EXPECT_EQ(1u, drmAllocation->storageInfo.getNumBanks()); - EXPECT_EQ(drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(drmMemoryOperationsHandler->residency.size(), 1u); EXPECT_TRUE(drmMemoryOperationsHandler->residency.find(allocationPtr) != drmMemoryOperationsHandler->residency.end()); EXPECT_EQ(drmMemoryOperationsHandler->isResident(nullptr, *allocationPtr), MemoryOperationsStatus::success); @@ -84,7 +84,7 @@ TEST_F(DrmMemoryOperationsHandlerBaseTest, whenEvictingResidentAllocationThenAll EXPECT_EQ(1u, drmAllocation->storageInfo.getNumBanks()); EXPECT_EQ(drmMemoryOperationsHandler->residency.size(), 0u); - EXPECT_EQ(drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(drmMemoryOperationsHandler->isResident(nullptr, *allocationPtr), MemoryOperationsStatus::success); EXPECT_EQ(drmMemoryOperationsHandler->residency.size(), 1u); EXPECT_TRUE(drmMemoryOperationsHandler->residency.find(allocationPtr) != drmMemoryOperationsHandler->residency.end()); @@ -174,7 +174,7 @@ TEST_F(DrmMemoryOperationsHandlerBaseTest, givenOperationsHandlerWhenCallingMake drmMemoryOperationsHandler->makeResidentWithinOsContextCallBase = false; initializeAllocation(1); - drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false); + drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false, false); EXPECT_EQ(drmMemoryOperationsHandler->flushDummyExecCalled, 0u); } TEST_F(DrmMemoryOperationsHandlerBaseTest, givenOperationsHandlerWhenMakeResidentWithinOsContextReturnFailhenFlushDummyExecNotCalled) { @@ -183,7 +183,7 @@ TEST_F(DrmMemoryOperationsHandlerBaseTest, givenOperationsHandlerWhenMakeResiden drmMemoryOperationsHandler->makeResidentWithinOsContextResult = MemoryOperationsStatus::failed; initializeAllocation(1); - drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), true); + drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), true, false); EXPECT_EQ(drmMemoryOperationsHandler->flushDummyExecCalled, 0u); } @@ -192,7 +192,7 @@ TEST_F(DrmMemoryOperationsHandlerBaseTest, givenOperationsHandlerWhenCallingMake drmMemoryOperationsHandler->makeResidentWithinOsContextCallBase = false; initializeAllocation(1); - drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), true); + drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), true, false); EXPECT_EQ(drmMemoryOperationsHandler->flushDummyExecCalled, 1u); } @@ -202,7 +202,7 @@ TEST_F(DrmMemoryOperationsHandlerBaseTest, givenOperationsHandlerWhenFlushReturn drmMemoryOperationsHandler->makeResidentWithinOsContextCallBase = false; initializeAllocation(1); - drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), true); + drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), true, false); EXPECT_EQ(drmMemoryOperationsHandler->evictWithinOsContextCalled, 1u); } @@ -212,7 +212,7 @@ TEST_F(DrmMemoryOperationsHandlerBaseTest, givenOperationsHandlerWhenFlushReturn drmMemoryOperationsHandler->makeResidentWithinOsContextCallBase = false; initializeAllocation(1); - drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), true); + drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), true, false); EXPECT_EQ(drmMemoryOperationsHandler->evictWithinOsContextCalled, 0u); } diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_operations_handler_with_aub_dump_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_operations_handler_with_aub_dump_tests.cpp index 4cc98191ef..01a8fc772e 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_operations_handler_with_aub_dump_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_operations_handler_with_aub_dump_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023-2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -53,7 +53,7 @@ struct DrmMemoryOperationsHandlerWithAubDumpTest : public ::testing::Test { }; TEST_F(DrmMemoryOperationsHandlerWithAubDumpTest, whenMakingAllocationResidentThenAllocationIsResident) { - EXPECT_EQ(drmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(drmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(drmMemoryOperationsHandlerWithAubDumpMock->residency.size(), 1u); EXPECT_TRUE(drmMemoryOperationsHandlerWithAubDumpMock->residency.find(allocationPtr) != drmMemoryOperationsHandlerWithAubDumpMock->residency.end()); EXPECT_EQ(drmMemoryOperationsHandlerWithAubDumpMock->isResident(nullptr, graphicsAllocation), MemoryOperationsStatus::success); @@ -66,7 +66,7 @@ TEST_F(DrmMemoryOperationsHandlerWithAubDumpTest, givenRegularAllocationWhenFree mockAubMemoryOperationsHandler->setAubManager(&aubManager); allocationPtr->setAubWritable(true, 0xff); - EXPECT_EQ(drmMemoryOperationsHandlerWithAubDumpMock->makeResident(device.get(), ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(drmMemoryOperationsHandlerWithAubDumpMock->makeResident(device.get(), ArrayRef(&allocationPtr, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(drmMemoryOperationsHandlerWithAubDumpMock->residency.size(), 1u); EXPECT_TRUE(drmMemoryOperationsHandlerWithAubDumpMock->residency.find(allocationPtr) != drmMemoryOperationsHandlerWithAubDumpMock->residency.end()); EXPECT_EQ(1u, mockAubMemoryOperationsHandler->residentAllocations.size()); @@ -90,7 +90,7 @@ TEST_F(DrmMemoryOperationsHandlerWithAubDumpTest, whenEvictingResidentAllocation GraphicsAllocation *mockDrmAllocation = new MockDrmAllocation(AllocationType::unknown, MemoryPool::localMemory, bos); EXPECT_EQ(drmMemoryOperationsHandlerWithAubDumpMock->residency.size(), 0u); - EXPECT_EQ(drmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(&mockDrmAllocation, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(drmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(&mockDrmAllocation, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(drmMemoryOperationsHandlerWithAubDumpMock->isResident(nullptr, *mockDrmAllocation), MemoryOperationsStatus::success); EXPECT_EQ(drmMemoryOperationsHandlerWithAubDumpMock->residency.size(), 1u); EXPECT_TRUE(drmMemoryOperationsHandlerWithAubDumpMock->residency.find(mockDrmAllocation) != drmMemoryOperationsHandlerWithAubDumpMock->residency.end()); diff --git a/shared/test/unit_test/os_interface/linux/drm_query_prelim_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_query_prelim_tests.cpp index b742c56512..5690a7ae67 100644 --- a/shared/test/unit_test/os_interface/linux/drm_query_prelim_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_query_prelim_tests.cpp @@ -390,7 +390,7 @@ TEST(DrmBufferObjectTestPrelim, givenBufferObjectSetToColourWithBindWhenBindingT OsContextLinux osContext(drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); osContext.ensureContextInitialized(false); - bo.bind(&osContext, 0); + bo.bind(&osContext, 0, false); ASSERT_TRUE(drm.context.receivedVmBind); EXPECT_EQ(drm.context.receivedVmBind->length, MemoryConstants::pageSize64k); EXPECT_EQ(drm.context.receivedVmBind->start, 0xffeeffee); @@ -462,7 +462,7 @@ TEST(DrmBufferObjectTestPrelim, givenBufferObjectMarkedForCaptureWhenBindingThen osContext.ensureContextInitialized(false); bo.markForCapture(); - bo.bind(&osContext, 0); + bo.bind(&osContext, 0, false); ASSERT_TRUE(drm.context.receivedVmBind); EXPECT_TRUE(drm.context.receivedVmBind->flags & DrmPrelimHelper::getCaptureVmBindFlag()); } @@ -479,7 +479,7 @@ TEST(DrmBufferObjectTestPrelim, givenNoActiveDirectSubmissionAndForceUseImmediat OsContextLinux osContext(drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); osContext.ensureContextInitialized(false); - bo.bind(&osContext, 0); + bo.bind(&osContext, 0, false); ASSERT_TRUE(drm.context.receivedVmBind); EXPECT_TRUE(drm.context.receivedVmBind->flags & DrmPrelimHelper::getImmediateVmBindFlag()); EXPECT_NE(drm.context.receivedVmBind->extensions, 0u); @@ -496,7 +496,7 @@ TEST(DrmBufferObjectTestPrelim, whenBindingThenImmediateFlagIsSetAndExtensionLis osContext.ensureContextInitialized(false); osContext.setDirectSubmissionActive(); - bo.bind(&osContext, 0); + bo.bind(&osContext, 0, false); ASSERT_TRUE(drm.context.receivedVmBind); EXPECT_TRUE(drm.context.receivedVmBind->flags & DrmPrelimHelper::getImmediateVmBindFlag()); EXPECT_NE(drm.context.receivedVmBind->extensions, 0u); diff --git a/shared/test/unit_test/os_interface/linux/drm_vm_bind_prelim_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_vm_bind_prelim_tests.cpp index 9405e596f2..3d14b205c2 100644 --- a/shared/test/unit_test/os_interface/linux/drm_vm_bind_prelim_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_vm_bind_prelim_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2024 Intel Corporation + * Copyright (C) 2018-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -29,7 +29,7 @@ TEST(DrmVmBindTest, givenBoRequiringImmediateBindWhenBindingThenImmediateFlagIsP OsContextLinux osContext(drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); osContext.ensureContextInitialized(false); - bo.bind(&osContext, 0); + bo.bind(&osContext, 0, false); EXPECT_EQ(DrmPrelimHelper::getImmediateVmBindFlag(), drm.context.receivedVmBind->flags); } @@ -48,7 +48,7 @@ TEST(DrmVmBindTest, givenBoRequiringExplicitResidencyWhenBindingThenMakeResident OsContextLinux osContext(drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); osContext.ensureContextInitialized(false); uint32_t vmHandleId = 0; - bo.bind(&osContext, vmHandleId); + bo.bind(&osContext, vmHandleId, false); if (requireResidency) { EXPECT_EQ(DrmPrelimHelper::getImmediateVmBindFlag() | DrmPrelimHelper::getMakeResidentVmBindFlag(), drm.context.receivedVmBind->flags); @@ -77,7 +77,7 @@ TEST(DrmVmBindTest, OsContextLinux osContext(drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); osContext.ensureContextInitialized(false); uint32_t vmHandleId = 0; - bo.bind(&osContext, vmHandleId); + bo.bind(&osContext, vmHandleId, false); if (requireResidency) { EXPECT_EQ(DrmPrelimHelper::getImmediateVmBindFlag() | DrmPrelimHelper::getMakeResidentVmBindFlag(), drm.context.receivedVmBind->flags); @@ -105,7 +105,7 @@ TEST(DrmVmBindTest, givenPerContextVmsAndBoRequiringExplicitResidencyWhenBinding MockOsContextLinux osContext(drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); osContext.ensureContextInitialized(false); uint32_t vmHandleId = 0; - bo.bind(&osContext, vmHandleId); + bo.bind(&osContext, vmHandleId, false); if (requireResidency) { EXPECT_EQ(DrmPrelimHelper::getImmediateVmBindFlag() | DrmPrelimHelper::getMakeResidentVmBindFlag(), drm.context.receivedVmBind->flags); @@ -144,7 +144,7 @@ TEST(DrmVmBindTest, whenCallingWaitForBindThenWaitUserFenceIsCalled) { OsContextLinux osContext(drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); osContext.ensureContextInitialized(false); uint32_t vmHandleId = 0; - bo.bind(&osContext, vmHandleId); + bo.bind(&osContext, vmHandleId, false); drm.waitForBind(vmHandleId); @@ -171,7 +171,7 @@ TEST(DrmVmBindTest, givenUseKmdMigrationWhenCallingBindBoOnUnifiedSharedMemoryTh MockDrmAllocation allocation(0u, AllocationType::unifiedSharedMemory, MemoryPool::localMemory); allocation.bufferObjects[0] = &bo; - allocation.bindBO(&bo, &osContext, vmHandleId, nullptr, true); + allocation.bindBO(&bo, &osContext, vmHandleId, nullptr, true, false); EXPECT_TRUE(allocation.shouldAllocationPageFault(&drm)); EXPECT_FALSE(bo.isExplicitResidencyRequired()); @@ -194,7 +194,7 @@ TEST(DrmVmBindTest, givenDrmWithPageFaultSupportWhenCallingBindBoOnUnifiedShared MockDrmAllocation allocation(0u, AllocationType::unifiedSharedMemory, MemoryPool::localMemory); allocation.bufferObjects[0] = &bo; - allocation.bindBO(&bo, &osContext, vmHandleId, nullptr, true); + allocation.bindBO(&bo, &osContext, vmHandleId, nullptr, true, false); auto &productHelper = drm.getRootDeviceEnvironment().getHelper(); auto kmdMigrationSupported = productHelper.isKmdMigrationSupported(); diff --git a/shared/test/unit_test/os_interface/linux/os_context_linux_tests.cpp b/shared/test/unit_test/os_interface/linux/os_context_linux_tests.cpp index 2ea7fbc09b..2443800bde 100644 --- a/shared/test/unit_test/os_interface/linux/os_context_linux_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/os_context_linux_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2024 Intel Corporation + * Copyright (C) 2019-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -116,3 +116,49 @@ TEST(OSContextLinux, givenPerContextVmsAndBindCompleteWhenWaitForPagingFenceThen EXPECT_EQ(0u, drm.waitUserFenceParams.size()); } + +TEST(OSContextLinux, givenPerContextVmsAndBindNotCompleteWhenWaitForPagingFenceGivenFenceValThenContextFenceIsPassedToWaitUserFenceIoctl) { + auto executionEnvironment = std::make_unique(); + DrmMock drm{*executionEnvironment->rootDeviceEnvironments[0]}; + drm.requirePerContextVM = true; + + MockOsContextLinux osContext(drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); + osContext.ensureContextInitialized(false); + + drm.pagingFence[0] = 26u; + + osContext.pagingFence[0] = 46u; + uint64_t fenceValToWait = 51u; + + osContext.waitForPagingFenceGivenFenceVal(fenceValToWait); + + EXPECT_EQ(1u, drm.waitUserFenceParams.size()); + EXPECT_EQ(0u, drm.waitUserFenceParams[0].ctxId); + EXPECT_EQ(castToUint64(&osContext.pagingFence[0]), drm.waitUserFenceParams[0].address); + EXPECT_EQ(drm.ioctlHelper->getWaitUserFenceSoftFlag(), drm.waitUserFenceParams[0].flags); + EXPECT_EQ(fenceValToWait, drm.waitUserFenceParams[0].value); + EXPECT_EQ(-1, drm.waitUserFenceParams[0].timeout); + + drm.requirePerContextVM = false; + osContext.waitForPagingFenceGivenFenceVal(fenceValToWait); + + EXPECT_EQ(castToUint64(&drm.pagingFence[0]), drm.waitUserFenceParams[1].address); + EXPECT_EQ(drm.ioctlHelper->getWaitUserFenceSoftFlag(), drm.waitUserFenceParams[1].flags); + EXPECT_EQ(fenceValToWait, drm.waitUserFenceParams[1].value); +} + +TEST(OSContextLinux, givenPerContextVmsAndBindCompleteWhenWaitForPagingFenceGivenFenceValThenWaitUserFenceIoctlIsNotCalled) { + auto executionEnvironment = std::make_unique(); + DrmMock drm{*executionEnvironment->rootDeviceEnvironments[0]}; + drm.requirePerContextVM = true; + + MockOsContextLinux osContext(drm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); + osContext.ensureContextInitialized(false); + + osContext.pagingFence[0] = 3u; + uint64_t fenceValToWait = 3u; + + osContext.waitForPagingFenceGivenFenceVal(fenceValToWait); + + EXPECT_EQ(0u, drm.waitUserFenceParams.size()); +} diff --git a/shared/test/unit_test/os_interface/windows/wddm_memory_operations_handler_with_aub_dump_tests.cpp b/shared/test/unit_test/os_interface/windows/wddm_memory_operations_handler_with_aub_dump_tests.cpp index 0720eac024..e18568d959 100644 --- a/shared/test/unit_test/os_interface/windows/wddm_memory_operations_handler_with_aub_dump_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/wddm_memory_operations_handler_with_aub_dump_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023-2024 Intel Corporation + * Copyright (C) 2023-2025 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -66,7 +66,7 @@ struct WddmMemoryOperationsHandlerWithAubDumpTest : public WddmTest { }; TEST_F(WddmMemoryOperationsHandlerWithAubDumpTest, givenRegularAllocationWhenMakingResidentAllocationThenMakeResidentCalled) { - EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->isResident(nullptr, *wddmAllocation), MemoryOperationsStatus::success); EXPECT_TRUE(mockAubMemoryOperationsHandler->makeResidentCalled); EXPECT_TRUE(mockAubMemoryOperationsHandler->isResidentCalled); @@ -74,14 +74,14 @@ TEST_F(WddmMemoryOperationsHandlerWithAubDumpTest, givenRegularAllocationWhenMak TEST_F(WddmMemoryOperationsHandlerWithAubDumpTest, givenFragmentedAllocationWhenMakingResidentAllocationThenMakeResidentCalled) { allocationPtr = wddmFragmentedAllocation.get(); - EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->isResident(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success); EXPECT_TRUE(mockAubMemoryOperationsHandler->makeResidentCalled); EXPECT_TRUE(mockAubMemoryOperationsHandler->isResidentCalled); } TEST_F(WddmMemoryOperationsHandlerWithAubDumpTest, givenVariousAllocationsWhenMakingResidentAllocationThenMakeResidentCalled) { - EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(allocationData), false), MemoryOperationsStatus::success); + EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(allocationData), false, false), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->isResident(nullptr, *wddmAllocation), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->isResident(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success); EXPECT_TRUE(mockAubMemoryOperationsHandler->makeResidentCalled); @@ -90,7 +90,7 @@ TEST_F(WddmMemoryOperationsHandlerWithAubDumpTest, givenVariousAllocationsWhenMa TEST_F(WddmMemoryOperationsHandlerWithAubDumpTest, givenRegularAllocationWhenEvictingResidentAllocationThenEvictCalled) { wddm->callBaseEvict = true; - EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->evict(nullptr, *wddmAllocation), MemoryOperationsStatus::success); EXPECT_EQ(0u, gdi->getEvictArg().Flags.EvictOnlyIfNecessary); EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->isResident(nullptr, *wddmAllocation), MemoryOperationsStatus::memoryNotFound); @@ -108,7 +108,7 @@ TEST_F(WddmMemoryOperationsHandlerWithAubDumpTest, givenRegularAllocationWhenFre device->executionEnvironment->rootDeviceEnvironments[0]->setHwInfoAndInitHelpers(defaultHwInfo.get()); allocationPtr->setAubWritable(true, 0xff); - EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->makeResident(device.get(), ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->makeResident(device.get(), ArrayRef(&allocationPtr, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(1u, mockAubMemoryOperationsHandler->residentAllocations.size()); EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->free(device.get(), *wddmAllocation), MemoryOperationsStatus::success); @@ -120,7 +120,7 @@ TEST_F(WddmMemoryOperationsHandlerWithAubDumpTest, givenRegularAllocationWhenFre TEST_F(WddmMemoryOperationsHandlerWithAubDumpTest, givenFragmentedAllocationWhenEvictingResidentAllocationThenEvictCalled) { allocationPtr = wddmFragmentedAllocation.get(); - EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false, false), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->evict(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->isResident(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::memoryNotFound); EXPECT_TRUE(mockAubMemoryOperationsHandler->makeResidentCalled); @@ -128,7 +128,7 @@ TEST_F(WddmMemoryOperationsHandlerWithAubDumpTest, givenFragmentedAllocationWhen } TEST_F(WddmMemoryOperationsHandlerWithAubDumpTest, givenVariousAllocationsWhenEvictingResidentAllocationThenEvictCalled) { - EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(allocationData), false), MemoryOperationsStatus::success); + EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->makeResident(nullptr, ArrayRef(allocationData), false, false), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->evict(nullptr, *wddmAllocation), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->isResident(nullptr, *wddmAllocation), MemoryOperationsStatus::memoryNotFound); EXPECT_EQ(wddmMemoryOperationsHandlerWithAubDumpMock->evict(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success);