From d9624a270c288bae66d2f029fb56eff13c39f9bb Mon Sep 17 00:00:00 2001 From: Lukasz Jobczyk Date: Tue, 21 Oct 2025 12:00:55 +0000 Subject: [PATCH] refactor: Move residency controller to wddm Resolves: NEO-13315 Signed-off-by: Lukasz Jobczyk --- shared/source/os_interface/device_factory.cpp | 6 + .../os_interface/linux/os_interface_linux.cpp | 2 + shared/source/os_interface/os_interface.h | 1 + .../os_interface/windows/os_context_win.cpp | 11 +- .../os_interface/windows/os_context_win.h | 4 +- .../os_interface/windows/os_interface_win.cpp | 7 + .../os_interface/windows/trim_callback.cpp | 210 +++++++++++------- .../windows/trim_callback_stub.cpp | 2 +- .../source/os_interface/windows/wddm/wddm.cpp | 9 +- .../source/os_interface/windows/wddm/wddm.h | 17 +- .../windows/wddm_device_command_stream.inl | 3 +- .../windows/wddm_residency_controller.cpp | 13 +- .../windows/wddm_residency_controller.h | 9 +- .../test/common/mocks/mock_os_context_win.h | 1 + shared/test/common/mocks/mock_wddm.cpp | 4 +- shared/test/common/mocks/mock_wddm.h | 2 +- .../mocks/mock_wddm_residency_controller.h | 1 - .../windows/os_context_win_tests.cpp | 9 +- .../windows/os_interface_win_tests.cpp | 4 +- .../os_interface/windows/wddm20_tests.cpp | 3 +- .../wddm_residency_controller_tests.cpp | 92 +++----- .../windows/wddm_windows_tests.cpp | 3 +- 22 files changed, 229 insertions(+), 184 deletions(-) diff --git a/shared/source/os_interface/device_factory.cpp b/shared/source/os_interface/device_factory.cpp index cc5fee1b03..54deec7261 100644 --- a/shared/source/os_interface/device_factory.cpp +++ b/shared/source/os_interface/device_factory.cpp @@ -330,6 +330,12 @@ std::vector> DeviceFactory::createDevices(ExecutionEnvir } } + for (auto &rootDeviceEnv : executionEnvironment.rootDeviceEnvironments) { + if (rootDeviceEnv->osInterface) { + rootDeviceEnv->osInterface->registerTrimCallback(); + } + } + return devices; } diff --git a/shared/source/os_interface/linux/os_interface_linux.cpp b/shared/source/os_interface/linux/os_interface_linux.cpp index a64d8c8266..6ebabcf559 100644 --- a/shared/source/os_interface/linux/os_interface_linux.cpp +++ b/shared/source/os_interface/linux/os_interface_linux.cpp @@ -73,4 +73,6 @@ uint32_t OSInterface::getAggregatedProcessCount() const { return 0; } +void OSInterface::registerTrimCallback() {} + } // namespace NEO diff --git a/shared/source/os_interface/os_interface.h b/shared/source/os_interface/os_interface.h index 004077b9ec..ef55dbf67b 100644 --- a/shared/source/os_interface/os_interface.h +++ b/shared/source/os_interface/os_interface.h @@ -122,6 +122,7 @@ class OSInterface : public NonCopyableClass { MOCKABLE_VIRTUAL bool isLockablePointer(bool isLockable) const; MOCKABLE_VIRTUAL bool isSizeWithinThresholdForStaging(const void *ptr, size_t size) const; MOCKABLE_VIRTUAL uint32_t getAggregatedProcessCount() const; + void registerTrimCallback(); static bool osEnabled64kbPages; static bool are64kbPagesEnabled(); diff --git a/shared/source/os_interface/windows/os_context_win.cpp b/shared/source/os_interface/windows/os_context_win.cpp index 4158c90b68..4152818b76 100644 --- a/shared/source/os_interface/windows/os_context_win.cpp +++ b/shared/source/os_interface/windows/os_context_win.cpp @@ -24,11 +24,7 @@ OsContext *OsContextWin::create(OSInterface *osInterface, uint32_t rootDeviceInd return new OsContext(rootDeviceIndex, contextId, engineDescriptor); } -OsContextWin::OsContextWin(Wddm &wddm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor) - : OsContext(rootDeviceIndex, contextId, engineDescriptor), - residencyController(wddm), - wddm(wddm) { -} +OsContextWin::OsContextWin(Wddm &wddm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor) : OsContext(rootDeviceIndex, contextId, engineDescriptor), wddm(wddm) {} bool OsContextWin::initializeContext(bool allocateInterrupt) { @@ -45,9 +41,6 @@ bool OsContextWin::initializeContext(bool allocateInterrupt) { UNRECOVERABLE_IF(!wddmInterface->createHwQueue(*this)); } UNRECOVERABLE_IF(!wddmInterface->createMonitoredFence(*this)); - - residencyController.registerCallback(); - UNRECOVERABLE_IF(!residencyController.isInitialized()); } return true; @@ -101,6 +94,8 @@ uint32_t OsContextWin::getDeviceNodeMask() { return hwDeviceID->getAdapterNodeMask(); } +WddmResidencyController &OsContextWin::getResidencyController() { return wddm.getResidencyController(); } + uint64_t OsContextWin::getOfflineDumpContextId(uint32_t deviceIndex) const { return 0; } diff --git a/shared/source/os_interface/windows/os_context_win.h b/shared/source/os_interface/windows/os_context_win.h index 2884ff5102..11f0ddefae 100644 --- a/shared/source/os_interface/windows/os_context_win.h +++ b/shared/source/os_interface/windows/os_context_win.h @@ -33,7 +33,7 @@ class OsContextWin : public OsContext { void setHwQueue(HardwareQueue hardwareQueue) { this->hardwareQueue = hardwareQueue; } bool isDirectSubmissionSupported() const override; Wddm *getWddm() const { return &wddm; } - MOCKABLE_VIRTUAL WddmResidencyController &getResidencyController() { return residencyController; } + MOCKABLE_VIRTUAL WddmResidencyController &getResidencyController(); static OsContext *create(OSInterface *osInterface, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor); MonitoredFence &getMonitoredFence() { return monitoredFence; } void resetMonitoredFenceParams(D3DKMT_HANDLE &handle, uint64_t *cpuAddress, D3DGPU_VIRTUAL_ADDRESS &gpuAddress); @@ -48,8 +48,6 @@ class OsContextWin : public OsContext { protected: bool initializeContext(bool allocateInterrupt) override; - WddmResidencyController residencyController; - HardwareQueue hardwareQueue; MonitoredFence monitoredFence = {}; diff --git a/shared/source/os_interface/windows/os_interface_win.cpp b/shared/source/os_interface/windows/os_interface_win.cpp index 7d2a31e903..d6953ca439 100644 --- a/shared/source/os_interface/windows/os_interface_win.cpp +++ b/shared/source/os_interface/windows/os_interface_win.cpp @@ -34,4 +34,11 @@ uint32_t OSInterface::getAggregatedProcessCount() const { return 0; } +void OSInterface::registerTrimCallback() { + if (driverModel && driverModel->getDriverModelType() == DriverModelType::wddm) { + driverModel->as()->getResidencyController().registerCallback(); + UNRECOVERABLE_IF(!driverModel->as()->getResidencyController().isInitialized()); + } +} + } // namespace NEO diff --git a/shared/source/os_interface/windows/trim_callback.cpp b/shared/source/os_interface/windows/trim_callback.cpp index e112617249..c436defdc5 100644 --- a/shared/source/os_interface/windows/trim_callback.cpp +++ b/shared/source/os_interface/windows/trim_callback.cpp @@ -15,7 +15,11 @@ namespace NEO { -VOID *Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController) { +VOID *Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback) { + if (!hwDeviceId || !this->rootDeviceEnvironment.executionEnvironment.osEnvironment || !getGdi()) { + return nullptr; + } + if (debugManager.flags.DoNotRegisterTrimCallback.get()) { return nullptr; } @@ -33,6 +37,10 @@ VOID *Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, Wd } void Wddm::unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, VOID *trimCallbackHandle) { + if (!hwDeviceId || !this->rootDeviceEnvironment.executionEnvironment.osEnvironment || !getGdi()) { + return; + } + DEBUG_BREAK_IF(callback == nullptr); if (trimCallbackHandle == nullptr || isShutdownInProgress()) { return; @@ -48,7 +56,6 @@ void Wddm::unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, V void APIENTRY WddmResidencyController::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification) { auto residencyController = static_cast(trimNotification->Context); DEBUG_BREAK_IF(residencyController == nullptr); - UNRECOVERABLE_IF(residencyController->csr == nullptr); auto lock = residencyController->acquireTrimCallbackLock(); residencyController->trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim); @@ -57,74 +64,94 @@ void APIENTRY WddmResidencyController::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICAT void WddmResidencyController::trimResidency(const D3DDDI_TRIMRESIDENCYSET_FLAGS &flags, uint64_t bytes) { std::chrono::high_resolution_clock::time_point callbackStart; perfLogResidencyTrimCallbackBegin(wddm.getResidencyLogger(), callbackStart); - auto &osContext = static_cast(this->csr->getOsContext()); - uint32_t osContextId = osContext.getContextId(); if (flags.PeriodicTrim) { uint64_t sizeToTrim = 0; auto lock = this->acquireLock(); WddmAllocation *wddmAllocation = nullptr; std::vector handlesToEvict; + std::vector allocationsToRemove; + allocationsToRemove.reserve(evictionAllocations.size()); handlesToEvict.reserve(evictionAllocations.size()); - for (auto allocationIter = evictionAllocations.begin(); allocationIter != evictionAllocations.end();) { - wddmAllocation = reinterpret_cast(*allocationIter); - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", osContext.getLastTrimFenceValue()); + this->wddm.forEachContextWithinWddm([&](const EngineControl &engine) { + for (auto allocationIter = evictionAllocations.begin(); allocationIter != evictionAllocations.end(); ++allocationIter) { + auto &osContext = static_cast(*engine.osContext); + uint32_t osContextId = osContext.getContextId(); - if (osContext.wasAllocationUsedSinceLastTrim(wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId))) { - allocationIter++; - continue; - } + wddmAllocation = reinterpret_cast(*allocationIter); - if (wddmAllocation->isAlwaysResident(osContextId)) { - allocationIter = evictionAllocations.erase(allocationIter); - continue; - } + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", osContext.getLastTrimFenceValue()); - if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { - for (auto i = 0u; i < wddmAllocation->getNumGmms(); i++) { - handlesToEvict.push_back(wddmAllocation->getHandles()[i]); + if (osContext.wasAllocationUsedSinceLastTrim(wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId))) { + continue; + } + + if (wddmAllocation->isAlwaysResident(osContextId)) { + continue; + } + + if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { + if (!wddmAllocation->getResidencyData().resident[osContextId]) { + continue; + } + for (auto i = 0u; i < wddmAllocation->getNumGmms(); i++) { + handlesToEvict.push_back(wddmAllocation->getHandles()[i]); + } + } else { + for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { + AllocationStorageData &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId]; + if (!osContext.wasAllocationUsedSinceLastTrim(fragmentStorageData.residency->getFenceValueForContextId(osContextId))) { + auto osHandle = static_cast(wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage); + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", osHandle->handle, "lastFence =", + wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId)); + + handlesToEvict.push_back(static_cast(fragmentStorageData.osHandleStorage)->handle); + fragmentStorageData.residency->resident[osContextId] = false; + } + } + } + + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation, gpu address = ", std::hex, wddmAllocation->getGpuAddress(), "lastFence =", wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId)); + wddmAllocation->getResidencyData().resident[osContextId] = false; + + if (std::none_of(wddmAllocation->getResidencyData().resident.begin(), wddmAllocation->getResidencyData().resident.end(), [](bool resident) { return resident; })) { + allocationsToRemove.push_back(wddmAllocation); } } - for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { - AllocationStorageData &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId]; - if (!osContext.wasAllocationUsedSinceLastTrim(fragmentStorageData.residency->getFenceValueForContextId(osContextId))) { - auto osHandle = static_cast(wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage); - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", osHandle->handle, "lastFence =", - wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId)); + this->wddm.evict(handlesToEvict.data(), static_cast(handlesToEvict.size()), sizeToTrim, false); + handlesToEvict.clear(); + }); - handlesToEvict.push_back(static_cast(fragmentStorageData.osHandleStorage)->handle); - fragmentStorageData.residency->resident[osContextId] = false; - } - } - - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation, gpu address = ", std::hex, wddmAllocation->getGpuAddress(), "lastFence =", wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId)); - wddmAllocation->getResidencyData().resident[osContextId] = false; - allocationIter = evictionAllocations.erase(allocationIter); - } - - this->wddm.evict(handlesToEvict.data(), static_cast(handlesToEvict.size()), sizeToTrim, false); + std::erase_if(evictionAllocations, [&allocationsToRemove](GraphicsAllocation *graphicsAllocation) { + return std::find_if(allocationsToRemove.begin(), allocationsToRemove.end(), [&graphicsAllocation](WddmAllocation *graphicsAllocationToRemove) { + return graphicsAllocationToRemove == graphicsAllocation; + }) != allocationsToRemove.end(); + }); } if (flags.TrimToBudget) { - auto csrLock = csr->obtainUniqueOwnership(); + StackVec, 5> csrLocks; + this->wddm.forEachContextWithinWddm([&](const EngineControl &engine) { + csrLocks.push_back(engine.commandStreamReceiver->obtainUniqueOwnership()); + }); auto lock = this->acquireLock(); trimResidencyToBudget(bytes); } if (flags.PeriodicTrim || flags.RestartPeriodicTrim) { - osContext.updateLastTrimFenceValue(); - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", osContext.getLastTrimFenceValue()); + this->wddm.forEachContextWithinWddm([&](const EngineControl &engine) { + auto osContext = static_cast(engine.osContext); + osContext->updateLastTrimFenceValue(); + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", osContext->getLastTrimFenceValue()); + }); } perfLogResidencyTrimCallbackEnd(wddm.getResidencyLogger(), flags.Value, this, callbackStart); } bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes) { - this->csr->drainPagingFenceQueue(); - auto &osContext = static_cast(this->csr->getOsContext()); - uint32_t osContextId = osContext.getContextId(); uint64_t sizeToTrim = 0; uint64_t numberOfBytesToTrim = bytes; WddmAllocation *wddmAllocation = nullptr; @@ -133,55 +160,78 @@ bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes) { auto allocationIter = evictionAllocations.begin(); std::vector handlesToEvict; handlesToEvict.reserve(evictionAllocations.size()); - while (numberOfBytesToTrim > 0 && allocationIter != evictionAllocations.end()) { - wddmAllocation = reinterpret_cast(*allocationIter); - uint64_t lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId); - auto &monitoredFence = osContext.getMonitoredFence(); + std::vector allocationsToRemove; + allocationsToRemove.reserve(evictionAllocations.size()); - if (lastFence > monitoredFence.lastSubmittedFence) { - allocationIter++; - continue; - } + this->wddm.forEachContextWithinWddm([&](const EngineControl &engine) { + auto csr = engine.commandStreamReceiver; + csr->drainPagingFenceQueue(); + auto &osContext = static_cast(csr->getOsContext()); + uint32_t osContextId = osContext.getContextId(); - if (wddmAllocation->isAlwaysResident(osContextId)) { - allocationIter = evictionAllocations.erase(allocationIter); - continue; - } + while (numberOfBytesToTrim > 0 && allocationIter != evictionAllocations.end()) { + wddmAllocation = reinterpret_cast(*allocationIter); + uint64_t lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId); + auto &monitoredFence = osContext.getMonitoredFence(); - uint64_t sizeEvicted = 0; - - if (lastFence > *monitoredFence.cpuAddress) { - this->wddm.waitFromCpu(lastFence, osContext.getMonitoredFence(), false); - } - - if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { - for (auto i = 0u; i < wddmAllocation->getNumGmms(); i++) { - handlesToEvict.push_back(wddmAllocation->getHandles()[i]); + if (lastFence > monitoredFence.lastSubmittedFence) { + ++allocationIter; + continue; } - sizeEvicted = wddmAllocation->getAlignedSize(); - } else { - auto &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData; - for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { - if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) { - handlesToEvict.push_back(static_cast(fragmentStorageData[allocationId].osHandleStorage)->handle); - fragmentStorageData[allocationId].residency->resident[osContextId] = false; - sizeEvicted += fragmentStorageData[allocationId].fragmentSize; + + if (wddmAllocation->isAlwaysResident(osContextId)) { + ++allocationIter; + continue; + } + + uint64_t sizeEvicted = 0; + + if (lastFence > *monitoredFence.cpuAddress) { + this->wddm.waitFromCpu(lastFence, osContext.getMonitoredFence(), false); + } + + if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { + for (auto i = 0u; i < wddmAllocation->getNumGmms(); i++) { + handlesToEvict.push_back(wddmAllocation->getHandles()[i]); + } + sizeEvicted = wddmAllocation->getAlignedSize(); + } else { + auto &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData; + for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { + if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) { + handlesToEvict.push_back(static_cast(fragmentStorageData[allocationId].osHandleStorage)->handle); + fragmentStorageData[allocationId].residency->resident[osContextId] = false; + sizeEvicted += fragmentStorageData[allocationId].fragmentSize; + } } } - } - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation, gpu address = ", std::hex, wddmAllocation->getGpuAddress(), "lastFence =", wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId)); + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation, gpu address = ", std::hex, wddmAllocation->getGpuAddress(), "lastFence =", wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId)); - if (sizeEvicted >= numberOfBytesToTrim) { - numberOfBytesToTrim = 0; - } else { - numberOfBytesToTrim -= sizeEvicted; + if (sizeEvicted >= numberOfBytesToTrim) { + numberOfBytesToTrim = 0; + } else { + numberOfBytesToTrim -= sizeEvicted; + } + + wddmAllocation->getResidencyData().resident[osContextId] = false; + + if (std::none_of(wddmAllocation->getResidencyData().resident.begin(), wddmAllocation->getResidencyData().resident.end(), [](bool resident) { return resident; })) { + allocationsToRemove.push_back(wddmAllocation); + } + + ++allocationIter; } - wddmAllocation->getResidencyData().resident[osContextId] = false; - allocationIter = evictionAllocations.erase(allocationIter); - } + this->wddm.evict(handlesToEvict.data(), static_cast(handlesToEvict.size()), sizeToTrim, true); + handlesToEvict.clear(); + }); + + std::erase_if(evictionAllocations, [&allocationsToRemove](GraphicsAllocation *graphicsAllocation) { + return std::find_if(allocationsToRemove.begin(), allocationsToRemove.end(), [&graphicsAllocation](WddmAllocation *graphicsAllocationToRemove) { + return graphicsAllocationToRemove == graphicsAllocation; + }) != allocationsToRemove.end(); + }); - this->wddm.evict(handlesToEvict.data(), static_cast(handlesToEvict.size()), sizeToTrim, true); return numberOfBytesToTrim == 0; } diff --git a/shared/source/os_interface/windows/trim_callback_stub.cpp b/shared/source/os_interface/windows/trim_callback_stub.cpp index 13c52b68fc..8189adbfa2 100644 --- a/shared/source/os_interface/windows/trim_callback_stub.cpp +++ b/shared/source/os_interface/windows/trim_callback_stub.cpp @@ -11,7 +11,7 @@ namespace NEO { -VOID *Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController) { +VOID *Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback) { return nullptr; } diff --git a/shared/source/os_interface/windows/wddm/wddm.cpp b/shared/source/os_interface/windows/wddm/wddm.cpp index d3437434ff..52c41d4c51 100644 --- a/shared/source/os_interface/windows/wddm/wddm.cpp +++ b/shared/source/os_interface/windows/wddm/wddm.cpp @@ -54,7 +54,7 @@ Wddm::CreateDXGIFactoryFcn Wddm::createDxgiFactory = getCreateDxgiFactory(); Wddm::GetSystemInfoFcn Wddm::getSystemInfo = getGetSystemInfo(); Wddm::Wddm(std::unique_ptr &&hwDeviceIdIn, RootDeviceEnvironment &rootDeviceEnvironment) - : DriverModel(DriverModelType::wddm), hwDeviceId(std::move(hwDeviceIdIn)), rootDeviceEnvironment(rootDeviceEnvironment) { + : DriverModel(DriverModelType::wddm), residencyController(*this), hwDeviceId(std::move(hwDeviceIdIn)), rootDeviceEnvironment(rootDeviceEnvironment) { UNRECOVERABLE_IF(!hwDeviceId); featureTable.reset(new FeatureTable()); workaroundTable.reset(new WorkaroundTable()); @@ -76,6 +76,7 @@ Wddm::Wddm(std::unique_ptr &&hwDeviceIdIn, RootDeviceEnvironment } Wddm::~Wddm() { + this->residencyController.unregisterCallback(); temporaryResources.reset(); destroyPagingQueue(); destroyDevice(); @@ -610,7 +611,7 @@ bool Wddm::mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_A bool ret = true; auto &productHelper = rootDeviceEnvironment.getHelper(); if (gmm->isCompressionEnabled() && productHelper.isPageTableManagerSupported(*rootDeviceEnvironment.getHardwareInfo())) { - this->forEachContextWithinWddm([&](const EngineControl &engine) { + this->forEachContextWithinWddm([&](const EngineControl &engine) { if (engine.commandStreamReceiver->pageTableManager.get()) { ret &= engine.commandStreamReceiver->pageTableManager->updateAuxTable(gpuPtr, gmm, true); } @@ -1196,7 +1197,7 @@ bool Wddm::waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredF if (!skipResourceCleanup() && lastFenceValue > *monitoredFence.cpuAddress) { CommandStreamReceiver *csr = nullptr; - this->forEachContextWithinWddm([&monitoredFence, &csr](const EngineControl &engine) { + this->forEachContextWithinWddm([&monitoredFence, &csr](const EngineControl &engine) { auto &contextMonitoredFence = static_cast(engine.osContext)->getMonitoredFence(); if (contextMonitoredFence.cpuAddress == monitoredFence.cpuAddress) { csr = engine.commandStreamReceiver; @@ -1439,7 +1440,7 @@ void Wddm::setNewResourceBoundToPageTable() { if (!this->rootDeviceEnvironment.getProductHelper().isTlbFlushRequired()) { return; } - this->forEachContextWithinWddm([](const EngineControl &engine) { engine.osContext->setNewResourceBound(); }); + this->forEachContextWithinWddm([](const EngineControl &engine) { engine.osContext->setNewResourceBound(); }); } bool Wddm::needsNotifyAubCaptureCallback() const { diff --git a/shared/source/os_interface/windows/wddm/wddm.h b/shared/source/os_interface/windows/wddm/wddm.h index a062e23115..b05fc8c7df 100644 --- a/shared/source/os_interface/windows/wddm/wddm.h +++ b/shared/source/os_interface/windows/wddm/wddm.h @@ -13,6 +13,7 @@ #include "shared/source/os_interface/windows/sharedata_wrapper.h" #include "shared/source/os_interface/windows/wddm/wddm_defs.h" #include "shared/source/os_interface/windows/wddm_memory_manager.h" +#include "shared/source/os_interface/windows/wddm_residency_controller.h" #include @@ -30,7 +31,6 @@ class ProductHelper; class SettingsReader; class WddmAllocation; class WddmInterface; -class WddmResidencyController; class WddmResidencyLogger; class WddmResidentAllocationsContainer; @@ -95,7 +95,8 @@ class Wddm : public DriverModel { MOCKABLE_VIRTUAL bool waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredFence, bool busyWait); MOCKABLE_VIRTUAL NTSTATUS escape(D3DKMT_ESCAPE &escapeCommand); - MOCKABLE_VIRTUAL VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController); + WddmResidencyController &getResidencyController() { return residencyController; } + MOCKABLE_VIRTUAL VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback); void unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, VOID *trimCallbackHandle); MOCKABLE_VIRTUAL void releaseReservedAddress(void *reservedAddress); MOCKABLE_VIRTUAL bool reserveValidAddressRange(size_t size, void *&reservedMem); @@ -217,12 +218,18 @@ class Wddm : public DriverModel { return additionalAdapterInfoOptions; } - template + template void forEachContextWithinWddm(F func) { for (auto rootDeviceIndex = 0u; rootDeviceIndex < rootDeviceEnvironment.executionEnvironment.rootDeviceEnvironments.size(); rootDeviceIndex++) { if (rootDeviceEnvironment.executionEnvironment.rootDeviceEnvironments[rootDeviceIndex].get() == &rootDeviceEnvironment) { for (auto &engine : rootDeviceEnvironment.executionEnvironment.memoryManager->getRegisteredEngines(rootDeviceIndex)) { - func(engine); + if constexpr (onlyInitialized) { + if (engine.osContext->isInitialized()) { + func(engine); + } + } else { + func(engine); + } } } } @@ -262,6 +269,8 @@ class Wddm : public DriverModel { bool isReadOnlyFlagFallbackSupported() const; bool isReadOnlyFlagFallbackAvailable(const D3DKMT_CREATEALLOCATION &createAllocation) const; + WddmResidencyController residencyController; + GMM_GFX_PARTITIONING gfxPartition{}; ADAPTER_BDF adapterBDF{}; diff --git a/shared/source/os_interface/windows/wddm_device_command_stream.inl b/shared/source/os_interface/windows/wddm_device_command_stream.inl index 4eb298e3f3..17cfefb37e 100644 --- a/shared/source/os_interface/windows/wddm_device_command_stream.inl +++ b/shared/source/os_interface/windows/wddm_device_command_stream.inl @@ -147,7 +147,7 @@ SubmissionStatus WddmCommandStreamReceiver::flush(BatchBuffer &batchB template SubmissionStatus WddmCommandStreamReceiver::processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId) { - return static_cast(this->osContext)->getResidencyController().makeResidentResidencyAllocations(allocationsForResidency, this->requiresBlockingResidencyHandling, *static_cast(this->osContext)) ? SubmissionStatus::success : SubmissionStatus::outOfMemory; + return static_cast(this->osContext)->getResidencyController().makeResidentResidencyAllocations(allocationsForResidency, this->requiresBlockingResidencyHandling, this) ? SubmissionStatus::success : SubmissionStatus::outOfMemory; } template @@ -212,7 +212,6 @@ CommandStreamReceiver *createWddmDeviceCommandStreamReceiver(bool withAubDump, template void WddmCommandStreamReceiver::setupContext(OsContext &osContext) { this->osContext = &osContext; - static_cast(this->osContext)->getResidencyController().setCommandStreamReceiver(this); } template diff --git a/shared/source/os_interface/windows/wddm_residency_controller.cpp b/shared/source/os_interface/windows/wddm_residency_controller.cpp index 1c506debeb..ee9acf187e 100644 --- a/shared/source/os_interface/windows/wddm_residency_controller.cpp +++ b/shared/source/os_interface/windows/wddm_residency_controller.cpp @@ -22,10 +22,10 @@ WddmResidencyController::WddmResidencyController(Wddm &wddm) : wddm(wddm) { } void WddmResidencyController::registerCallback() { - this->trimCallbackHandle = wddm.registerTrimCallback(WddmResidencyController::trimCallback, *this); + this->trimCallbackHandle = wddm.registerTrimCallback(WddmResidencyController::trimCallback); } -WddmResidencyController::~WddmResidencyController() { +void WddmResidencyController::unregisterCallback() { auto lock = this->acquireTrimCallbackLock(); wddm.unregisterTrimCallback(WddmResidencyController::trimCallback, this->trimCallbackHandle); lock.unlock(); @@ -54,7 +54,8 @@ std::unique_lock WddmResidencyController::acquireTrimCallbackLock() { * * @return returns true if all allocations either succeeded or are pending to be resident */ -bool WddmResidencyController::makeResidentResidencyAllocations(ResidencyContainer &allocationsForResidency, bool &requiresBlockingResidencyHandling, OsContextWin &osContext) { +bool WddmResidencyController::makeResidentResidencyAllocations(ResidencyContainer &allocationsForResidency, bool &requiresBlockingResidencyHandling, CommandStreamReceiver *csr) { + auto &osContext = static_cast(csr->getOsContext()); auto osContextId = osContext.getContextId(); const size_t residencyCount = allocationsForResidency.size(); requiresBlockingResidencyHandling = false; @@ -159,17 +160,13 @@ size_t WddmResidencyController::fillHandlesContainer(ResidencyContainer &allocat bool WddmResidencyController::isInitialized() const { bool requiresTrimCallbacks = OSInterface::requiresSupportForWddmTrimNotification; - requiresTrimCallbacks = requiresTrimCallbacks && (false == debugManager.flags.DoNotRegisterTrimCallback.get()); + requiresTrimCallbacks = requiresTrimCallbacks && (false == debugManager.flags.DoNotRegisterTrimCallback.get()) && wddm.getHwDeviceId() && wddm.getRootDeviceEnvironment().executionEnvironment.osEnvironment && wddm.getGdi(); if (requiresTrimCallbacks) { return trimCallbackHandle != nullptr; } return true; } -void WddmResidencyController::setCommandStreamReceiver(CommandStreamReceiver *csr) { - this->csr = csr; -} - void WddmResidencyController::removeAllocation(ResidencyContainer &container, GraphicsAllocation *gfxAllocation) { std::unique_lock lock1(this->lock, std::defer_lock); std::unique_lock lock2(this->trimCallbackLock, std::defer_lock); diff --git a/shared/source/os_interface/windows/wddm_residency_controller.h b/shared/source/os_interface/windows/wddm_residency_controller.h index 397e645453..e52ffa22cf 100644 --- a/shared/source/os_interface/windows/wddm_residency_controller.h +++ b/shared/source/os_interface/windows/wddm_residency_controller.h @@ -30,7 +30,7 @@ class OsContextWin; class WddmResidencyController { public: WddmResidencyController(Wddm &wddm); - MOCKABLE_VIRTUAL ~WddmResidencyController(); + MOCKABLE_VIRTUAL ~WddmResidencyController() = default; static void APIENTRY trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification); @@ -38,6 +38,7 @@ class WddmResidencyController { [[nodiscard]] std::unique_lock acquireTrimCallbackLock(); void registerCallback(); + void unregisterCallback(); void trimResidency(const D3DDDI_TRIMRESIDENCYSET_FLAGS &flags, uint64_t bytes); bool trimResidencyToBudget(uint64_t bytes); @@ -45,12 +46,10 @@ class WddmResidencyController { bool isMemoryBudgetExhausted() const { return memoryBudgetExhausted; } void setMemoryBudgetExhausted() { memoryBudgetExhausted = true; } - bool makeResidentResidencyAllocations(ResidencyContainer &allocationsForResidency, bool &requiresBlockingResidencyHandling, OsContextWin &osContext); + bool makeResidentResidencyAllocations(ResidencyContainer &allocationsForResidency, bool &requiresBlockingResidencyHandling, CommandStreamReceiver *csr); bool isInitialized() const; - void setCommandStreamReceiver(CommandStreamReceiver *csr); - void removeAllocation(ResidencyContainer &container, GraphicsAllocation *gfxAllocation); ResidencyContainer &getEvictionAllocations() { @@ -68,8 +67,6 @@ class WddmResidencyController { bool memoryBudgetExhausted = false; - CommandStreamReceiver *csr = nullptr; - ResidencyContainer evictionAllocations; ResidencyContainer backupResidencyContainer; // Stores allocations which should be resident diff --git a/shared/test/common/mocks/mock_os_context_win.h b/shared/test/common/mocks/mock_os_context_win.h index f0086e4001..eb72670a44 100644 --- a/shared/test/common/mocks/mock_os_context_win.h +++ b/shared/test/common/mocks/mock_os_context_win.h @@ -12,6 +12,7 @@ namespace NEO { class MockOsContextWin : public OsContextWin { public: + using OsContext::contextInitialized; using OsContextWin::lastTrimFenceValue; MockOsContextWin(Wddm &wddm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor) diff --git a/shared/test/common/mocks/mock_wddm.cpp b/shared/test/common/mocks/mock_wddm.cpp index 403ea6d3aa..a2a83e8d5a 100644 --- a/shared/test/common/mocks/mock_wddm.cpp +++ b/shared/test/common/mocks/mock_wddm.cpp @@ -316,9 +316,9 @@ bool WddmMock::reserveValidAddressRange(size_t size, void *&reservedMem) { } return ret; } -VOID *WddmMock::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController) { +VOID *WddmMock::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback) { registerTrimCallbackResult.called++; - return Wddm::registerTrimCallback(callback, residencyController); + return Wddm::registerTrimCallback(callback); } NTSTATUS WddmMock::reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS baseAddress, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size, D3DGPU_VIRTUAL_ADDRESS *reservedAddress) { diff --git a/shared/test/common/mocks/mock_wddm.h b/shared/test/common/mocks/mock_wddm.h index 8e9e3b7d6d..4a00b43c39 100644 --- a/shared/test/common/mocks/mock_wddm.h +++ b/shared/test/common/mocks/mock_wddm.h @@ -105,7 +105,7 @@ class WddmMock : public Wddm { void *virtualAlloc(void *inPtr, size_t size, bool topDownHint) override; void virtualFree(void *ptr, size_t size) override; void releaseReservedAddress(void *reservedAddress) override; - VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController) override; + VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback) override; NTSTATUS reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS baseAddress, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size, D3DGPU_VIRTUAL_ADDRESS *reservedAddress) override; bool reserveValidAddressRange(size_t size, void *&reservedMem) override; PLATFORM *getGfxPlatform() { return gfxPlatform.get(); } diff --git a/shared/test/common/mocks/mock_wddm_residency_controller.h b/shared/test/common/mocks/mock_wddm_residency_controller.h index be8b356f9f..68ab163b16 100644 --- a/shared/test/common/mocks/mock_wddm_residency_controller.h +++ b/shared/test/common/mocks/mock_wddm_residency_controller.h @@ -11,7 +11,6 @@ namespace NEO { class MockWddmResidencyController : public WddmResidencyController { public: - using WddmResidencyController::csr; using WddmResidencyController::lock; using WddmResidencyController::trimCallbackHandle; using WddmResidencyController::trimResidency; diff --git a/shared/test/unit_test/os_interface/windows/os_context_win_tests.cpp b/shared/test/unit_test/os_interface/windows/os_context_win_tests.cpp index a8fb58878a..79ec6eadc6 100644 --- a/shared/test/unit_test/os_interface/windows/os_context_win_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/os_context_win_tests.cpp @@ -44,10 +44,13 @@ TEST_F(OsContextWinTest, givenWddm20WhenCreatingWddmMonitorFenceFailThenOsContex EXPECT_ANY_THROW(osContext->ensureContextInitialized(false)); } -TEST_F(OsContextWinTest, givenWddm20WhenRegisterTrimCallbackFailThenOsContextCreationFails) { +TEST_F(OsContextWinTest, givenWddm20WhenRegisterTrimCallbackFailThenResidencyControllerIsNotInitialized) { *getRegisterTrimNotificationFailCallFcn() = true; - osContext = std::make_unique(*osInterface->getDriverModel()->as(), 0, 0u, EngineDescriptorHelper::getDefaultDescriptor(engineTypeUsage, preemptionMode)); - EXPECT_ANY_THROW(osContext->ensureContextInitialized(false)); + WddmMock wddm(*rootDeviceEnvironment); + wddm.init(); + + wddm.getResidencyController().registerCallback(); + EXPECT_FALSE(wddm.getResidencyController().isInitialized()); } TEST_F(OsContextWinTest, givenWddm20WhenRegisterTrimCallbackIsDisabledThenOsContextIsInitialized) { diff --git a/shared/test/unit_test/os_interface/windows/os_interface_win_tests.cpp b/shared/test/unit_test/os_interface/windows/os_interface_win_tests.cpp index c9497ef1f1..91c053bdc7 100644 --- a/shared/test/unit_test/os_interface/windows/os_interface_win_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/os_interface_win_tests.cpp @@ -36,7 +36,7 @@ TEST_F(OsInterfaceTest, GivenWindowsWhenCreateEentIsCalledThenValidEventHandleIs EXPECT_EQ(TRUE, ret); } -TEST(OsContextTest, givenWddmWhenCreateOsContextAfterInitWddmThenOsContextIsInitializedTrimCallbackIsRegisteredMemoryOperationsHandlerCreated) { +TEST(OsContextTest, givenWddmWhenCreateOsContextAfterInitWddmThenOsContextIsInitializedTrimCallbackIsNotRegisteredMemoryOperationsHandlerCreated) { MockExecutionEnvironment executionEnvironment; RootDeviceEnvironment rootDeviceEnvironment(executionEnvironment); auto wddm = new WddmMock(rootDeviceEnvironment); @@ -48,7 +48,7 @@ TEST(OsContextTest, givenWddmWhenCreateOsContextAfterInitWddmThenOsContextIsInit EngineDescriptorHelper::getDefaultDescriptor(gfxCoreHelper.getGpgpuEngineInstances(rootDeviceEnvironment)[0], preemptionMode)); osContext->ensureContextInitialized(false); EXPECT_EQ(osContext->getWddm(), wddm); - EXPECT_EQ(1u, wddm->registerTrimCallbackResult.called); + EXPECT_EQ(0u, wddm->registerTrimCallbackResult.called); } TEST_F(OsInterfaceTest, GivenWindowsOsWhenCheckForNewResourceImplicitFlushSupportThenReturnTrue) { diff --git a/shared/test/unit_test/os_interface/windows/wddm20_tests.cpp b/shared/test/unit_test/os_interface/windows/wddm20_tests.cpp index 00f5325a5a..1959d22780 100644 --- a/shared/test/unit_test/os_interface/windows/wddm20_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/wddm20_tests.cpp @@ -1139,8 +1139,7 @@ TEST_F(Wddm20Tests, whenContextIsInitializedThenApplyAdditionalContextFlagsIsCal TEST_F(Wddm20Tests, givenTrimCallbackRegistrationIsDisabledInDebugVariableWhenRegisteringCallbackThenReturnNullptr) { DebugManagerStateRestore stateRestore; debugManager.flags.DoNotRegisterTrimCallback.set(true); - WddmResidencyController residencyController{*wddm}; - EXPECT_EQ(nullptr, wddm->registerTrimCallback([](D3DKMT_TRIMNOTIFICATION *) {}, residencyController)); + EXPECT_EQ(nullptr, wddm->registerTrimCallback([](D3DKMT_TRIMNOTIFICATION *) {})); } using WddmLockWithMakeResidentTests = Wddm20Tests; diff --git a/shared/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp b/shared/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp index dd3192ca49..bed0ef205c 100644 --- a/shared/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp @@ -53,7 +53,6 @@ struct WddmResidencyControllerTest : ::testing::Test { wddm->getWddmInterface()->createMonitoredFence(*mockOsContextWin); residencyController = &mockOsContextWin->mockResidencyController; csr.reset(createCommandStream(*executionEnvironment, 0u, 1)); - residencyController->setCommandStreamReceiver(csr.get()); csr->setupContext(*mockOsContextWin); } @@ -71,23 +70,30 @@ struct WddmResidencyControllerWithGdiTest : ::testing::Test { void SetUp() override { executionEnvironment = std::make_unique(); executionEnvironment->initializeMemoryManager(); - rootDeviceEnvironment = std::make_unique(*executionEnvironment); + rootDeviceEnvironment = executionEnvironment->rootDeviceEnvironments[0].get(); wddm = static_cast(Wddm::createWddm(nullptr, *rootDeviceEnvironment)); gdi = new MockGdi(); wddm->resetGdi(gdi); wddm->init(); mockOsContextWin = std::make_unique(*wddm, 0, osContextId, EngineDescriptorHelper::getDefaultDescriptor()); + mockOsContextWin->contextInitialized = true; wddm->getWddmInterface()->createMonitoredFence(*mockOsContextWin); csr.reset(createCommandStream(*executionEnvironment, 0u, 1)); + auto &engines = executionEnvironment->memoryManager->getRegisteredEngines(0); + const_cast(engines).emplace_back(csr.get(), mockOsContextWin.get()); residencyController = &mockOsContextWin->mockResidencyController; - residencyController->setCommandStreamReceiver(csr.get()); csr->setupContext(*mockOsContextWin); residencyController->registerCallback(); } + void TearDown() override { + auto &engines = executionEnvironment->memoryManager->getRegisteredEngines(0); + const_cast(engines).pop_back(); + } + std::unique_ptr executionEnvironment; - std::unique_ptr rootDeviceEnvironment; + RootDeviceEnvironment *rootDeviceEnvironment; WddmMock *wddm = nullptr; std::unique_ptr mockOsContextWin; std::unique_ptr csr; @@ -109,14 +115,13 @@ struct WddmResidencyControllerWithMockWddmTest : public WddmResidencyControllerT csr.reset(createCommandStream(executionEnvironment, 0u, 1)); auto &gfxCoreHelper = executionEnvironment.rootDeviceEnvironments[0]->getHelper(); - osContext = memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor(gfxCoreHelper.getGpgpuEngineInstances(*executionEnvironment.rootDeviceEnvironments[0])[0], - preemptionMode)); + osContext = executionEnvironment.memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor(gfxCoreHelper.getGpgpuEngineInstances(*executionEnvironment.rootDeviceEnvironments[0])[0], + preemptionMode)); csr->setupContext(*osContext); osContext->ensureContextInitialized(false); osContext->incRefInternal(); residencyController = &static_cast(osContext)->getResidencyController(); - residencyController->setCommandStreamReceiver(csr.get()); gmmHelper = executionEnvironment.rootDeviceEnvironments[0]->getGmmHelper(); } @@ -148,14 +153,13 @@ struct WddmResidencyControllerWithGdiAndMemoryManagerTest : ::testing::Test { memoryManager = std::make_unique(executionEnvironment); csr.reset(createCommandStream(executionEnvironment, 0u, 1)); auto &gfxCoreHelper = executionEnvironment.rootDeviceEnvironments[0]->getHelper(); - osContext = memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor(gfxCoreHelper.getGpgpuEngineInstances(*executionEnvironment.rootDeviceEnvironments[0])[0], - PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo))); + osContext = executionEnvironment.memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor(gfxCoreHelper.getGpgpuEngineInstances(*executionEnvironment.rootDeviceEnvironments[0])[0], + PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo))); osContext->ensureContextInitialized(false); osContext->incRefInternal(); residencyController = &static_cast(osContext)->getResidencyController(); - residencyController->setCommandStreamReceiver(csr.get()); csr->setupContext(*osContext); gmmHelper = executionEnvironment.rootDeviceEnvironments[0]->getGmmHelper(); } @@ -182,6 +186,7 @@ TEST(WddmResidencyController, givenWddmResidencyControllerWhenItIsConstructedThe auto wddm = static_cast(Wddm::createWddm(nullptr, *executionEnvironment.rootDeviceEnvironments[0].get())); wddm->resetGdi(gdi); wddm->init(); + EXPECT_EQ(0u, wddm->registerTrimCallbackResult.called); std::memset(&gdi->getRegisterTrimNotificationArg(), 0, sizeof(D3DKMT_REGISTERTRIMNOTIFICATION)); NEO::MockWddmResidencyController residencyController{*wddm}; @@ -200,16 +205,14 @@ TEST(WddmResidencyController, givenWddmResidencyControllerWhenRegisterCallbackTh auto gdi = new MockGdi(); auto wddm = static_cast(Wddm::createWddm(nullptr, *executionEnvironment.rootDeviceEnvironments[0].get())); wddm->resetGdi(gdi); - wddm->init(); std::memset(&gdi->getRegisterTrimNotificationArg(), 0, sizeof(D3DKMT_REGISTERTRIMNOTIFICATION)); - - WddmResidencyController residencyController{*wddm}; - residencyController.registerCallback(); + wddm->init(); + wddm->getResidencyController().registerCallback(); EXPECT_EQ(1u, wddm->registerTrimCallbackResult.called); EXPECT_EQ(reinterpret_cast(WddmResidencyController::trimCallback), gdi->getRegisterTrimNotificationArg().Callback); - EXPECT_EQ(reinterpret_cast(&residencyController), gdi->getRegisterTrimNotificationArg().Context); + EXPECT_EQ(reinterpret_cast(&wddm->getResidencyController()), gdi->getRegisterTrimNotificationArg().Context); EXPECT_EQ(wddm->getDeviceHandle(), gdi->getRegisterTrimNotificationArg().hDevice); } @@ -230,17 +233,6 @@ TEST_F(WddmResidencyControllerTest, givenWddmResidencyControllerThenUpdateLastTr EXPECT_EQ(12345u, mockOsContextWin->lastTrimFenceValue); } -TEST_F(WddmResidencyControllerWithGdiTest, givenWddmResidencyControllerWhenItIsDestructedThenUnregisterTrimCallback) { - auto trimCallbackHandle = residencyController->trimCallbackHandle; - auto trimCallbackAddress = reinterpret_cast(WddmResidencyController::trimCallback); - - std::memset(&gdi->getUnregisterTrimNotificationArg(), 0, sizeof(D3DKMT_UNREGISTERTRIMNOTIFICATION)); - mockOsContextWin.reset(); - - EXPECT_EQ(trimCallbackAddress, gdi->getUnregisterTrimNotificationArg().Callback); - EXPECT_EQ(trimCallbackHandle, gdi->getUnregisterTrimNotificationArg().Handle); -} - TEST_F(WddmResidencyControllerWithGdiTest, givenWddmResidencyControllerWhenItIsDestructedDuringProcessShutdownThenDontUnregisterTrimCallback) { wddm->shutdownStatus = true; @@ -286,15 +278,15 @@ TEST_F(WddmResidencyControllerWithGdiTest, givenNotUsedAllocationsFromPreviousPe wddm->evictResult.called = 0; wddm->callBaseEvict = true; - residencyController->getEvictionAllocations().push_back(&allocation1); - residencyController->getEvictionAllocations().push_back(&allocation2); + wddm->getResidencyController().getEvictionAllocations().push_back(&allocation1); + wddm->getResidencyController().getEvictionAllocations().push_back(&allocation2); - residencyController->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); + wddm->getResidencyController().trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); // Single evict called EXPECT_EQ(1u, wddm->evictResult.called); EXPECT_EQ(1u, gdi->getEvictArg().Flags.EvictOnlyIfNecessary); - EXPECT_TRUE(residencyController->getEvictionAllocations().empty()); + EXPECT_TRUE(wddm->getResidencyController().getEvictionAllocations().empty()); // marked nonresident EXPECT_FALSE(allocation1.getResidencyData().resident[osContextId]); EXPECT_FALSE(allocation2.getResidencyData().resident[osContextId]); @@ -377,9 +369,9 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, givenTripleAllocation wddm->evictResult.called = 0; wddm->callBaseEvict = true; - residencyController->getEvictionAllocations().push_back(allocationTriple); + wddm->getResidencyController().getEvictionAllocations().push_back(allocationTriple); - residencyController->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); + wddm->getResidencyController().trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); // 2 fragments evicted with one call EXPECT_EQ(1u, wddm->evictResult.called); @@ -751,15 +743,6 @@ TEST_F(WddmResidencyControllerLockTest, givenTrimToBudgetWhenTrimmingResidencyTh EXPECT_EQ(1u, residencyController->acquireLockCallCount); } -HWTEST_F(WddmResidencyControllerLockTest, givenTrimToBudgetWhenTrimmingToBudgetThenLockCsr) { - D3DKMT_TRIMNOTIFICATION trimNotification = {0}; - trimNotification.Flags.TrimToBudget = 1; - trimNotification.NumBytesToTrim = 0; - - residencyController->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); - EXPECT_EQ(1u, static_cast *>(residencyController->csr)->recursiveLockCounter); -} - TEST_F(WddmResidencyControllerLockTest, givenPeriodicTrimAndTrimToBudgetWhenTrimmingResidencyThenLockTwice) { D3DKMT_TRIMNOTIFICATION trimNotification = {0}; trimNotification.Flags.PeriodicTrim = 1; @@ -777,7 +760,7 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, WhenMakingResidentRes MockWddmAllocation allocation4(gmmHelper); ResidencyContainer residencyPack{&allocation1, &allocation2, &allocation3, &allocation4}; bool requiresBlockingResidencyHandling = true; - residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); + residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); EXPECT_TRUE(allocation1.getResidencyData().resident[osContextId]); EXPECT_TRUE(allocation2.getResidencyData().resident[osContextId]); @@ -794,7 +777,7 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, WhenMakingResidentRes static_cast(osContext)->getMonitoredFence().currentFenceValue = 20; bool requiresBlockingResidencyHandling = true; - residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); + residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); EXPECT_EQ(20u, allocation1.getResidencyData().getFenceValueForContextId(osContext->getContextId())); EXPECT_EQ(20u, allocation2.getResidencyData().getFenceValueForContextId(osContext->getContextId())); @@ -814,7 +797,7 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, GivenTripleAllocation WddmAllocation *allocationTriple = (WddmAllocation *)memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), false, 2 * MemoryConstants::pageSize}, ptr); ResidencyContainer residencyPack{&allocation1, allocationTriple, &allocation2}; bool requiresBlockingResidencyHandling = true; - residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); + residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); for (uint32_t i = 0; i < allocationTriple->fragmentsStorage.fragmentCount; i++) { EXPECT_TRUE(allocationTriple->fragmentsStorage.fragmentStorageData[i].residency->resident[osContextId]); @@ -834,7 +817,7 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, GivenTripleAllocation static_cast(osContext)->getMonitoredFence().currentFenceValue = 20; bool requiresBlockingResidencyHandling = true; ResidencyContainer residencyPack{&allocation1, allocationTriple, &allocation2}; - residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); + residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); for (uint32_t i = 0; i < allocationTriple->fragmentsStorage.fragmentCount; i++) { EXPECT_EQ(20u, allocationTriple->fragmentsStorage.fragmentStorageData[i].residency->getFenceValueForContextId(0)); @@ -854,7 +837,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallin ResidencyContainer residencyPack{&allocation1, &allocation2, &allocation3, &allocation4}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); EXPECT_FALSE(result); @@ -876,7 +859,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallin ResidencyContainer residencyPack{&allocation1, &allocation2, &allocation3, &allocation4}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); EXPECT_FALSE(result); EXPECT_EQ(residencyPack.size(), 0u); @@ -895,7 +878,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallin ResidencyContainer residencyPack{&allocation1, allocationTriple, &allocation2}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); EXPECT_FALSE(result); @@ -917,7 +900,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallin ResidencyContainer residencyPack{&allocation1}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); EXPECT_FALSE(result); EXPECT_NE(wddm->makeResidentParamsPassed[0].cantTrimFurther, wddm->makeResidentParamsPassed[1].cantTrimFurther); @@ -932,7 +915,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenAllocationPackPassedWhenCal allocation2.handle = 2; ResidencyContainer residencyPack{&allocation1, &allocation2}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); EXPECT_TRUE(result); EXPECT_EQ(2 * EngineLimits::maxHandleCount, wddm->makeResidentResult.handleCount); EXPECT_EQ(false, wddm->makeResidentResult.cantTrimFurther); @@ -957,7 +940,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsAndTrimToB ResidencyContainer residencyPack{&allocation1}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); EXPECT_TRUE(result); @@ -983,7 +966,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsAndTrimToB ResidencyContainer residencyPack{&allocation1, &allocationAlreadyResident}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); EXPECT_TRUE(result); @@ -1000,7 +983,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallin wddm->makeResidentResults = {false, true}; bool requiresBlockingResidencyHandling = true; - residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); + residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); EXPECT_TRUE(residencyController->isMemoryBudgetExhausted()); EXPECT_EQ(2u, wddm->makeResidentResult.called); } @@ -1037,7 +1020,6 @@ struct WddmResidencyControllerWithMockWddmMakeResidentTest : public WddmResidenc osContext->incRefInternal(); residencyController = &static_cast(osContext)->getResidencyController(); - residencyController->setCommandStreamReceiver(csr.get()); csr->setupContext(*osContext); gmmHelper = executionEnvironment.rootDeviceEnvironments[0]->getGmmHelper(); } @@ -1052,7 +1034,7 @@ TEST_F(WddmResidencyControllerWithMockWddmMakeResidentTest, givenMakeResidentFai MockWddmAllocation allocation1(gmmHelper); ResidencyContainer residencyPack{&allocation1}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); EXPECT_TRUE(result); EXPECT_EQ(3u, wddm->makeResidentResult.called); diff --git a/shared/test/unit_test/os_interface/windows/wddm_windows_tests.cpp b/shared/test/unit_test/os_interface/windows/wddm_windows_tests.cpp index 0ad95db01f..a1d278b0b5 100644 --- a/shared/test/unit_test/os_interface/windows/wddm_windows_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/wddm_windows_tests.cpp @@ -117,8 +117,7 @@ HWTEST_F(Wddm20InstrumentationTest, WhenConfiguringDeviceAddressSpaceThenTrueIsR } TEST_F(Wddm20Tests, givenSuccessWhenRegisteringTrimCallbackThenReturnTrimCallbackHandle) { - WddmResidencyController residencyController{*wddm}; - auto trimCallbackHandle = wddm->registerTrimCallback([](D3DKMT_TRIMNOTIFICATION *) {}, residencyController); + auto trimCallbackHandle = wddm->registerTrimCallback([](D3DKMT_TRIMNOTIFICATION *) {}); EXPECT_NE(nullptr, trimCallbackHandle); }