From ad058781651024e15ea04e2f4870c1cc7a4e075d Mon Sep 17 00:00:00 2001 From: Compute-Runtime-Validation Date: Fri, 24 Oct 2025 09:25:41 +0200 Subject: [PATCH] Revert "refactor: Move residency controller to wddm" This reverts commit d9624a270c288bae66d2f029fb56eff13c39f9bb. Signed-off-by: Compute-Runtime-Validation --- shared/source/os_interface/device_factory.cpp | 6 - .../os_interface/linux/os_interface_linux.cpp | 2 - shared/source/os_interface/os_interface.h | 1 - .../os_interface/windows/os_context_win.cpp | 11 +- .../os_interface/windows/os_context_win.h | 4 +- .../os_interface/windows/os_interface_win.cpp | 7 - .../os_interface/windows/trim_callback.cpp | 220 +++++++----------- .../windows/trim_callback_stub.cpp | 2 +- .../source/os_interface/windows/wddm/wddm.cpp | 9 +- .../source/os_interface/windows/wddm/wddm.h | 17 +- .../windows/wddm_device_command_stream.inl | 3 +- .../windows/wddm_residency_controller.cpp | 13 +- .../windows/wddm_residency_controller.h | 9 +- .../test/common/mocks/mock_os_context_win.h | 1 - shared/test/common/mocks/mock_wddm.cpp | 4 +- shared/test/common/mocks/mock_wddm.h | 2 +- .../mocks/mock_wddm_residency_controller.h | 1 + .../windows/os_context_win_tests.cpp | 9 +- .../windows/os_interface_win_tests.cpp | 4 +- .../os_interface/windows/wddm20_tests.cpp | 3 +- .../wddm_residency_controller_tests.cpp | 92 +++++--- .../windows/wddm_windows_tests.cpp | 3 +- 22 files changed, 189 insertions(+), 234 deletions(-) diff --git a/shared/source/os_interface/device_factory.cpp b/shared/source/os_interface/device_factory.cpp index 54deec7261..cc5fee1b03 100644 --- a/shared/source/os_interface/device_factory.cpp +++ b/shared/source/os_interface/device_factory.cpp @@ -330,12 +330,6 @@ std::vector> DeviceFactory::createDevices(ExecutionEnvir } } - for (auto &rootDeviceEnv : executionEnvironment.rootDeviceEnvironments) { - if (rootDeviceEnv->osInterface) { - rootDeviceEnv->osInterface->registerTrimCallback(); - } - } - return devices; } diff --git a/shared/source/os_interface/linux/os_interface_linux.cpp b/shared/source/os_interface/linux/os_interface_linux.cpp index 6ebabcf559..a64d8c8266 100644 --- a/shared/source/os_interface/linux/os_interface_linux.cpp +++ b/shared/source/os_interface/linux/os_interface_linux.cpp @@ -73,6 +73,4 @@ uint32_t OSInterface::getAggregatedProcessCount() const { return 0; } -void OSInterface::registerTrimCallback() {} - } // namespace NEO diff --git a/shared/source/os_interface/os_interface.h b/shared/source/os_interface/os_interface.h index ef55dbf67b..004077b9ec 100644 --- a/shared/source/os_interface/os_interface.h +++ b/shared/source/os_interface/os_interface.h @@ -122,7 +122,6 @@ class OSInterface : public NonCopyableClass { MOCKABLE_VIRTUAL bool isLockablePointer(bool isLockable) const; MOCKABLE_VIRTUAL bool isSizeWithinThresholdForStaging(const void *ptr, size_t size) const; MOCKABLE_VIRTUAL uint32_t getAggregatedProcessCount() const; - void registerTrimCallback(); static bool osEnabled64kbPages; static bool are64kbPagesEnabled(); diff --git a/shared/source/os_interface/windows/os_context_win.cpp b/shared/source/os_interface/windows/os_context_win.cpp index 4152818b76..4158c90b68 100644 --- a/shared/source/os_interface/windows/os_context_win.cpp +++ b/shared/source/os_interface/windows/os_context_win.cpp @@ -24,7 +24,11 @@ OsContext *OsContextWin::create(OSInterface *osInterface, uint32_t rootDeviceInd return new OsContext(rootDeviceIndex, contextId, engineDescriptor); } -OsContextWin::OsContextWin(Wddm &wddm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor) : OsContext(rootDeviceIndex, contextId, engineDescriptor), wddm(wddm) {} +OsContextWin::OsContextWin(Wddm &wddm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor) + : OsContext(rootDeviceIndex, contextId, engineDescriptor), + residencyController(wddm), + wddm(wddm) { +} bool OsContextWin::initializeContext(bool allocateInterrupt) { @@ -41,6 +45,9 @@ bool OsContextWin::initializeContext(bool allocateInterrupt) { UNRECOVERABLE_IF(!wddmInterface->createHwQueue(*this)); } UNRECOVERABLE_IF(!wddmInterface->createMonitoredFence(*this)); + + residencyController.registerCallback(); + UNRECOVERABLE_IF(!residencyController.isInitialized()); } return true; @@ -94,8 +101,6 @@ uint32_t OsContextWin::getDeviceNodeMask() { return hwDeviceID->getAdapterNodeMask(); } -WddmResidencyController &OsContextWin::getResidencyController() { return wddm.getResidencyController(); } - uint64_t OsContextWin::getOfflineDumpContextId(uint32_t deviceIndex) const { return 0; } diff --git a/shared/source/os_interface/windows/os_context_win.h b/shared/source/os_interface/windows/os_context_win.h index 11f0ddefae..2884ff5102 100644 --- a/shared/source/os_interface/windows/os_context_win.h +++ b/shared/source/os_interface/windows/os_context_win.h @@ -33,7 +33,7 @@ class OsContextWin : public OsContext { void setHwQueue(HardwareQueue hardwareQueue) { this->hardwareQueue = hardwareQueue; } bool isDirectSubmissionSupported() const override; Wddm *getWddm() const { return &wddm; } - MOCKABLE_VIRTUAL WddmResidencyController &getResidencyController(); + MOCKABLE_VIRTUAL WddmResidencyController &getResidencyController() { return residencyController; } static OsContext *create(OSInterface *osInterface, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor); MonitoredFence &getMonitoredFence() { return monitoredFence; } void resetMonitoredFenceParams(D3DKMT_HANDLE &handle, uint64_t *cpuAddress, D3DGPU_VIRTUAL_ADDRESS &gpuAddress); @@ -48,6 +48,8 @@ class OsContextWin : public OsContext { protected: bool initializeContext(bool allocateInterrupt) override; + WddmResidencyController residencyController; + HardwareQueue hardwareQueue; MonitoredFence monitoredFence = {}; diff --git a/shared/source/os_interface/windows/os_interface_win.cpp b/shared/source/os_interface/windows/os_interface_win.cpp index d6953ca439..7d2a31e903 100644 --- a/shared/source/os_interface/windows/os_interface_win.cpp +++ b/shared/source/os_interface/windows/os_interface_win.cpp @@ -34,11 +34,4 @@ uint32_t OSInterface::getAggregatedProcessCount() const { return 0; } -void OSInterface::registerTrimCallback() { - if (driverModel && driverModel->getDriverModelType() == DriverModelType::wddm) { - driverModel->as()->getResidencyController().registerCallback(); - UNRECOVERABLE_IF(!driverModel->as()->getResidencyController().isInitialized()); - } -} - } // namespace NEO diff --git a/shared/source/os_interface/windows/trim_callback.cpp b/shared/source/os_interface/windows/trim_callback.cpp index c436defdc5..e112617249 100644 --- a/shared/source/os_interface/windows/trim_callback.cpp +++ b/shared/source/os_interface/windows/trim_callback.cpp @@ -15,11 +15,7 @@ namespace NEO { -VOID *Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback) { - if (!hwDeviceId || !this->rootDeviceEnvironment.executionEnvironment.osEnvironment || !getGdi()) { - return nullptr; - } - +VOID *Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController) { if (debugManager.flags.DoNotRegisterTrimCallback.get()) { return nullptr; } @@ -37,10 +33,6 @@ VOID *Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback) { } void Wddm::unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, VOID *trimCallbackHandle) { - if (!hwDeviceId || !this->rootDeviceEnvironment.executionEnvironment.osEnvironment || !getGdi()) { - return; - } - DEBUG_BREAK_IF(callback == nullptr); if (trimCallbackHandle == nullptr || isShutdownInProgress()) { return; @@ -56,6 +48,7 @@ void Wddm::unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, V void APIENTRY WddmResidencyController::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification) { auto residencyController = static_cast(trimNotification->Context); DEBUG_BREAK_IF(residencyController == nullptr); + UNRECOVERABLE_IF(residencyController->csr == nullptr); auto lock = residencyController->acquireTrimCallbackLock(); residencyController->trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim); @@ -64,94 +57,74 @@ void APIENTRY WddmResidencyController::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICAT void WddmResidencyController::trimResidency(const D3DDDI_TRIMRESIDENCYSET_FLAGS &flags, uint64_t bytes) { std::chrono::high_resolution_clock::time_point callbackStart; perfLogResidencyTrimCallbackBegin(wddm.getResidencyLogger(), callbackStart); + auto &osContext = static_cast(this->csr->getOsContext()); + uint32_t osContextId = osContext.getContextId(); if (flags.PeriodicTrim) { uint64_t sizeToTrim = 0; auto lock = this->acquireLock(); WddmAllocation *wddmAllocation = nullptr; std::vector handlesToEvict; - std::vector allocationsToRemove; - allocationsToRemove.reserve(evictionAllocations.size()); handlesToEvict.reserve(evictionAllocations.size()); + for (auto allocationIter = evictionAllocations.begin(); allocationIter != evictionAllocations.end();) { + wddmAllocation = reinterpret_cast(*allocationIter); - this->wddm.forEachContextWithinWddm([&](const EngineControl &engine) { - for (auto allocationIter = evictionAllocations.begin(); allocationIter != evictionAllocations.end(); ++allocationIter) { - auto &osContext = static_cast(*engine.osContext); - uint32_t osContextId = osContext.getContextId(); + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", osContext.getLastTrimFenceValue()); - wddmAllocation = reinterpret_cast(*allocationIter); + if (osContext.wasAllocationUsedSinceLastTrim(wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId))) { + allocationIter++; + continue; + } - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", osContext.getLastTrimFenceValue()); + if (wddmAllocation->isAlwaysResident(osContextId)) { + allocationIter = evictionAllocations.erase(allocationIter); + continue; + } - if (osContext.wasAllocationUsedSinceLastTrim(wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId))) { - continue; - } - - if (wddmAllocation->isAlwaysResident(osContextId)) { - continue; - } - - if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { - if (!wddmAllocation->getResidencyData().resident[osContextId]) { - continue; - } - for (auto i = 0u; i < wddmAllocation->getNumGmms(); i++) { - handlesToEvict.push_back(wddmAllocation->getHandles()[i]); - } - } else { - for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { - AllocationStorageData &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId]; - if (!osContext.wasAllocationUsedSinceLastTrim(fragmentStorageData.residency->getFenceValueForContextId(osContextId))) { - auto osHandle = static_cast(wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage); - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", osHandle->handle, "lastFence =", - wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId)); - - handlesToEvict.push_back(static_cast(fragmentStorageData.osHandleStorage)->handle); - fragmentStorageData.residency->resident[osContextId] = false; - } - } - } - - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation, gpu address = ", std::hex, wddmAllocation->getGpuAddress(), "lastFence =", wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId)); - wddmAllocation->getResidencyData().resident[osContextId] = false; - - if (std::none_of(wddmAllocation->getResidencyData().resident.begin(), wddmAllocation->getResidencyData().resident.end(), [](bool resident) { return resident; })) { - allocationsToRemove.push_back(wddmAllocation); + if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { + for (auto i = 0u; i < wddmAllocation->getNumGmms(); i++) { + handlesToEvict.push_back(wddmAllocation->getHandles()[i]); } } - this->wddm.evict(handlesToEvict.data(), static_cast(handlesToEvict.size()), sizeToTrim, false); - handlesToEvict.clear(); - }); + for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { + AllocationStorageData &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId]; + if (!osContext.wasAllocationUsedSinceLastTrim(fragmentStorageData.residency->getFenceValueForContextId(osContextId))) { + auto osHandle = static_cast(wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage); + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", osHandle->handle, "lastFence =", + wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId)); - std::erase_if(evictionAllocations, [&allocationsToRemove](GraphicsAllocation *graphicsAllocation) { - return std::find_if(allocationsToRemove.begin(), allocationsToRemove.end(), [&graphicsAllocation](WddmAllocation *graphicsAllocationToRemove) { - return graphicsAllocationToRemove == graphicsAllocation; - }) != allocationsToRemove.end(); - }); + handlesToEvict.push_back(static_cast(fragmentStorageData.osHandleStorage)->handle); + fragmentStorageData.residency->resident[osContextId] = false; + } + } + + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation, gpu address = ", std::hex, wddmAllocation->getGpuAddress(), "lastFence =", wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId)); + wddmAllocation->getResidencyData().resident[osContextId] = false; + allocationIter = evictionAllocations.erase(allocationIter); + } + + this->wddm.evict(handlesToEvict.data(), static_cast(handlesToEvict.size()), sizeToTrim, false); } if (flags.TrimToBudget) { - StackVec, 5> csrLocks; - this->wddm.forEachContextWithinWddm([&](const EngineControl &engine) { - csrLocks.push_back(engine.commandStreamReceiver->obtainUniqueOwnership()); - }); + auto csrLock = csr->obtainUniqueOwnership(); auto lock = this->acquireLock(); trimResidencyToBudget(bytes); } if (flags.PeriodicTrim || flags.RestartPeriodicTrim) { - this->wddm.forEachContextWithinWddm([&](const EngineControl &engine) { - auto osContext = static_cast(engine.osContext); - osContext->updateLastTrimFenceValue(); - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", osContext->getLastTrimFenceValue()); - }); + osContext.updateLastTrimFenceValue(); + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", osContext.getLastTrimFenceValue()); } perfLogResidencyTrimCallbackEnd(wddm.getResidencyLogger(), flags.Value, this, callbackStart); } bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes) { + this->csr->drainPagingFenceQueue(); + auto &osContext = static_cast(this->csr->getOsContext()); + uint32_t osContextId = osContext.getContextId(); uint64_t sizeToTrim = 0; uint64_t numberOfBytesToTrim = bytes; WddmAllocation *wddmAllocation = nullptr; @@ -160,78 +133,55 @@ bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes) { auto allocationIter = evictionAllocations.begin(); std::vector handlesToEvict; handlesToEvict.reserve(evictionAllocations.size()); - std::vector allocationsToRemove; - allocationsToRemove.reserve(evictionAllocations.size()); + while (numberOfBytesToTrim > 0 && allocationIter != evictionAllocations.end()) { + wddmAllocation = reinterpret_cast(*allocationIter); + uint64_t lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId); + auto &monitoredFence = osContext.getMonitoredFence(); - this->wddm.forEachContextWithinWddm([&](const EngineControl &engine) { - auto csr = engine.commandStreamReceiver; - csr->drainPagingFenceQueue(); - auto &osContext = static_cast(csr->getOsContext()); - uint32_t osContextId = osContext.getContextId(); - - while (numberOfBytesToTrim > 0 && allocationIter != evictionAllocations.end()) { - wddmAllocation = reinterpret_cast(*allocationIter); - uint64_t lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId); - auto &monitoredFence = osContext.getMonitoredFence(); - - if (lastFence > monitoredFence.lastSubmittedFence) { - ++allocationIter; - continue; - } - - if (wddmAllocation->isAlwaysResident(osContextId)) { - ++allocationIter; - continue; - } - - uint64_t sizeEvicted = 0; - - if (lastFence > *monitoredFence.cpuAddress) { - this->wddm.waitFromCpu(lastFence, osContext.getMonitoredFence(), false); - } - - if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { - for (auto i = 0u; i < wddmAllocation->getNumGmms(); i++) { - handlesToEvict.push_back(wddmAllocation->getHandles()[i]); - } - sizeEvicted = wddmAllocation->getAlignedSize(); - } else { - auto &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData; - for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { - if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) { - handlesToEvict.push_back(static_cast(fragmentStorageData[allocationId].osHandleStorage)->handle); - fragmentStorageData[allocationId].residency->resident[osContextId] = false; - sizeEvicted += fragmentStorageData[allocationId].fragmentSize; - } - } - } - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation, gpu address = ", std::hex, wddmAllocation->getGpuAddress(), "lastFence =", wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId)); - - if (sizeEvicted >= numberOfBytesToTrim) { - numberOfBytesToTrim = 0; - } else { - numberOfBytesToTrim -= sizeEvicted; - } - - wddmAllocation->getResidencyData().resident[osContextId] = false; - - if (std::none_of(wddmAllocation->getResidencyData().resident.begin(), wddmAllocation->getResidencyData().resident.end(), [](bool resident) { return resident; })) { - allocationsToRemove.push_back(wddmAllocation); - } - - ++allocationIter; + if (lastFence > monitoredFence.lastSubmittedFence) { + allocationIter++; + continue; } - this->wddm.evict(handlesToEvict.data(), static_cast(handlesToEvict.size()), sizeToTrim, true); - handlesToEvict.clear(); - }); + if (wddmAllocation->isAlwaysResident(osContextId)) { + allocationIter = evictionAllocations.erase(allocationIter); + continue; + } - std::erase_if(evictionAllocations, [&allocationsToRemove](GraphicsAllocation *graphicsAllocation) { - return std::find_if(allocationsToRemove.begin(), allocationsToRemove.end(), [&graphicsAllocation](WddmAllocation *graphicsAllocationToRemove) { - return graphicsAllocationToRemove == graphicsAllocation; - }) != allocationsToRemove.end(); - }); + uint64_t sizeEvicted = 0; + if (lastFence > *monitoredFence.cpuAddress) { + this->wddm.waitFromCpu(lastFence, osContext.getMonitoredFence(), false); + } + + if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { + for (auto i = 0u; i < wddmAllocation->getNumGmms(); i++) { + handlesToEvict.push_back(wddmAllocation->getHandles()[i]); + } + sizeEvicted = wddmAllocation->getAlignedSize(); + } else { + auto &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData; + for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { + if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) { + handlesToEvict.push_back(static_cast(fragmentStorageData[allocationId].osHandleStorage)->handle); + fragmentStorageData[allocationId].residency->resident[osContextId] = false; + sizeEvicted += fragmentStorageData[allocationId].fragmentSize; + } + } + } + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation, gpu address = ", std::hex, wddmAllocation->getGpuAddress(), "lastFence =", wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId)); + + if (sizeEvicted >= numberOfBytesToTrim) { + numberOfBytesToTrim = 0; + } else { + numberOfBytesToTrim -= sizeEvicted; + } + + wddmAllocation->getResidencyData().resident[osContextId] = false; + allocationIter = evictionAllocations.erase(allocationIter); + } + + this->wddm.evict(handlesToEvict.data(), static_cast(handlesToEvict.size()), sizeToTrim, true); return numberOfBytesToTrim == 0; } diff --git a/shared/source/os_interface/windows/trim_callback_stub.cpp b/shared/source/os_interface/windows/trim_callback_stub.cpp index 8189adbfa2..13c52b68fc 100644 --- a/shared/source/os_interface/windows/trim_callback_stub.cpp +++ b/shared/source/os_interface/windows/trim_callback_stub.cpp @@ -11,7 +11,7 @@ namespace NEO { -VOID *Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback) { +VOID *Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController) { return nullptr; } diff --git a/shared/source/os_interface/windows/wddm/wddm.cpp b/shared/source/os_interface/windows/wddm/wddm.cpp index 52c41d4c51..d3437434ff 100644 --- a/shared/source/os_interface/windows/wddm/wddm.cpp +++ b/shared/source/os_interface/windows/wddm/wddm.cpp @@ -54,7 +54,7 @@ Wddm::CreateDXGIFactoryFcn Wddm::createDxgiFactory = getCreateDxgiFactory(); Wddm::GetSystemInfoFcn Wddm::getSystemInfo = getGetSystemInfo(); Wddm::Wddm(std::unique_ptr &&hwDeviceIdIn, RootDeviceEnvironment &rootDeviceEnvironment) - : DriverModel(DriverModelType::wddm), residencyController(*this), hwDeviceId(std::move(hwDeviceIdIn)), rootDeviceEnvironment(rootDeviceEnvironment) { + : DriverModel(DriverModelType::wddm), hwDeviceId(std::move(hwDeviceIdIn)), rootDeviceEnvironment(rootDeviceEnvironment) { UNRECOVERABLE_IF(!hwDeviceId); featureTable.reset(new FeatureTable()); workaroundTable.reset(new WorkaroundTable()); @@ -76,7 +76,6 @@ Wddm::Wddm(std::unique_ptr &&hwDeviceIdIn, RootDeviceEnvironment } Wddm::~Wddm() { - this->residencyController.unregisterCallback(); temporaryResources.reset(); destroyPagingQueue(); destroyDevice(); @@ -611,7 +610,7 @@ bool Wddm::mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_A bool ret = true; auto &productHelper = rootDeviceEnvironment.getHelper(); if (gmm->isCompressionEnabled() && productHelper.isPageTableManagerSupported(*rootDeviceEnvironment.getHardwareInfo())) { - this->forEachContextWithinWddm([&](const EngineControl &engine) { + this->forEachContextWithinWddm([&](const EngineControl &engine) { if (engine.commandStreamReceiver->pageTableManager.get()) { ret &= engine.commandStreamReceiver->pageTableManager->updateAuxTable(gpuPtr, gmm, true); } @@ -1197,7 +1196,7 @@ bool Wddm::waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredF if (!skipResourceCleanup() && lastFenceValue > *monitoredFence.cpuAddress) { CommandStreamReceiver *csr = nullptr; - this->forEachContextWithinWddm([&monitoredFence, &csr](const EngineControl &engine) { + this->forEachContextWithinWddm([&monitoredFence, &csr](const EngineControl &engine) { auto &contextMonitoredFence = static_cast(engine.osContext)->getMonitoredFence(); if (contextMonitoredFence.cpuAddress == monitoredFence.cpuAddress) { csr = engine.commandStreamReceiver; @@ -1440,7 +1439,7 @@ void Wddm::setNewResourceBoundToPageTable() { if (!this->rootDeviceEnvironment.getProductHelper().isTlbFlushRequired()) { return; } - this->forEachContextWithinWddm([](const EngineControl &engine) { engine.osContext->setNewResourceBound(); }); + this->forEachContextWithinWddm([](const EngineControl &engine) { engine.osContext->setNewResourceBound(); }); } bool Wddm::needsNotifyAubCaptureCallback() const { diff --git a/shared/source/os_interface/windows/wddm/wddm.h b/shared/source/os_interface/windows/wddm/wddm.h index b05fc8c7df..a062e23115 100644 --- a/shared/source/os_interface/windows/wddm/wddm.h +++ b/shared/source/os_interface/windows/wddm/wddm.h @@ -13,7 +13,6 @@ #include "shared/source/os_interface/windows/sharedata_wrapper.h" #include "shared/source/os_interface/windows/wddm/wddm_defs.h" #include "shared/source/os_interface/windows/wddm_memory_manager.h" -#include "shared/source/os_interface/windows/wddm_residency_controller.h" #include @@ -31,6 +30,7 @@ class ProductHelper; class SettingsReader; class WddmAllocation; class WddmInterface; +class WddmResidencyController; class WddmResidencyLogger; class WddmResidentAllocationsContainer; @@ -95,8 +95,7 @@ class Wddm : public DriverModel { MOCKABLE_VIRTUAL bool waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredFence, bool busyWait); MOCKABLE_VIRTUAL NTSTATUS escape(D3DKMT_ESCAPE &escapeCommand); - WddmResidencyController &getResidencyController() { return residencyController; } - MOCKABLE_VIRTUAL VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback); + MOCKABLE_VIRTUAL VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController); void unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, VOID *trimCallbackHandle); MOCKABLE_VIRTUAL void releaseReservedAddress(void *reservedAddress); MOCKABLE_VIRTUAL bool reserveValidAddressRange(size_t size, void *&reservedMem); @@ -218,18 +217,12 @@ class Wddm : public DriverModel { return additionalAdapterInfoOptions; } - template + template void forEachContextWithinWddm(F func) { for (auto rootDeviceIndex = 0u; rootDeviceIndex < rootDeviceEnvironment.executionEnvironment.rootDeviceEnvironments.size(); rootDeviceIndex++) { if (rootDeviceEnvironment.executionEnvironment.rootDeviceEnvironments[rootDeviceIndex].get() == &rootDeviceEnvironment) { for (auto &engine : rootDeviceEnvironment.executionEnvironment.memoryManager->getRegisteredEngines(rootDeviceIndex)) { - if constexpr (onlyInitialized) { - if (engine.osContext->isInitialized()) { - func(engine); - } - } else { - func(engine); - } + func(engine); } } } @@ -269,8 +262,6 @@ class Wddm : public DriverModel { bool isReadOnlyFlagFallbackSupported() const; bool isReadOnlyFlagFallbackAvailable(const D3DKMT_CREATEALLOCATION &createAllocation) const; - WddmResidencyController residencyController; - GMM_GFX_PARTITIONING gfxPartition{}; ADAPTER_BDF adapterBDF{}; diff --git a/shared/source/os_interface/windows/wddm_device_command_stream.inl b/shared/source/os_interface/windows/wddm_device_command_stream.inl index 17cfefb37e..4eb298e3f3 100644 --- a/shared/source/os_interface/windows/wddm_device_command_stream.inl +++ b/shared/source/os_interface/windows/wddm_device_command_stream.inl @@ -147,7 +147,7 @@ SubmissionStatus WddmCommandStreamReceiver::flush(BatchBuffer &batchB template SubmissionStatus WddmCommandStreamReceiver::processResidency(ResidencyContainer &allocationsForResidency, uint32_t handleId) { - return static_cast(this->osContext)->getResidencyController().makeResidentResidencyAllocations(allocationsForResidency, this->requiresBlockingResidencyHandling, this) ? SubmissionStatus::success : SubmissionStatus::outOfMemory; + return static_cast(this->osContext)->getResidencyController().makeResidentResidencyAllocations(allocationsForResidency, this->requiresBlockingResidencyHandling, *static_cast(this->osContext)) ? SubmissionStatus::success : SubmissionStatus::outOfMemory; } template @@ -212,6 +212,7 @@ CommandStreamReceiver *createWddmDeviceCommandStreamReceiver(bool withAubDump, template void WddmCommandStreamReceiver::setupContext(OsContext &osContext) { this->osContext = &osContext; + static_cast(this->osContext)->getResidencyController().setCommandStreamReceiver(this); } template diff --git a/shared/source/os_interface/windows/wddm_residency_controller.cpp b/shared/source/os_interface/windows/wddm_residency_controller.cpp index ee9acf187e..1c506debeb 100644 --- a/shared/source/os_interface/windows/wddm_residency_controller.cpp +++ b/shared/source/os_interface/windows/wddm_residency_controller.cpp @@ -22,10 +22,10 @@ WddmResidencyController::WddmResidencyController(Wddm &wddm) : wddm(wddm) { } void WddmResidencyController::registerCallback() { - this->trimCallbackHandle = wddm.registerTrimCallback(WddmResidencyController::trimCallback); + this->trimCallbackHandle = wddm.registerTrimCallback(WddmResidencyController::trimCallback, *this); } -void WddmResidencyController::unregisterCallback() { +WddmResidencyController::~WddmResidencyController() { auto lock = this->acquireTrimCallbackLock(); wddm.unregisterTrimCallback(WddmResidencyController::trimCallback, this->trimCallbackHandle); lock.unlock(); @@ -54,8 +54,7 @@ std::unique_lock WddmResidencyController::acquireTrimCallbackLock() { * * @return returns true if all allocations either succeeded or are pending to be resident */ -bool WddmResidencyController::makeResidentResidencyAllocations(ResidencyContainer &allocationsForResidency, bool &requiresBlockingResidencyHandling, CommandStreamReceiver *csr) { - auto &osContext = static_cast(csr->getOsContext()); +bool WddmResidencyController::makeResidentResidencyAllocations(ResidencyContainer &allocationsForResidency, bool &requiresBlockingResidencyHandling, OsContextWin &osContext) { auto osContextId = osContext.getContextId(); const size_t residencyCount = allocationsForResidency.size(); requiresBlockingResidencyHandling = false; @@ -160,13 +159,17 @@ size_t WddmResidencyController::fillHandlesContainer(ResidencyContainer &allocat bool WddmResidencyController::isInitialized() const { bool requiresTrimCallbacks = OSInterface::requiresSupportForWddmTrimNotification; - requiresTrimCallbacks = requiresTrimCallbacks && (false == debugManager.flags.DoNotRegisterTrimCallback.get()) && wddm.getHwDeviceId() && wddm.getRootDeviceEnvironment().executionEnvironment.osEnvironment && wddm.getGdi(); + requiresTrimCallbacks = requiresTrimCallbacks && (false == debugManager.flags.DoNotRegisterTrimCallback.get()); if (requiresTrimCallbacks) { return trimCallbackHandle != nullptr; } return true; } +void WddmResidencyController::setCommandStreamReceiver(CommandStreamReceiver *csr) { + this->csr = csr; +} + void WddmResidencyController::removeAllocation(ResidencyContainer &container, GraphicsAllocation *gfxAllocation) { std::unique_lock lock1(this->lock, std::defer_lock); std::unique_lock lock2(this->trimCallbackLock, std::defer_lock); diff --git a/shared/source/os_interface/windows/wddm_residency_controller.h b/shared/source/os_interface/windows/wddm_residency_controller.h index e52ffa22cf..397e645453 100644 --- a/shared/source/os_interface/windows/wddm_residency_controller.h +++ b/shared/source/os_interface/windows/wddm_residency_controller.h @@ -30,7 +30,7 @@ class OsContextWin; class WddmResidencyController { public: WddmResidencyController(Wddm &wddm); - MOCKABLE_VIRTUAL ~WddmResidencyController() = default; + MOCKABLE_VIRTUAL ~WddmResidencyController(); static void APIENTRY trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification); @@ -38,7 +38,6 @@ class WddmResidencyController { [[nodiscard]] std::unique_lock acquireTrimCallbackLock(); void registerCallback(); - void unregisterCallback(); void trimResidency(const D3DDDI_TRIMRESIDENCYSET_FLAGS &flags, uint64_t bytes); bool trimResidencyToBudget(uint64_t bytes); @@ -46,10 +45,12 @@ class WddmResidencyController { bool isMemoryBudgetExhausted() const { return memoryBudgetExhausted; } void setMemoryBudgetExhausted() { memoryBudgetExhausted = true; } - bool makeResidentResidencyAllocations(ResidencyContainer &allocationsForResidency, bool &requiresBlockingResidencyHandling, CommandStreamReceiver *csr); + bool makeResidentResidencyAllocations(ResidencyContainer &allocationsForResidency, bool &requiresBlockingResidencyHandling, OsContextWin &osContext); bool isInitialized() const; + void setCommandStreamReceiver(CommandStreamReceiver *csr); + void removeAllocation(ResidencyContainer &container, GraphicsAllocation *gfxAllocation); ResidencyContainer &getEvictionAllocations() { @@ -67,6 +68,8 @@ class WddmResidencyController { bool memoryBudgetExhausted = false; + CommandStreamReceiver *csr = nullptr; + ResidencyContainer evictionAllocations; ResidencyContainer backupResidencyContainer; // Stores allocations which should be resident diff --git a/shared/test/common/mocks/mock_os_context_win.h b/shared/test/common/mocks/mock_os_context_win.h index eb72670a44..f0086e4001 100644 --- a/shared/test/common/mocks/mock_os_context_win.h +++ b/shared/test/common/mocks/mock_os_context_win.h @@ -12,7 +12,6 @@ namespace NEO { class MockOsContextWin : public OsContextWin { public: - using OsContext::contextInitialized; using OsContextWin::lastTrimFenceValue; MockOsContextWin(Wddm &wddm, uint32_t rootDeviceIndex, uint32_t contextId, const EngineDescriptor &engineDescriptor) diff --git a/shared/test/common/mocks/mock_wddm.cpp b/shared/test/common/mocks/mock_wddm.cpp index a2a83e8d5a..403ea6d3aa 100644 --- a/shared/test/common/mocks/mock_wddm.cpp +++ b/shared/test/common/mocks/mock_wddm.cpp @@ -316,9 +316,9 @@ bool WddmMock::reserveValidAddressRange(size_t size, void *&reservedMem) { } return ret; } -VOID *WddmMock::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback) { +VOID *WddmMock::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController) { registerTrimCallbackResult.called++; - return Wddm::registerTrimCallback(callback); + return Wddm::registerTrimCallback(callback, residencyController); } NTSTATUS WddmMock::reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS baseAddress, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size, D3DGPU_VIRTUAL_ADDRESS *reservedAddress) { diff --git a/shared/test/common/mocks/mock_wddm.h b/shared/test/common/mocks/mock_wddm.h index 4a00b43c39..8e9e3b7d6d 100644 --- a/shared/test/common/mocks/mock_wddm.h +++ b/shared/test/common/mocks/mock_wddm.h @@ -105,7 +105,7 @@ class WddmMock : public Wddm { void *virtualAlloc(void *inPtr, size_t size, bool topDownHint) override; void virtualFree(void *ptr, size_t size) override; void releaseReservedAddress(void *reservedAddress) override; - VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback) override; + VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController) override; NTSTATUS reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS baseAddress, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size, D3DGPU_VIRTUAL_ADDRESS *reservedAddress) override; bool reserveValidAddressRange(size_t size, void *&reservedMem) override; PLATFORM *getGfxPlatform() { return gfxPlatform.get(); } diff --git a/shared/test/common/mocks/mock_wddm_residency_controller.h b/shared/test/common/mocks/mock_wddm_residency_controller.h index 68ab163b16..be8b356f9f 100644 --- a/shared/test/common/mocks/mock_wddm_residency_controller.h +++ b/shared/test/common/mocks/mock_wddm_residency_controller.h @@ -11,6 +11,7 @@ namespace NEO { class MockWddmResidencyController : public WddmResidencyController { public: + using WddmResidencyController::csr; using WddmResidencyController::lock; using WddmResidencyController::trimCallbackHandle; using WddmResidencyController::trimResidency; diff --git a/shared/test/unit_test/os_interface/windows/os_context_win_tests.cpp b/shared/test/unit_test/os_interface/windows/os_context_win_tests.cpp index 79ec6eadc6..a8fb58878a 100644 --- a/shared/test/unit_test/os_interface/windows/os_context_win_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/os_context_win_tests.cpp @@ -44,13 +44,10 @@ TEST_F(OsContextWinTest, givenWddm20WhenCreatingWddmMonitorFenceFailThenOsContex EXPECT_ANY_THROW(osContext->ensureContextInitialized(false)); } -TEST_F(OsContextWinTest, givenWddm20WhenRegisterTrimCallbackFailThenResidencyControllerIsNotInitialized) { +TEST_F(OsContextWinTest, givenWddm20WhenRegisterTrimCallbackFailThenOsContextCreationFails) { *getRegisterTrimNotificationFailCallFcn() = true; - WddmMock wddm(*rootDeviceEnvironment); - wddm.init(); - - wddm.getResidencyController().registerCallback(); - EXPECT_FALSE(wddm.getResidencyController().isInitialized()); + osContext = std::make_unique(*osInterface->getDriverModel()->as(), 0, 0u, EngineDescriptorHelper::getDefaultDescriptor(engineTypeUsage, preemptionMode)); + EXPECT_ANY_THROW(osContext->ensureContextInitialized(false)); } TEST_F(OsContextWinTest, givenWddm20WhenRegisterTrimCallbackIsDisabledThenOsContextIsInitialized) { diff --git a/shared/test/unit_test/os_interface/windows/os_interface_win_tests.cpp b/shared/test/unit_test/os_interface/windows/os_interface_win_tests.cpp index 91c053bdc7..c9497ef1f1 100644 --- a/shared/test/unit_test/os_interface/windows/os_interface_win_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/os_interface_win_tests.cpp @@ -36,7 +36,7 @@ TEST_F(OsInterfaceTest, GivenWindowsWhenCreateEentIsCalledThenValidEventHandleIs EXPECT_EQ(TRUE, ret); } -TEST(OsContextTest, givenWddmWhenCreateOsContextAfterInitWddmThenOsContextIsInitializedTrimCallbackIsNotRegisteredMemoryOperationsHandlerCreated) { +TEST(OsContextTest, givenWddmWhenCreateOsContextAfterInitWddmThenOsContextIsInitializedTrimCallbackIsRegisteredMemoryOperationsHandlerCreated) { MockExecutionEnvironment executionEnvironment; RootDeviceEnvironment rootDeviceEnvironment(executionEnvironment); auto wddm = new WddmMock(rootDeviceEnvironment); @@ -48,7 +48,7 @@ TEST(OsContextTest, givenWddmWhenCreateOsContextAfterInitWddmThenOsContextIsInit EngineDescriptorHelper::getDefaultDescriptor(gfxCoreHelper.getGpgpuEngineInstances(rootDeviceEnvironment)[0], preemptionMode)); osContext->ensureContextInitialized(false); EXPECT_EQ(osContext->getWddm(), wddm); - EXPECT_EQ(0u, wddm->registerTrimCallbackResult.called); + EXPECT_EQ(1u, wddm->registerTrimCallbackResult.called); } TEST_F(OsInterfaceTest, GivenWindowsOsWhenCheckForNewResourceImplicitFlushSupportThenReturnTrue) { diff --git a/shared/test/unit_test/os_interface/windows/wddm20_tests.cpp b/shared/test/unit_test/os_interface/windows/wddm20_tests.cpp index 1959d22780..00f5325a5a 100644 --- a/shared/test/unit_test/os_interface/windows/wddm20_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/wddm20_tests.cpp @@ -1139,7 +1139,8 @@ TEST_F(Wddm20Tests, whenContextIsInitializedThenApplyAdditionalContextFlagsIsCal TEST_F(Wddm20Tests, givenTrimCallbackRegistrationIsDisabledInDebugVariableWhenRegisteringCallbackThenReturnNullptr) { DebugManagerStateRestore stateRestore; debugManager.flags.DoNotRegisterTrimCallback.set(true); - EXPECT_EQ(nullptr, wddm->registerTrimCallback([](D3DKMT_TRIMNOTIFICATION *) {})); + WddmResidencyController residencyController{*wddm}; + EXPECT_EQ(nullptr, wddm->registerTrimCallback([](D3DKMT_TRIMNOTIFICATION *) {}, residencyController)); } using WddmLockWithMakeResidentTests = Wddm20Tests; diff --git a/shared/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp b/shared/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp index bed0ef205c..dd3192ca49 100644 --- a/shared/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp @@ -53,6 +53,7 @@ struct WddmResidencyControllerTest : ::testing::Test { wddm->getWddmInterface()->createMonitoredFence(*mockOsContextWin); residencyController = &mockOsContextWin->mockResidencyController; csr.reset(createCommandStream(*executionEnvironment, 0u, 1)); + residencyController->setCommandStreamReceiver(csr.get()); csr->setupContext(*mockOsContextWin); } @@ -70,30 +71,23 @@ struct WddmResidencyControllerWithGdiTest : ::testing::Test { void SetUp() override { executionEnvironment = std::make_unique(); executionEnvironment->initializeMemoryManager(); - rootDeviceEnvironment = executionEnvironment->rootDeviceEnvironments[0].get(); + rootDeviceEnvironment = std::make_unique(*executionEnvironment); wddm = static_cast(Wddm::createWddm(nullptr, *rootDeviceEnvironment)); gdi = new MockGdi(); wddm->resetGdi(gdi); wddm->init(); mockOsContextWin = std::make_unique(*wddm, 0, osContextId, EngineDescriptorHelper::getDefaultDescriptor()); - mockOsContextWin->contextInitialized = true; wddm->getWddmInterface()->createMonitoredFence(*mockOsContextWin); csr.reset(createCommandStream(*executionEnvironment, 0u, 1)); - auto &engines = executionEnvironment->memoryManager->getRegisteredEngines(0); - const_cast(engines).emplace_back(csr.get(), mockOsContextWin.get()); residencyController = &mockOsContextWin->mockResidencyController; + residencyController->setCommandStreamReceiver(csr.get()); csr->setupContext(*mockOsContextWin); residencyController->registerCallback(); } - void TearDown() override { - auto &engines = executionEnvironment->memoryManager->getRegisteredEngines(0); - const_cast(engines).pop_back(); - } - std::unique_ptr executionEnvironment; - RootDeviceEnvironment *rootDeviceEnvironment; + std::unique_ptr rootDeviceEnvironment; WddmMock *wddm = nullptr; std::unique_ptr mockOsContextWin; std::unique_ptr csr; @@ -115,13 +109,14 @@ struct WddmResidencyControllerWithMockWddmTest : public WddmResidencyControllerT csr.reset(createCommandStream(executionEnvironment, 0u, 1)); auto &gfxCoreHelper = executionEnvironment.rootDeviceEnvironments[0]->getHelper(); - osContext = executionEnvironment.memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor(gfxCoreHelper.getGpgpuEngineInstances(*executionEnvironment.rootDeviceEnvironments[0])[0], - preemptionMode)); + osContext = memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor(gfxCoreHelper.getGpgpuEngineInstances(*executionEnvironment.rootDeviceEnvironments[0])[0], + preemptionMode)); csr->setupContext(*osContext); osContext->ensureContextInitialized(false); osContext->incRefInternal(); residencyController = &static_cast(osContext)->getResidencyController(); + residencyController->setCommandStreamReceiver(csr.get()); gmmHelper = executionEnvironment.rootDeviceEnvironments[0]->getGmmHelper(); } @@ -153,13 +148,14 @@ struct WddmResidencyControllerWithGdiAndMemoryManagerTest : ::testing::Test { memoryManager = std::make_unique(executionEnvironment); csr.reset(createCommandStream(executionEnvironment, 0u, 1)); auto &gfxCoreHelper = executionEnvironment.rootDeviceEnvironments[0]->getHelper(); - osContext = executionEnvironment.memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor(gfxCoreHelper.getGpgpuEngineInstances(*executionEnvironment.rootDeviceEnvironments[0])[0], - PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo))); + osContext = memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor(gfxCoreHelper.getGpgpuEngineInstances(*executionEnvironment.rootDeviceEnvironments[0])[0], + PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo))); osContext->ensureContextInitialized(false); osContext->incRefInternal(); residencyController = &static_cast(osContext)->getResidencyController(); + residencyController->setCommandStreamReceiver(csr.get()); csr->setupContext(*osContext); gmmHelper = executionEnvironment.rootDeviceEnvironments[0]->getGmmHelper(); } @@ -186,7 +182,6 @@ TEST(WddmResidencyController, givenWddmResidencyControllerWhenItIsConstructedThe auto wddm = static_cast(Wddm::createWddm(nullptr, *executionEnvironment.rootDeviceEnvironments[0].get())); wddm->resetGdi(gdi); wddm->init(); - EXPECT_EQ(0u, wddm->registerTrimCallbackResult.called); std::memset(&gdi->getRegisterTrimNotificationArg(), 0, sizeof(D3DKMT_REGISTERTRIMNOTIFICATION)); NEO::MockWddmResidencyController residencyController{*wddm}; @@ -205,14 +200,16 @@ TEST(WddmResidencyController, givenWddmResidencyControllerWhenRegisterCallbackTh auto gdi = new MockGdi(); auto wddm = static_cast(Wddm::createWddm(nullptr, *executionEnvironment.rootDeviceEnvironments[0].get())); wddm->resetGdi(gdi); + wddm->init(); std::memset(&gdi->getRegisterTrimNotificationArg(), 0, sizeof(D3DKMT_REGISTERTRIMNOTIFICATION)); - wddm->init(); - wddm->getResidencyController().registerCallback(); + + WddmResidencyController residencyController{*wddm}; + residencyController.registerCallback(); EXPECT_EQ(1u, wddm->registerTrimCallbackResult.called); EXPECT_EQ(reinterpret_cast(WddmResidencyController::trimCallback), gdi->getRegisterTrimNotificationArg().Callback); - EXPECT_EQ(reinterpret_cast(&wddm->getResidencyController()), gdi->getRegisterTrimNotificationArg().Context); + EXPECT_EQ(reinterpret_cast(&residencyController), gdi->getRegisterTrimNotificationArg().Context); EXPECT_EQ(wddm->getDeviceHandle(), gdi->getRegisterTrimNotificationArg().hDevice); } @@ -233,6 +230,17 @@ TEST_F(WddmResidencyControllerTest, givenWddmResidencyControllerThenUpdateLastTr EXPECT_EQ(12345u, mockOsContextWin->lastTrimFenceValue); } +TEST_F(WddmResidencyControllerWithGdiTest, givenWddmResidencyControllerWhenItIsDestructedThenUnregisterTrimCallback) { + auto trimCallbackHandle = residencyController->trimCallbackHandle; + auto trimCallbackAddress = reinterpret_cast(WddmResidencyController::trimCallback); + + std::memset(&gdi->getUnregisterTrimNotificationArg(), 0, sizeof(D3DKMT_UNREGISTERTRIMNOTIFICATION)); + mockOsContextWin.reset(); + + EXPECT_EQ(trimCallbackAddress, gdi->getUnregisterTrimNotificationArg().Callback); + EXPECT_EQ(trimCallbackHandle, gdi->getUnregisterTrimNotificationArg().Handle); +} + TEST_F(WddmResidencyControllerWithGdiTest, givenWddmResidencyControllerWhenItIsDestructedDuringProcessShutdownThenDontUnregisterTrimCallback) { wddm->shutdownStatus = true; @@ -278,15 +286,15 @@ TEST_F(WddmResidencyControllerWithGdiTest, givenNotUsedAllocationsFromPreviousPe wddm->evictResult.called = 0; wddm->callBaseEvict = true; - wddm->getResidencyController().getEvictionAllocations().push_back(&allocation1); - wddm->getResidencyController().getEvictionAllocations().push_back(&allocation2); + residencyController->getEvictionAllocations().push_back(&allocation1); + residencyController->getEvictionAllocations().push_back(&allocation2); - wddm->getResidencyController().trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); + residencyController->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); // Single evict called EXPECT_EQ(1u, wddm->evictResult.called); EXPECT_EQ(1u, gdi->getEvictArg().Flags.EvictOnlyIfNecessary); - EXPECT_TRUE(wddm->getResidencyController().getEvictionAllocations().empty()); + EXPECT_TRUE(residencyController->getEvictionAllocations().empty()); // marked nonresident EXPECT_FALSE(allocation1.getResidencyData().resident[osContextId]); EXPECT_FALSE(allocation2.getResidencyData().resident[osContextId]); @@ -369,9 +377,9 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, givenTripleAllocation wddm->evictResult.called = 0; wddm->callBaseEvict = true; - wddm->getResidencyController().getEvictionAllocations().push_back(allocationTriple); + residencyController->getEvictionAllocations().push_back(allocationTriple); - wddm->getResidencyController().trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); + residencyController->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); // 2 fragments evicted with one call EXPECT_EQ(1u, wddm->evictResult.called); @@ -743,6 +751,15 @@ TEST_F(WddmResidencyControllerLockTest, givenTrimToBudgetWhenTrimmingResidencyTh EXPECT_EQ(1u, residencyController->acquireLockCallCount); } +HWTEST_F(WddmResidencyControllerLockTest, givenTrimToBudgetWhenTrimmingToBudgetThenLockCsr) { + D3DKMT_TRIMNOTIFICATION trimNotification = {0}; + trimNotification.Flags.TrimToBudget = 1; + trimNotification.NumBytesToTrim = 0; + + residencyController->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); + EXPECT_EQ(1u, static_cast *>(residencyController->csr)->recursiveLockCounter); +} + TEST_F(WddmResidencyControllerLockTest, givenPeriodicTrimAndTrimToBudgetWhenTrimmingResidencyThenLockTwice) { D3DKMT_TRIMNOTIFICATION trimNotification = {0}; trimNotification.Flags.PeriodicTrim = 1; @@ -760,7 +777,7 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, WhenMakingResidentRes MockWddmAllocation allocation4(gmmHelper); ResidencyContainer residencyPack{&allocation1, &allocation2, &allocation3, &allocation4}; bool requiresBlockingResidencyHandling = true; - residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); + residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); EXPECT_TRUE(allocation1.getResidencyData().resident[osContextId]); EXPECT_TRUE(allocation2.getResidencyData().resident[osContextId]); @@ -777,7 +794,7 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, WhenMakingResidentRes static_cast(osContext)->getMonitoredFence().currentFenceValue = 20; bool requiresBlockingResidencyHandling = true; - residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); + residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); EXPECT_EQ(20u, allocation1.getResidencyData().getFenceValueForContextId(osContext->getContextId())); EXPECT_EQ(20u, allocation2.getResidencyData().getFenceValueForContextId(osContext->getContextId())); @@ -797,7 +814,7 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, GivenTripleAllocation WddmAllocation *allocationTriple = (WddmAllocation *)memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), false, 2 * MemoryConstants::pageSize}, ptr); ResidencyContainer residencyPack{&allocation1, allocationTriple, &allocation2}; bool requiresBlockingResidencyHandling = true; - residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); + residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); for (uint32_t i = 0; i < allocationTriple->fragmentsStorage.fragmentCount; i++) { EXPECT_TRUE(allocationTriple->fragmentsStorage.fragmentStorageData[i].residency->resident[osContextId]); @@ -817,7 +834,7 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, GivenTripleAllocation static_cast(osContext)->getMonitoredFence().currentFenceValue = 20; bool requiresBlockingResidencyHandling = true; ResidencyContainer residencyPack{&allocation1, allocationTriple, &allocation2}; - residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); + residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); for (uint32_t i = 0; i < allocationTriple->fragmentsStorage.fragmentCount; i++) { EXPECT_EQ(20u, allocationTriple->fragmentsStorage.fragmentStorageData[i].residency->getFenceValueForContextId(0)); @@ -837,7 +854,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallin ResidencyContainer residencyPack{&allocation1, &allocation2, &allocation3, &allocation4}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); EXPECT_FALSE(result); @@ -859,7 +876,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallin ResidencyContainer residencyPack{&allocation1, &allocation2, &allocation3, &allocation4}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); EXPECT_FALSE(result); EXPECT_EQ(residencyPack.size(), 0u); @@ -878,7 +895,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallin ResidencyContainer residencyPack{&allocation1, allocationTriple, &allocation2}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); EXPECT_FALSE(result); @@ -900,7 +917,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallin ResidencyContainer residencyPack{&allocation1}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); EXPECT_FALSE(result); EXPECT_NE(wddm->makeResidentParamsPassed[0].cantTrimFurther, wddm->makeResidentParamsPassed[1].cantTrimFurther); @@ -915,7 +932,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenAllocationPackPassedWhenCal allocation2.handle = 2; ResidencyContainer residencyPack{&allocation1, &allocation2}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); EXPECT_TRUE(result); EXPECT_EQ(2 * EngineLimits::maxHandleCount, wddm->makeResidentResult.handleCount); EXPECT_EQ(false, wddm->makeResidentResult.cantTrimFurther); @@ -940,7 +957,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsAndTrimToB ResidencyContainer residencyPack{&allocation1}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); EXPECT_TRUE(result); @@ -966,7 +983,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsAndTrimToB ResidencyContainer residencyPack{&allocation1, &allocationAlreadyResident}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); EXPECT_TRUE(result); @@ -983,7 +1000,7 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsWhenCallin wddm->makeResidentResults = {false, true}; bool requiresBlockingResidencyHandling = true; - residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); + residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); EXPECT_TRUE(residencyController->isMemoryBudgetExhausted()); EXPECT_EQ(2u, wddm->makeResidentResult.called); } @@ -1020,6 +1037,7 @@ struct WddmResidencyControllerWithMockWddmMakeResidentTest : public WddmResidenc osContext->incRefInternal(); residencyController = &static_cast(osContext)->getResidencyController(); + residencyController->setCommandStreamReceiver(csr.get()); csr->setupContext(*osContext); gmmHelper = executionEnvironment.rootDeviceEnvironments[0]->getGmmHelper(); } @@ -1034,7 +1052,7 @@ TEST_F(WddmResidencyControllerWithMockWddmMakeResidentTest, givenMakeResidentFai MockWddmAllocation allocation1(gmmHelper); ResidencyContainer residencyPack{&allocation1}; bool requiresBlockingResidencyHandling = true; - bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, csr.get()); + bool result = residencyController->makeResidentResidencyAllocations(residencyPack, requiresBlockingResidencyHandling, *static_cast(osContext)); EXPECT_TRUE(result); EXPECT_EQ(3u, wddm->makeResidentResult.called); diff --git a/shared/test/unit_test/os_interface/windows/wddm_windows_tests.cpp b/shared/test/unit_test/os_interface/windows/wddm_windows_tests.cpp index 16c9a9272c..c40d8eda86 100644 --- a/shared/test/unit_test/os_interface/windows/wddm_windows_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/wddm_windows_tests.cpp @@ -96,7 +96,8 @@ TEST(Wddm20EnumAdaptersTest, givenEmptyHardwareInfoWhenEnumAdapterIsCalledThenCa } TEST_F(Wddm20Tests, givenSuccessWhenRegisteringTrimCallbackThenReturnTrimCallbackHandle) { - auto trimCallbackHandle = wddm->registerTrimCallback([](D3DKMT_TRIMNOTIFICATION *) {}); + WddmResidencyController residencyController{*wddm}; + auto trimCallbackHandle = wddm->registerTrimCallback([](D3DKMT_TRIMNOTIFICATION *) {}, residencyController); EXPECT_NE(nullptr, trimCallbackHandle); }