From dc4de3c31b9939c07267069a358c706d8428bc94 Mon Sep 17 00:00:00 2001 From: Maciej Dziuban Date: Fri, 26 Oct 2018 12:22:47 +0200 Subject: [PATCH] Move trimResidency functions to WddmResidencyController Change-Id: I046fd34d5336b767ed38eda31e58e4a35ceee5f8 Signed-off-by: Maciej Dziuban --- .../os_interface/windows/os_context_win.cpp | 2 +- runtime/os_interface/windows/wddm/wddm.cpp | 6 +- runtime/os_interface/windows/wddm/wddm.h | 2 +- .../windows/wddm/wddm_interface.cpp | 8 +- .../windows/wddm/wddm_interface.h | 3 +- .../windows/wddm_device_command_stream.inl | 2 +- .../windows/wddm_memory_manager.cpp | 150 +----- .../windows/wddm_memory_manager.h | 2 - .../windows/wddm_residency_controller.cpp | 142 ++++++ .../windows/wddm_residency_controller.h | 5 +- unit_tests/mocks/mock_wddm.cpp | 4 +- unit_tests/mocks/mock_wddm.h | 2 +- .../windows/mock_wddm_memory_manager.h | 2 - .../os_interface/windows/wddm20_tests.cpp | 6 +- .../os_interface/windows/wddm23_tests.cpp | 2 +- .../windows/wddm_memory_manager_tests.cpp | 399 --------------- .../wddm_residency_controller_tests.cpp | 457 +++++++++++++++++- 17 files changed, 621 insertions(+), 573 deletions(-) diff --git a/runtime/os_interface/windows/os_context_win.cpp b/runtime/os_interface/windows/os_context_win.cpp index 6a0680c4c4..8455cfbb09 100644 --- a/runtime/os_interface/windows/os_context_win.cpp +++ b/runtime/os_interface/windows/os_context_win.cpp @@ -23,7 +23,7 @@ OsContextWin::OsContextImpl(Wddm &wddm, uint32_t osContextId) : wddm(wddm), resi return; } } - initialized = wddmInterface->createMonitoredFence(*this); + initialized = wddmInterface->createMonitoredFence(this->residencyController); }; OsContextWin::~OsContextImpl() { wddm.getWddmInterface()->destroyHwQueue(hwQueueHandle); diff --git a/runtime/os_interface/windows/wddm/wddm.cpp b/runtime/os_interface/windows/wddm/wddm.cpp index 4135ed6c4b..f2d2c3157a 100644 --- a/runtime/os_interface/windows/wddm/wddm.cpp +++ b/runtime/os_interface/windows/wddm/wddm.cpp @@ -768,13 +768,13 @@ bool Wddm::waitOnGPU(D3DKMT_HANDLE context) { return status == STATUS_SUCCESS; } -bool Wddm::waitFromCpu(uint64_t lastFenceValue, OsContextWin &osContext) { +bool Wddm::waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredFence) { NTSTATUS status = STATUS_SUCCESS; - if (lastFenceValue > *osContext.getResidencyController().getMonitoredFence().cpuAddress) { + if (lastFenceValue > *monitoredFence.cpuAddress) { D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMCPU waitFromCpu = {0}; waitFromCpu.ObjectCount = 1; - waitFromCpu.ObjectHandleArray = &osContext.getResidencyController().getMonitoredFence().fenceHandle; + waitFromCpu.ObjectHandleArray = &monitoredFence.fenceHandle; waitFromCpu.FenceValueArray = &lastFenceValue; waitFromCpu.hDevice = device; waitFromCpu.hAsyncEvent = NULL; diff --git a/runtime/os_interface/windows/wddm/wddm.h b/runtime/os_interface/windows/wddm/wddm.h index a7008da18b..f655ab3bdc 100644 --- a/runtime/os_interface/windows/wddm/wddm.h +++ b/runtime/os_interface/windows/wddm/wddm.h @@ -76,7 +76,7 @@ class Wddm { MOCKABLE_VIRTUAL bool queryAdapterInfo(); MOCKABLE_VIRTUAL bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, OsContextWin &osContext); - MOCKABLE_VIRTUAL bool waitFromCpu(uint64_t lastFenceValue, OsContextWin &osContext); + MOCKABLE_VIRTUAL bool waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredFence); NTSTATUS escape(D3DKMT_ESCAPE &escapeCommand); VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmMemoryManager *memoryManager); diff --git a/runtime/os_interface/windows/wddm/wddm_interface.cpp b/runtime/os_interface/windows/wddm/wddm_interface.cpp index a6d6b329e2..672fbadede 100644 --- a/runtime/os_interface/windows/wddm/wddm_interface.cpp +++ b/runtime/os_interface/windows/wddm/wddm_interface.cpp @@ -18,7 +18,7 @@ bool WddmInterface20::createHwQueue(PreemptionMode preemptionMode, OsContextWin } void WddmInterface20::destroyHwQueue(D3DKMT_HANDLE hwQueue) {} -bool WddmInterface::createMonitoredFence(OsContextWin &osContext) { +bool WddmInterface::createMonitoredFence(WddmResidencyController &residencyController) { NTSTATUS Status; D3DKMT_CREATESYNCHRONIZATIONOBJECT2 CreateSynchronizationObject = {0}; CreateSynchronizationObject.hDevice = wddm.getDevice(); @@ -29,9 +29,9 @@ bool WddmInterface::createMonitoredFence(OsContextWin &osContext) { DEBUG_BREAK_IF(STATUS_SUCCESS != Status); - osContext.getResidencyController().resetMonitoredFenceParams(CreateSynchronizationObject.hSyncObject, - reinterpret_cast(CreateSynchronizationObject.Info.MonitoredFence.FenceValueCPUVirtualAddress), - CreateSynchronizationObject.Info.MonitoredFence.FenceValueGPUVirtualAddress); + residencyController.resetMonitoredFenceParams(CreateSynchronizationObject.hSyncObject, + reinterpret_cast(CreateSynchronizationObject.Info.MonitoredFence.FenceValueCPUVirtualAddress), + CreateSynchronizationObject.Info.MonitoredFence.FenceValueGPUVirtualAddress); return Status == STATUS_SUCCESS; } diff --git a/runtime/os_interface/windows/wddm/wddm_interface.h b/runtime/os_interface/windows/wddm/wddm_interface.h index c308241e0e..bf0e1a2340 100644 --- a/runtime/os_interface/windows/wddm/wddm_interface.h +++ b/runtime/os_interface/windows/wddm/wddm_interface.h @@ -15,6 +15,7 @@ namespace OCLRT { class Gdi; class Wddm; +class WddmResidencyController; using OsContextWin = OsContext::OsContextImpl; @@ -25,7 +26,7 @@ class WddmInterface { WddmInterface() = delete; virtual bool createHwQueue(PreemptionMode preemptionMode, OsContextWin &osContext) = 0; virtual void destroyHwQueue(D3DKMT_HANDLE hwQueue) = 0; - bool createMonitoredFence(OsContextWin &osContext); + bool createMonitoredFence(WddmResidencyController &residencyController); virtual const bool hwQueuesSupported() = 0; virtual bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, OsContextWin &osContext) = 0; Wddm &wddm; diff --git a/runtime/os_interface/windows/wddm_device_command_stream.inl b/runtime/os_interface/windows/wddm_device_command_stream.inl index 24de8aebf2..680ed5772d 100644 --- a/runtime/os_interface/windows/wddm_device_command_stream.inl +++ b/runtime/os_interface/windows/wddm_device_command_stream.inl @@ -158,7 +158,7 @@ MemoryManager *WddmCommandStreamReceiver::createMemoryManager(bool en template bool WddmCommandStreamReceiver::waitForFlushStamp(FlushStamp &flushStampToWait, OsContext &osContext) { - return wddm->waitFromCpu(flushStampToWait, *osContext.get()); + return wddm->waitFromCpu(flushStampToWait, osContext.get()->getResidencyController().getMonitoredFence()); } template diff --git a/runtime/os_interface/windows/wddm_memory_manager.cpp b/runtime/os_interface/windows/wddm_memory_manager.cpp index 70318e24ee..119b4c9c47 100644 --- a/runtime/os_interface/windows/wddm_memory_manager.cpp +++ b/runtime/os_interface/windows/wddm_memory_manager.cpp @@ -63,7 +63,7 @@ void APIENTRY WddmMemoryManager::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *t } wddmMemMngr->getRegisteredOsContext(0)->get()->getResidencyController().acquireTrimCallbackLock(); - wddmMemMngr->trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim); + wddmMemMngr->getRegisteredOsContext(0)->get()->getResidencyController().trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim); wddmMemMngr->getRegisteredOsContext(0)->get()->getResidencyController().releaseTrimCallbackLock(); } @@ -526,7 +526,7 @@ bool WddmMemoryManager::makeResidentResidencyAllocations(ResidencyContainer &all uint64_t bytesToTrim = 0; while ((result = wddm->makeResident(handlesForResidency.get(), totalHandlesCount, false, &bytesToTrim)) == false) { this->memoryBudgetExhausted = true; - bool trimmingDone = trimResidencyToBudget(bytesToTrim); + bool trimmingDone = this->getRegisteredOsContext(0u)->get()->getResidencyController().trimResidencyToBudget(bytesToTrim); bool cantTrimFurther = !trimmingDone; if (cantTrimFurther) { result = wddm->makeResident(handlesForResidency.get(), totalHandlesCount, true, &bytesToTrim); @@ -571,152 +571,6 @@ void WddmMemoryManager::makeNonResidentEvictionAllocations(ResidencyContainer &e osContext.get()->getResidencyController().releaseLock(); } -void WddmMemoryManager::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes) { - OsContext &osContext = *getRegisteredOsContext(0); - if (flags.PeriodicTrim) { - bool periodicTrimDone = false; - D3DKMT_HANDLE fragmentEvictHandles[3] = {0}; - uint64_t sizeToTrim = 0; - - osContext.get()->getResidencyController().acquireLock(); - - WddmAllocation *wddmAllocation = nullptr; - while ((wddmAllocation = osContext.get()->getResidencyController().getTrimCandidateHead()) != nullptr) { - - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", osContext.get()->getResidencyController().getLastTrimFenceValue()); - - // allocation was not used from last periodic trim - if (wddmAllocation->getResidencyData().getFenceValueForContextId(0) <= osContext.get()->getResidencyController().getLastTrimFenceValue()) { - - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation: handle =", wddmAllocation->handle, "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(0)); - - uint32_t fragmentsToEvict = 0; - - if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation: handle =", wddmAllocation->handle, "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(0)); - wddm->evict(&wddmAllocation->handle, 1, sizeToTrim); - } - - for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { - if (wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(0) <= osContext.get()->getResidencyController().getLastTrimFenceValue()) { - - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle, "lastFence =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(0)); - - fragmentEvictHandles[fragmentsToEvict++] = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle; - wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->resident = false; - } - } - - if (fragmentsToEvict != 0) { - wddm->evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim); - } - - wddmAllocation->getResidencyData().resident = false; - - osContext.get()->getResidencyController().removeFromTrimCandidateList(wddmAllocation, false); - } else { - periodicTrimDone = true; - break; - } - } - - if (osContext.get()->getResidencyController().checkTrimCandidateListCompaction()) { - osContext.get()->getResidencyController().compactTrimCandidateList(); - } - - osContext.get()->getResidencyController().releaseLock(); - } - - if (flags.TrimToBudget) { - - osContext.get()->getResidencyController().acquireLock(); - - trimResidencyToBudget(bytes); - - osContext.get()->getResidencyController().releaseLock(); - } - - if (flags.PeriodicTrim || flags.RestartPeriodicTrim) { - const auto newPeriodicTrimFenceValue = *osContext.get()->getResidencyController().getMonitoredFence().cpuAddress; - osContext.get()->getResidencyController().setLastTrimFenceValue(newPeriodicTrimFenceValue); - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", newPeriodicTrimFenceValue); - } -} - -bool WddmMemoryManager::trimResidencyToBudget(uint64_t bytes) { - bool trimToBudgetDone = false; - D3DKMT_HANDLE fragmentEvictHandles[3] = {0}; - uint64_t numberOfBytesToTrim = bytes; - WddmAllocation *wddmAllocation = nullptr; - auto &osContext = *getRegisteredOsContext(0); - - trimToBudgetDone = (numberOfBytesToTrim == 0); - - while (!trimToBudgetDone) { - uint64_t lastFence = 0; - wddmAllocation = osContext.get()->getResidencyController().getTrimCandidateHead(); - - if (wddmAllocation == nullptr) { - break; - } - - lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(0); - auto &monitoredFence = osContext.get()->getResidencyController().getMonitoredFence(); - - if (lastFence <= monitoredFence.lastSubmittedFence) { - uint32_t fragmentsToEvict = 0; - uint64_t sizeEvicted = 0; - uint64_t sizeToTrim = 0; - - if (lastFence > *monitoredFence.cpuAddress) { - wddm->waitFromCpu(lastFence, *osContext.get()); - } - - if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { - wddm->evict(&wddmAllocation->handle, 1, sizeToTrim); - - sizeEvicted = wddmAllocation->getAlignedSize(); - } else { - auto &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData; - for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { - if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(0) <= monitoredFence.lastSubmittedFence) { - fragmentEvictHandles[fragmentsToEvict++] = fragmentStorageData[allocationId].osHandleStorage->handle; - } - } - - if (fragmentsToEvict != 0) { - wddm->evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim); - - for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { - if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(0) <= monitoredFence.lastSubmittedFence) { - fragmentStorageData[allocationId].residency->resident = false; - sizeEvicted += fragmentStorageData[allocationId].fragmentSize; - } - } - } - } - - if (sizeEvicted >= numberOfBytesToTrim) { - numberOfBytesToTrim = 0; - } else { - numberOfBytesToTrim -= sizeEvicted; - } - - wddmAllocation->getResidencyData().resident = false; - osContext.get()->getResidencyController().removeFromTrimCandidateList(wddmAllocation, false); - trimToBudgetDone = (numberOfBytesToTrim == 0); - } else { - trimToBudgetDone = true; - } - } - - if (bytes > numberOfBytesToTrim && osContext.get()->getResidencyController().checkTrimCandidateListCompaction()) { - osContext.get()->getResidencyController().compactTrimCandidateList(); - } - - return numberOfBytesToTrim == 0; -} - bool WddmMemoryManager::mapAuxGpuVA(GraphicsAllocation *graphicsAllocation) { return wddm->updateAuxTable(graphicsAllocation->getGpuAddress(), graphicsAllocation->gmm, true); } diff --git a/runtime/os_interface/windows/wddm_memory_manager.h b/runtime/os_interface/windows/wddm_memory_manager.h index cc1461f773..dc56abb714 100644 --- a/runtime/os_interface/windows/wddm_memory_manager.h +++ b/runtime/os_interface/windows/wddm_memory_manager.h @@ -79,8 +79,6 @@ class WddmMemoryManager : public MemoryManager { AlignedMallocRestrictions *getAlignedMallocRestrictions() override; protected: - void trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes); - bool trimResidencyToBudget(uint64_t bytes); VOID *trimCallbackHandle = nullptr; GraphicsAllocation *createAllocationFromHandle(osHandle handle, bool requireSpecificBitness, bool ntHandle); diff --git a/runtime/os_interface/windows/wddm_residency_controller.cpp b/runtime/os_interface/windows/wddm_residency_controller.cpp index 9f4726b120..5d13b8e28b 100644 --- a/runtime/os_interface/windows/wddm_residency_controller.cpp +++ b/runtime/os_interface/windows/wddm_residency_controller.cpp @@ -172,4 +172,146 @@ void WddmResidencyController::resetMonitoredFenceParams(D3DKMT_HANDLE &handle, u monitoredFence.gpuAddress = gpuAddress; } +void WddmResidencyController::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes) { + if (flags.PeriodicTrim) { + bool periodicTrimDone = false; + D3DKMT_HANDLE fragmentEvictHandles[3] = {0}; + uint64_t sizeToTrim = 0; + + WddmAllocation *wddmAllocation = nullptr; + while ((wddmAllocation = this->getTrimCandidateHead()) != nullptr) { + + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", this->getLastTrimFenceValue()); + + // allocation was not used from last periodic trim + if (wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId) <= this->getLastTrimFenceValue()) { + + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation: handle =", wddmAllocation->handle, "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(osContextId)); + + uint32_t fragmentsToEvict = 0; + + if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation: handle =", wddmAllocation->handle, "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(osContextId)); + this->wddm.evict(&wddmAllocation->handle, 1, sizeToTrim); + } + + for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { + if (wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= this->getLastTrimFenceValue()) { + + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle, "lastFence =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId)); + + fragmentEvictHandles[fragmentsToEvict++] = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle; + wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->resident = false; + } + } + + if (fragmentsToEvict != 0) { + this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim); + } + + wddmAllocation->getResidencyData().resident = false; + + this->removeFromTrimCandidateList(wddmAllocation, false); + } else { + periodicTrimDone = true; + break; + } + } + + if (this->checkTrimCandidateListCompaction()) { + this->compactTrimCandidateList(); + } + + this->releaseLock(); + } + + if (flags.TrimToBudget) { + + this->acquireLock(); + + trimResidencyToBudget(bytes); + + this->releaseLock(); + } + + if (flags.PeriodicTrim || flags.RestartPeriodicTrim) { + const auto newPeriodicTrimFenceValue = *this->getMonitoredFence().cpuAddress; + this->setLastTrimFenceValue(newPeriodicTrimFenceValue); + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", newPeriodicTrimFenceValue); + } +} + +bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes) { + bool trimToBudgetDone = false; + D3DKMT_HANDLE fragmentEvictHandles[3] = {0}; + uint64_t numberOfBytesToTrim = bytes; + WddmAllocation *wddmAllocation = nullptr; + + trimToBudgetDone = (numberOfBytesToTrim == 0); + + while (!trimToBudgetDone) { + uint64_t lastFence = 0; + wddmAllocation = this->getTrimCandidateHead(); + + if (wddmAllocation == nullptr) { + break; + } + + lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId); + auto &monitoredFence = this->getMonitoredFence(); + + if (lastFence <= monitoredFence.lastSubmittedFence) { + uint32_t fragmentsToEvict = 0; + uint64_t sizeEvicted = 0; + uint64_t sizeToTrim = 0; + + if (lastFence > *monitoredFence.cpuAddress) { + this->wddm.waitFromCpu(lastFence, this->getMonitoredFence()); + } + + if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { + this->wddm.evict(&wddmAllocation->handle, 1, sizeToTrim); + + sizeEvicted = wddmAllocation->getAlignedSize(); + } else { + auto &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData; + for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { + if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) { + fragmentEvictHandles[fragmentsToEvict++] = fragmentStorageData[allocationId].osHandleStorage->handle; + } + } + + if (fragmentsToEvict != 0) { + this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim); + + for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { + if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) { + fragmentStorageData[allocationId].residency->resident = false; + sizeEvicted += fragmentStorageData[allocationId].fragmentSize; + } + } + } + } + + if (sizeEvicted >= numberOfBytesToTrim) { + numberOfBytesToTrim = 0; + } else { + numberOfBytesToTrim -= sizeEvicted; + } + + wddmAllocation->getResidencyData().resident = false; + this->removeFromTrimCandidateList(wddmAllocation, false); + trimToBudgetDone = (numberOfBytesToTrim == 0); + } else { + trimToBudgetDone = true; + } + } + + if (bytes > numberOfBytesToTrim && this->checkTrimCandidateListCompaction()) { + this->compactTrimCandidateList(); + } + + return numberOfBytesToTrim == 0; +} + } // namespace OCLRT diff --git a/runtime/os_interface/windows/wddm_residency_controller.h b/runtime/os_interface/windows/wddm_residency_controller.h index 8513999085..330e8e8bbe 100644 --- a/runtime/os_interface/windows/wddm_residency_controller.h +++ b/runtime/os_interface/windows/wddm_residency_controller.h @@ -46,9 +46,13 @@ class WddmResidencyController { MonitoredFence &getMonitoredFence() { return monitoredFence; } void resetMonitoredFenceParams(D3DKMT_HANDLE &handle, uint64_t *cpuAddress, D3DGPU_VIRTUAL_ADDRESS &gpuAddress); + void trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes); + bool trimResidencyToBudget(uint64_t bytes); + protected: Wddm &wddm; uint32_t osContextId; + MonitoredFence monitoredFence = {}; std::atomic lock = false; std::atomic_flag trimCallbackLock = ATOMIC_FLAG_INIT; @@ -56,6 +60,5 @@ class WddmResidencyController { uint64_t lastTrimFenceValue = 0u; ResidencyContainer trimCandidateList; uint32_t trimCandidatesCount = 0; - MonitoredFence monitoredFence = {}; }; } // namespace OCLRT diff --git a/unit_tests/mocks/mock_wddm.cpp b/unit_tests/mocks/mock_wddm.cpp index b70f782e8e..7b16a9b110 100644 --- a/unit_tests/mocks/mock_wddm.cpp +++ b/unit_tests/mocks/mock_wddm.cpp @@ -184,10 +184,10 @@ GMM_GFX_PARTITIONING *WddmMock::getGfxPartitionPtr() { return &gfxPartition; } -bool WddmMock::waitFromCpu(uint64_t lastFenceValue, OsContextWin &osContext) { +bool WddmMock::waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredFence) { waitFromCpuResult.called++; waitFromCpuResult.uint64ParamPassed = lastFenceValue; - return waitFromCpuResult.success = Wddm::waitFromCpu(lastFenceValue, osContext); + return waitFromCpuResult.success = Wddm::waitFromCpu(lastFenceValue, monitoredFence); } void *WddmMock::virtualAlloc(void *inPtr, size_t size, unsigned long flags, unsigned long type) { diff --git a/unit_tests/mocks/mock_wddm.h b/unit_tests/mocks/mock_wddm.h index e8c76cf38b..0c8afae4bf 100644 --- a/unit_tests/mocks/mock_wddm.h +++ b/unit_tests/mocks/mock_wddm.h @@ -73,7 +73,7 @@ class WddmMock : public Wddm { bool openAdapter() override; void setHeap32(uint64_t base, uint64_t size); GMM_GFX_PARTITIONING *getGfxPartitionPtr(); - bool waitFromCpu(uint64_t lastFenceValue, OsContextWin &osContext) override; + bool waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredFence) override; void *virtualAlloc(void *inPtr, size_t size, unsigned long flags, unsigned long type) override; int virtualFree(void *ptr, size_t size, unsigned long flags) override; void releaseReservedAddress(void *reservedAddress) override; diff --git a/unit_tests/os_interface/windows/mock_wddm_memory_manager.h b/unit_tests/os_interface/windows/mock_wddm_memory_manager.h index f75ad55e38..fd88d41446 100644 --- a/unit_tests/os_interface/windows/mock_wddm_memory_manager.h +++ b/unit_tests/os_interface/windows/mock_wddm_memory_manager.h @@ -17,8 +17,6 @@ class MockWddmMemoryManager : public WddmMemoryManager { public: using BaseClass::createWddmAllocation; using BaseClass::trimCallbackHandle; - using BaseClass::trimResidency; - using BaseClass::trimResidencyToBudget; using BaseClass::WddmMemoryManager; MockWddmMemoryManager(Wddm *wddm, ExecutionEnvironment &executionEnvironment) : WddmMemoryManager(false, false, wddm, executionEnvironment) { diff --git a/unit_tests/os_interface/windows/wddm20_tests.cpp b/unit_tests/os_interface/windows/wddm20_tests.cpp index 1dd362fc69..8ca2f4dbd4 100644 --- a/unit_tests/os_interface/windows/wddm20_tests.cpp +++ b/unit_tests/os_interface/windows/wddm20_tests.cpp @@ -670,7 +670,7 @@ TEST_F(Wddm20Tests, WhenLastFenceLessEqualThanMonitoredThenWaitFromCpuIsNotCalle gdi->getWaitFromCpuArg().ObjectCount = 0; gdi->getWaitFromCpuArg().ObjectHandleArray = nullptr; - auto status = wddm->waitFromCpu(10, *osContextWin); + auto status = wddm->waitFromCpu(10, osContextWin->getResidencyController().getMonitoredFence()); EXPECT_TRUE(status); @@ -693,7 +693,7 @@ TEST_F(Wddm20Tests, WhenLastFenceGreaterThanMonitoredThenWaitFromCpuIsCalled) { gdi->getWaitFromCpuArg().ObjectCount = 0; gdi->getWaitFromCpuArg().ObjectHandleArray = nullptr; - auto status = wddm->waitFromCpu(20, *osContextWin); + auto status = wddm->waitFromCpu(20, osContextWin->getResidencyController().getMonitoredFence()); EXPECT_TRUE(status); @@ -708,7 +708,7 @@ TEST_F(Wddm20Tests, createMonitoredFenceIsInitializedWithFenceValueZeroAndCurren gdi->getCreateSynchronizationObject2Arg().Info.MonitoredFence.InitialFenceValue = 300; - wddm->wddmInterface->createMonitoredFence(*osContextWin); + wddm->wddmInterface->createMonitoredFence(osContextWin->getResidencyController()); EXPECT_EQ(0u, gdi->getCreateSynchronizationObject2Arg().Info.MonitoredFence.InitialFenceValue); EXPECT_EQ(1u, osContextWin->getResidencyController().getMonitoredFence().currentFenceValue); diff --git a/unit_tests/os_interface/windows/wddm23_tests.cpp b/unit_tests/os_interface/windows/wddm23_tests.cpp index cbd1a4d9e1..d99ca9465f 100644 --- a/unit_tests/os_interface/windows/wddm23_tests.cpp +++ b/unit_tests/os_interface/windows/wddm23_tests.cpp @@ -118,7 +118,7 @@ TEST_F(Wddm23Tests, givenCmdBufferWhenSubmitCalledThenSetAllRequiredFiledsAndUpd } TEST_F(Wddm23Tests, whenMonitoredFenceIsCreatedThenSetupAllRequiredFields) { - wddm->wddmInterface->createMonitoredFence(*osContextWin); + wddm->wddmInterface->createMonitoredFence(osContextWin->getResidencyController()); EXPECT_NE(nullptr, osContextWin->getResidencyController().getMonitoredFence().cpuAddress); EXPECT_EQ(1u, osContextWin->getResidencyController().getMonitoredFence().currentFenceValue); diff --git a/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp b/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp index c247e53840..9506649dd1 100644 --- a/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp +++ b/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp @@ -953,405 +953,6 @@ TEST_F(WddmMemoryManagerResidencyTest, givenWddmMemoryManagerWhenCallingDestruct EXPECT_EQ(trimCallbackHandle, unregisterNotification.Handle); } -TEST_F(WddmMemoryManagerResidencyTest, givenNotUsedAllocationsFromPreviousPeriodicTrimWhenTrimResidencyPeriodicTrimIsCalledThenAllocationsAreEvictedMarkedAndRemovedFromTrimCandidateList) { - D3DKMT_TRIMNOTIFICATION trimNotification = {0}; - trimNotification.Flags.PeriodicTrim = 1; - trimNotification.NumBytesToTrim = 0; - - // allocations have fence value == 0 by default - MockWddmAllocation allocation1, allocation2; - - allocation1.getResidencyData().updateCompletionData(0, osContext->getContextId()); - allocation2.getResidencyData().updateCompletionData(0, osContext->getContextId()); - - allocation1.getResidencyData().resident = true; - allocation2.getResidencyData().resident = true; - - // Set last periodic fence value - osContext->get()->getResidencyController().setLastTrimFenceValue(10); - osContext->get()->getResidencyController().setLastTrimFenceValue(10); - // Set current fence value to greater value - osContext->get()->getResidencyController().getMonitoredFence().currentFenceValue = 20; - - wddm->makeNonResidentResult.called = 0; - - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation1); - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation2); - - memoryManager->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); - - // 2 allocations evicted - EXPECT_EQ(2u, wddm->makeNonResidentResult.called); - // removed from trim candidate list - EXPECT_EQ(0u, osContext->get()->getResidencyController().peekTrimCandidateList().size()); - // marked nonresident - EXPECT_FALSE(allocation1.getResidencyData().resident); - EXPECT_FALSE(allocation2.getResidencyData().resident); -} - -TEST_F(WddmMemoryManagerResidencyTest, givenOneUsedAllocationFromPreviousPeriodicTrimWhenTrimResidencyPeriodicTrimIsCalledThenOneAllocationIsTrimmed) { - D3DKMT_TRIMNOTIFICATION trimNotification = {0}; - trimNotification.Flags.PeriodicTrim = 1; - trimNotification.NumBytesToTrim = 0; - - // allocations have fence value == 0 by default - MockWddmAllocation allocation1, allocation2; - allocation1.getResidencyData().resident = true; - // mark allocation used from last periodic trim - allocation1.getResidencyData().updateCompletionData(0, osContext->getContextId()); - allocation2.getResidencyData().updateCompletionData(11, osContext->getContextId()); - allocation2.getResidencyData().resident = true; - - // Set last periodic fence value - osContext->get()->getResidencyController().setLastTrimFenceValue(10); - // Set current fence value to greater value - osContext->get()->getResidencyController().getMonitoredFence().currentFenceValue = 20; - - wddm->makeNonResidentResult.called = 0; - - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation1); - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation2); - - memoryManager->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); - - // 1 allocation evicted - EXPECT_EQ(1u, wddm->makeNonResidentResult.called); - // removed from trim candidate list - EXPECT_EQ(trimListUnusedPosition, allocation1.getTrimCandidateListPosition(osContext->getContextId())); - - //marked nonresident - EXPECT_FALSE(allocation1.getResidencyData().resident); - // second stays resident - EXPECT_TRUE(allocation2.getResidencyData().resident); -} - -TEST_F(WddmMemoryManagerResidencyTest, givenTripleAllocationWithUsedAndUnusedFragmentsSincePreviousTrimWhenTrimResidencyPeriodicTrimIsCalledThenProperFragmentsAreEvictedAndMarked) { - D3DKMT_TRIMNOTIFICATION trimNotification = {0}; - trimNotification.Flags.PeriodicTrim = 1; - trimNotification.NumBytesToTrim = 0; - void *ptr = reinterpret_cast(wddm->virtualAllocAddress + 0x1500); - // 3-fragment Allocation - WddmAllocation *allocationTriple = (WddmAllocation *)memoryManager->allocateGraphicsMemory(8196, ptr); - // whole allocation unused since previous trim - allocationTriple->getResidencyData().updateCompletionData(0, osContext->getContextId()); - - EXPECT_EQ(3u, allocationTriple->fragmentsStorage.fragmentCount); - - allocationTriple->fragmentsStorage.fragmentStorageData[0].residency->updateCompletionData(0, osContext->getContextId()); - allocationTriple->fragmentsStorage.fragmentStorageData[0].residency->resident = true; - // this fragment was used - allocationTriple->fragmentsStorage.fragmentStorageData[1].residency->updateCompletionData(11, osContext->getContextId()); - allocationTriple->fragmentsStorage.fragmentStorageData[0].residency->resident = true; - - allocationTriple->fragmentsStorage.fragmentStorageData[2].residency->updateCompletionData(0, osContext->getContextId()); - allocationTriple->fragmentsStorage.fragmentStorageData[2].residency->resident = true; - - // Set last periodic fence value - osContext->get()->getResidencyController().setLastTrimFenceValue(10); - // Set current fence value to greater value - osContext->get()->getResidencyController().getMonitoredFence().currentFenceValue = 20; - - wddm->makeNonResidentResult.called = 0; - - osContext->get()->getResidencyController().addToTrimCandidateList(allocationTriple); - - memoryManager->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); - - // 2 fragments evicted with one call - EXPECT_EQ(1u, wddm->makeNonResidentResult.called); - // marked nonresident - EXPECT_FALSE(allocationTriple->fragmentsStorage.fragmentStorageData[0].residency->resident); - EXPECT_FALSE(allocationTriple->fragmentsStorage.fragmentStorageData[2].residency->resident); - - memoryManager->freeGraphicsMemory(allocationTriple); -} - -TEST_F(WddmMemoryManagerResidencyTest, givenPeriodicTrimWhenTrimCallbackCalledThenLastPeriodicTrimFenceIsSetToCurrentFenceValue) { - platform()->peekExecutionEnvironment()->osInterface = std::move(osInterface); - platform()->initialize(); - D3DKMT_TRIMNOTIFICATION trimNotification = {0}; - trimNotification.Flags.PeriodicTrim = 1; - trimNotification.NumBytesToTrim = 0; - - // Set last periodic fence value - osContext->get()->getResidencyController().setLastTrimFenceValue(10); - // Set current fence value to greater value - *osContext->get()->getResidencyController().getMonitoredFence().cpuAddress = 20; - - memoryManager->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); - - EXPECT_EQ(20u, osContext->get()->getResidencyController().getLastTrimFenceValue()); -} - -TEST_F(WddmMemoryManagerResidencyTest, givenRestartPeriodicTrimWhenTrimCallbackCalledThenLastPeriodicTrimFenceIsSetToCurrentFenceValue) { - platform()->peekExecutionEnvironment()->osInterface = std::move(osInterface); - platform()->initialize(); - D3DKMT_TRIMNOTIFICATION trimNotification = {0}; - trimNotification.Flags.RestartPeriodicTrim = 1; - trimNotification.NumBytesToTrim = 0; - - // Set last periodic fence value - osContext->get()->getResidencyController().setLastTrimFenceValue(10); - // Set current fence value to greater value - *osContext->get()->getResidencyController().getMonitoredFence().cpuAddress = 20; - - memoryManager->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); - - EXPECT_EQ(20u, osContext->get()->getResidencyController().getLastTrimFenceValue()); -} - -TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetWithZeroSizeReturnsTrue) { - bool status = memoryManager->trimResidencyToBudget(0); - - EXPECT_TRUE(status); -} - -TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetAllDoneAllocations) { - gdi->setNonZeroNumBytesToTrimInEvict(); - - MockWddmAllocation allocation1, allocation2, allocation3; - - allocation1.getResidencyData().resident = true; - allocation1.getResidencyData().updateCompletionData(0, osContext->getContextId()); - - allocation2.getResidencyData().updateCompletionData(1, osContext->getContextId()); - allocation2.getResidencyData().resident = true; - - allocation3.getResidencyData().updateCompletionData(2, osContext->getContextId()); - allocation3.getResidencyData().resident = true; - - *osContext->get()->getResidencyController().getMonitoredFence().cpuAddress = 1; - osContext->get()->getResidencyController().getMonitoredFence().lastSubmittedFence = 1; - osContext->get()->getResidencyController().getMonitoredFence().currentFenceValue = 1; - - wddm->makeNonResidentResult.called = 0; - - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation1); - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation2); - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation3); - - memoryManager->trimResidencyToBudget(3 * 4096); - - EXPECT_EQ(2u, wddm->makeNonResidentResult.called); - - EXPECT_EQ(1u, osContext->get()->getResidencyController().peekTrimCandidatesCount()); - osContext->get()->getResidencyController().compactTrimCandidateList(); - EXPECT_EQ(1u, osContext->get()->getResidencyController().peekTrimCandidateList().size()); - - EXPECT_EQ(trimListUnusedPosition, allocation1.getTrimCandidateListPosition(osContext->getContextId())); - EXPECT_EQ(trimListUnusedPosition, allocation2.getTrimCandidateListPosition(osContext->getContextId())); - EXPECT_NE(trimListUnusedPosition, allocation3.getTrimCandidateListPosition(osContext->getContextId())); -} - -TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetReturnsFalseWhenNumBytesToTrimIsNotZero) { - gdi->setNonZeroNumBytesToTrimInEvict(); - - MockWddmAllocation allocation1; - - allocation1.getResidencyData().resident = true; - allocation1.getResidencyData().updateCompletionData(0, osContext->getContextId()); - - *osContext->get()->getResidencyController().getMonitoredFence().cpuAddress = 1; - osContext->get()->getResidencyController().getMonitoredFence().lastSubmittedFence = 1; - - wddm->makeNonResidentResult.called = 0; - - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation1); - - bool status = memoryManager->trimResidencyToBudget(3 * 4096); - - EXPECT_EQ(1u, wddm->makeNonResidentResult.called); - EXPECT_EQ(0u, osContext->get()->getResidencyController().peekTrimCandidateList().size()); - - EXPECT_FALSE(status); -} - -TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetStopsEvictingWhenNumBytesToTrimIsZero) { - WddmAllocation allocation1(reinterpret_cast(0x1000), 0x1000, reinterpret_cast(0x1000), 0x1000, nullptr, MemoryPool::MemoryNull, memoryManager->getOsContextCount()), - allocation2(reinterpret_cast(0x1000), 0x3000, reinterpret_cast(0x1000), 0x3000, nullptr, MemoryPool::MemoryNull, memoryManager->getOsContextCount()), - allocation3(reinterpret_cast(0x1000), 0x1000, reinterpret_cast(0x1000), 0x1000, nullptr, MemoryPool::MemoryNull, memoryManager->getOsContextCount()); - - allocation1.getResidencyData().resident = true; - allocation1.getResidencyData().updateCompletionData(0, osContext->getContextId()); - - allocation2.getResidencyData().updateCompletionData(1, osContext->getContextId()); - allocation2.getResidencyData().resident = true; - - allocation3.getResidencyData().updateCompletionData(2, osContext->getContextId()); - allocation3.getResidencyData().resident = true; - - *osContext->get()->getResidencyController().getMonitoredFence().cpuAddress = 1; - osContext->get()->getResidencyController().getMonitoredFence().lastSubmittedFence = 1; - osContext->get()->getResidencyController().getMonitoredFence().currentFenceValue = 1; - - wddm->makeNonResidentResult.called = 0; - - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation1); - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation2); - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation3); - - bool status = memoryManager->trimResidencyToBudget(3 * 4096); - - EXPECT_TRUE(status); - EXPECT_EQ(2u, wddm->makeNonResidentResult.called); - EXPECT_EQ(1u, osContext->get()->getResidencyController().peekTrimCandidateList().size()); - - EXPECT_EQ(trimListUnusedPosition, allocation1.getTrimCandidateListPosition(osContext->getContextId())); - EXPECT_EQ(trimListUnusedPosition, allocation2.getTrimCandidateListPosition(osContext->getContextId())); - EXPECT_NE(trimListUnusedPosition, allocation3.getTrimCandidateListPosition(osContext->getContextId())); -} - -TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetMarksEvictedAllocationNonResident) { - gdi->setNonZeroNumBytesToTrimInEvict(); - - MockWddmAllocation allocation1, allocation2, allocation3; - - allocation1.getResidencyData().resident = true; - allocation1.getResidencyData().updateCompletionData(0, osContext->getContextId()); - - allocation2.getResidencyData().updateCompletionData(1, osContext->getContextId()); - allocation2.getResidencyData().resident = true; - - allocation3.getResidencyData().updateCompletionData(2, osContext->getContextId()); - allocation3.getResidencyData().resident = true; - - *osContext->get()->getResidencyController().getMonitoredFence().cpuAddress = 1; - osContext->get()->getResidencyController().getMonitoredFence().lastSubmittedFence = 1; - osContext->get()->getResidencyController().getMonitoredFence().currentFenceValue = 1; - - wddm->makeNonResidentResult.called = 0; - - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation1); - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation2); - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation3); - - bool status = memoryManager->trimResidencyToBudget(3 * 4096); - - EXPECT_FALSE(allocation1.getResidencyData().resident); - EXPECT_FALSE(allocation2.getResidencyData().resident); - EXPECT_TRUE(allocation3.getResidencyData().resident); -} - -TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetWaitsFromCpuWhenLastFenceIsGreaterThanMonitored) { - gdi->setNonZeroNumBytesToTrimInEvict(); - - MockWddmAllocation allocation1; - - allocation1.getResidencyData().resident = true; - allocation1.getResidencyData().updateCompletionData(2, osContext->getContextId()); - - *osContext->get()->getResidencyController().getMonitoredFence().cpuAddress = 1; - osContext->get()->getResidencyController().getMonitoredFence().lastSubmittedFence = 2; - osContext->get()->getResidencyController().getMonitoredFence().currentFenceValue = 3; - - wddm->makeNonResidentResult.called = 0; - wddm->waitFromCpuResult.called = 0; - - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation1); - - gdi->getWaitFromCpuArg().hDevice = (D3DKMT_HANDLE)0; - - bool status = memoryManager->trimResidencyToBudget(3 * 4096); - - EXPECT_EQ(1u, wddm->makeNonResidentResult.called); - EXPECT_FALSE(allocation1.getResidencyData().resident); - - EXPECT_EQ(wddm->getDevice(), gdi->getWaitFromCpuArg().hDevice); -} - -TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetEvictsDoneFragmentsOnly) { - gdi->setNonZeroNumBytesToTrimInEvict(); - void *ptr = reinterpret_cast(wddm->virtualAllocAddress + 0x1000); - WddmAllocation allocation1(ptr, 0x1000, ptr, 0x1000, nullptr, MemoryPool::MemoryNull, memoryManager->getOsContextCount()); - WddmAllocation allocation2(ptr, 0x1000, ptr, 0x1000, nullptr, MemoryPool::MemoryNull, memoryManager->getOsContextCount()); - - allocation1.getResidencyData().resident = true; - allocation1.getResidencyData().updateCompletionData(0, osContext->getContextId()); - - allocation2.getResidencyData().updateCompletionData(1, osContext->getContextId()); - allocation2.getResidencyData().resident = true; - - void *ptrTriple = reinterpret_cast(reinterpret_cast(ptr) + 0x500); - WddmAllocation *allocationTriple = static_cast(memoryManager->allocateGraphicsMemory(8196, ptrTriple)); - - allocationTriple->getResidencyData().updateCompletionData(1, osContext->getContextId()); - allocationTriple->getResidencyData().resident = true; - - EXPECT_EQ(3u, allocationTriple->fragmentsStorage.fragmentCount); - - for (uint32_t i = 0; i < 3; i++) { - allocationTriple->fragmentsStorage.fragmentStorageData[i].residency->updateCompletionData(1, osContext->getContextId()); - allocationTriple->fragmentsStorage.fragmentStorageData[i].residency->resident = true; - } - - // This should not be evicted - allocationTriple->fragmentsStorage.fragmentStorageData[1].residency->updateCompletionData(2, osContext->getContextId()); - - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation1); - osContext->get()->getResidencyController().addToTrimCandidateList(allocationTriple); - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation2); - - *osContext->get()->getResidencyController().getMonitoredFence().cpuAddress = 1; - osContext->get()->getResidencyController().getMonitoredFence().lastSubmittedFence = 1; - osContext->get()->getResidencyController().getMonitoredFence().currentFenceValue = 2; - - wddm->makeNonResidentResult.called = 0; - - bool status = memoryManager->trimResidencyToBudget(3 * 4096); - - EXPECT_EQ(2u, wddm->makeNonResidentResult.called); - - EXPECT_FALSE(allocationTriple->fragmentsStorage.fragmentStorageData[0].residency->resident); - EXPECT_TRUE(allocationTriple->fragmentsStorage.fragmentStorageData[1].residency->resident); - EXPECT_FALSE(allocationTriple->fragmentsStorage.fragmentStorageData[2].residency->resident); - - memoryManager->freeGraphicsMemory(allocationTriple); -} - -TEST_F(WddmMemoryManagerResidencyTest, givenThreeAllocationsAlignedSizeBiggerThanAllocSizeWhenBudgetEqualTwoAlignedAllocationThenEvictOnlyTwo) { - gdi->setNonZeroNumBytesToTrimInEvict(); - size_t underlyingSize = 0xF00; - size_t alignedSize = 0x1000; - size_t budget = 2 * alignedSize; - - //trim budget should consider aligned size, not underlying, so if function considers underlying, it should evict three, not two - EXPECT_GT((3 * underlyingSize), budget); - EXPECT_LT((2 * underlyingSize), budget); - void *ptr1 = reinterpret_cast(wddm->virtualAllocAddress + 0x1000); - void *ptr2 = reinterpret_cast(wddm->virtualAllocAddress + 0x3000); - void *ptr3 = reinterpret_cast(wddm->virtualAllocAddress + 0x5000); - - WddmAllocation allocation1(ptr1, underlyingSize, ptr1, alignedSize, nullptr, MemoryPool::MemoryNull, memoryManager->getOsContextCount()); - WddmAllocation allocation2(ptr2, underlyingSize, ptr2, alignedSize, nullptr, MemoryPool::MemoryNull, memoryManager->getOsContextCount()); - WddmAllocation allocation3(ptr3, underlyingSize, ptr3, alignedSize, nullptr, MemoryPool::MemoryNull, memoryManager->getOsContextCount()); - - allocation1.getResidencyData().resident = true; - allocation1.getResidencyData().updateCompletionData(0, osContext->getContextId()); - - allocation2.getResidencyData().updateCompletionData(1, osContext->getContextId()); - allocation2.getResidencyData().resident = true; - - allocation3.getResidencyData().updateCompletionData(1, osContext->getContextId()); - allocation3.getResidencyData().resident = true; - - *osContext->get()->getResidencyController().getMonitoredFence().cpuAddress = 1; - osContext->get()->getResidencyController().getMonitoredFence().lastSubmittedFence = 1; - osContext->get()->getResidencyController().getMonitoredFence().currentFenceValue = 1; - - wddm->makeNonResidentResult.called = 0; - - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation1); - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation2); - osContext->get()->getResidencyController().addToTrimCandidateList(&allocation3); - - bool status = memoryManager->trimResidencyToBudget(budget); - EXPECT_TRUE(status); - - EXPECT_FALSE(allocation1.getResidencyData().resident); - EXPECT_FALSE(allocation2.getResidencyData().resident); - EXPECT_TRUE(allocation3.getResidencyData().resident); -} - TEST_F(BufferWithWddmMemory, ValidHostPtr) { flags = CL_MEM_USE_HOST_PTR; diff --git a/unit_tests/os_interface/windows/wddm_residency_controller_tests.cpp b/unit_tests/os_interface/windows/wddm_residency_controller_tests.cpp index aa49a00fa6..8856f59cbb 100644 --- a/unit_tests/os_interface/windows/wddm_residency_controller_tests.cpp +++ b/unit_tests/os_interface/windows/wddm_residency_controller_tests.cpp @@ -5,10 +5,16 @@ * */ -#include "runtime/os_interface/windows/wddm_residency_controller.h" +#include "runtime/execution_environment/execution_environment.h" #include "runtime/os_interface/os_context.h" +#include "runtime/os_interface/windows/os_context_win.h" +#include "runtime/os_interface/os_interface.h" +#include "runtime/os_interface/windows/os_interface.h" +#include "runtime/os_interface/windows/wddm_residency_controller.h" #include "unit_tests/mocks/mock_wddm.h" +#include "unit_tests/os_interface/windows/mock_gdi_interface.h" #include "unit_tests/os_interface/windows/mock_wddm_allocation.h" +#include "unit_tests/os_interface/windows/mock_wddm_memory_manager.h" #include "test.h" #include @@ -19,6 +25,8 @@ class MockWddmResidencyController : public WddmResidencyController { public: using WddmResidencyController::trimCandidateList; using WddmResidencyController::trimCandidatesCount; + using WddmResidencyController::trimResidency; + using WddmResidencyController::trimResidencyToBudget; using WddmResidencyController::WddmResidencyController; }; @@ -26,15 +34,61 @@ struct WddmResidencyControllerTest : ::testing::Test { const uint32_t osContextId = 0u; void SetUp() { - wddm = std::unique_ptr(Wddm::createWddm()); + wddm = std::unique_ptr(static_cast(Wddm::createWddm())); wddm->init(); residencyController = std::make_unique(*wddm, osContextId); } - std::unique_ptr wddm; + std::unique_ptr wddm; std::unique_ptr residencyController; }; +struct WddmResidencyControllerWithGdiTest : public WddmResidencyControllerTest { + void SetUp() { + WddmResidencyControllerTest::SetUp(); + gdi = new MockGdi(); + wddm->gdi.reset(gdi); + wddm->getWddmInterface()->createMonitoredFence(*residencyController); + } + + MockGdi *gdi; +}; + +struct WddmResidencyControllerWithGdiAndMemoryManagerTest : ::testing::Test { + const uint32_t osContextId = 0u; + + void SetUp() { + wddm = static_cast(Wddm::createWddm()); + wddm->init(); + gdi = new MockGdi(); + wddm->gdi.reset(gdi); + + osInterface = std::make_unique(); + osInterface->get()->setWddm(wddm); + osContext = new OsContext(osInterface.get(), osContextId); + osContext->incRefInternal(); + residencyController = &osContext->get()->getResidencyController(); + + executionEnvironment = std::make_unique(); + + memoryManager = std::make_unique(wddm, *executionEnvironment); + memoryManager->registerOsContext(osContext); + } + + void TearDown() { + osContext->decRefInternal(); + } + + std::unique_ptr osInterface; + std::unique_ptr executionEnvironment; + std::unique_ptr memoryManager; + + WddmMock *wddm = nullptr; + OsContext *osContext = nullptr; + MockGdi *gdi = nullptr; + WddmResidencyController *residencyController = nullptr; +}; + TEST_F(WddmResidencyControllerTest, givenUsedAllocationWhenCallingRemoveFromTrimCandidateListIfUsedThenRemoveIt) { MockWddmAllocation allocation; residencyController->addToTrimCandidateList(&allocation); @@ -239,3 +293,400 @@ TEST_F(WddmResidencyControllerTest, checkTrimCandidateListCompaction) { comapactionRequired = residencyController->checkTrimCandidateListCompaction(); EXPECT_FALSE(comapactionRequired); } + +TEST_F(WddmResidencyControllerWithGdiTest, givenNotUsedAllocationsFromPreviousPeriodicTrimWhenTrimResidencyPeriodicTrimIsCalledThenAllocationsAreEvictedMarkedAndRemovedFromTrimCandidateList) { + D3DKMT_TRIMNOTIFICATION trimNotification = {0}; + trimNotification.Flags.PeriodicTrim = 1; + trimNotification.NumBytesToTrim = 0; + + // allocations have fence value == 0 by default + MockWddmAllocation allocation1, allocation2; + + allocation1.getResidencyData().updateCompletionData(0, osContextId); + allocation2.getResidencyData().updateCompletionData(0, osContextId); + + allocation1.getResidencyData().resident = true; + allocation2.getResidencyData().resident = true; + + // Set last periodic fence value + residencyController->setLastTrimFenceValue(10); + residencyController->setLastTrimFenceValue(10); + // Set current fence value to greater value + residencyController->getMonitoredFence().currentFenceValue = 20; + + wddm->makeNonResidentResult.called = 0; + + residencyController->addToTrimCandidateList(&allocation1); + residencyController->addToTrimCandidateList(&allocation2); + + residencyController->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); + + // 2 allocations evicted + EXPECT_EQ(2u, wddm->makeNonResidentResult.called); + // removed from trim candidate list + EXPECT_EQ(0u, residencyController->peekTrimCandidateList().size()); + // marked nonresident + EXPECT_FALSE(allocation1.getResidencyData().resident); + EXPECT_FALSE(allocation2.getResidencyData().resident); +} + +TEST_F(WddmResidencyControllerWithGdiTest, givenOneUsedAllocationFromPreviousPeriodicTrimWhenTrimResidencyPeriodicTrimIsCalledThenOneAllocationIsTrimmed) { + D3DKMT_TRIMNOTIFICATION trimNotification = {0}; + trimNotification.Flags.PeriodicTrim = 1; + trimNotification.NumBytesToTrim = 0; + + // allocations have fence value == 0 by default + MockWddmAllocation allocation1, allocation2; + allocation1.getResidencyData().resident = true; + // mark allocation used from last periodic trim + allocation1.getResidencyData().updateCompletionData(0, osContextId); + allocation2.getResidencyData().updateCompletionData(11, osContextId); + allocation2.getResidencyData().resident = true; + + // Set last periodic fence value + residencyController->setLastTrimFenceValue(10); + // Set current fence value to greater value + residencyController->getMonitoredFence().currentFenceValue = 20; + + wddm->makeNonResidentResult.called = 0; + + residencyController->addToTrimCandidateList(&allocation1); + residencyController->addToTrimCandidateList(&allocation2); + + residencyController->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); + + // 1 allocation evicted + EXPECT_EQ(1u, wddm->makeNonResidentResult.called); + // removed from trim candidate list + EXPECT_EQ(trimListUnusedPosition, allocation1.getTrimCandidateListPosition(osContextId)); + + //marked nonresident + EXPECT_FALSE(allocation1.getResidencyData().resident); + // second stays resident + EXPECT_TRUE(allocation2.getResidencyData().resident); +} + +TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, givenTripleAllocationWithUsedAndUnusedFragmentsSincePreviousTrimWhenTrimResidencyPeriodicTrimIsCalledThenProperFragmentsAreEvictedAndMarked) { + D3DKMT_TRIMNOTIFICATION trimNotification = {0}; + trimNotification.Flags.PeriodicTrim = 1; + trimNotification.NumBytesToTrim = 0; + + // 3-fragment Allocation + void *ptr = reinterpret_cast(wddm->virtualAllocAddress + 0x1500); + auto allocationTriple = static_cast(memoryManager->allocateGraphicsMemory(8196, ptr)); + + // whole allocation unused since previous trim + allocationTriple->getResidencyData().updateCompletionData(0, osContextId); + + EXPECT_EQ(3u, allocationTriple->fragmentsStorage.fragmentCount); + + allocationTriple->fragmentsStorage.fragmentStorageData[0].residency->updateCompletionData(0, osContextId); + allocationTriple->fragmentsStorage.fragmentStorageData[0].residency->resident = true; + // this fragment was used + allocationTriple->fragmentsStorage.fragmentStorageData[1].residency->updateCompletionData(11, osContextId); + allocationTriple->fragmentsStorage.fragmentStorageData[0].residency->resident = true; + + allocationTriple->fragmentsStorage.fragmentStorageData[2].residency->updateCompletionData(0, osContextId); + allocationTriple->fragmentsStorage.fragmentStorageData[2].residency->resident = true; + + // Set last periodic fence value + residencyController->setLastTrimFenceValue(10); + // Set current fence value to greater value + residencyController->getMonitoredFence().currentFenceValue = 20; + + wddm->makeNonResidentResult.called = 0; + + residencyController->addToTrimCandidateList(allocationTriple); + + residencyController->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); + + // 2 fragments evicted with one call + EXPECT_EQ(1u, wddm->makeNonResidentResult.called); + // marked nonresident + EXPECT_FALSE(allocationTriple->fragmentsStorage.fragmentStorageData[0].residency->resident); + EXPECT_FALSE(allocationTriple->fragmentsStorage.fragmentStorageData[2].residency->resident); + + memoryManager->freeGraphicsMemory(allocationTriple); +} + +TEST_F(WddmResidencyControllerWithGdiTest, givenPeriodicTrimWhenTrimCallbackCalledThenLastPeriodicTrimFenceIsSetToCurrentFenceValue) { + D3DKMT_TRIMNOTIFICATION trimNotification = {0}; + trimNotification.Flags.PeriodicTrim = 1; + trimNotification.NumBytesToTrim = 0; + + // Set last periodic fence value + residencyController->setLastTrimFenceValue(10); + // Set current fence value to greater value + *residencyController->getMonitoredFence().cpuAddress = 20; + + residencyController->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); + + EXPECT_EQ(20u, residencyController->getLastTrimFenceValue()); +} + +TEST_F(WddmResidencyControllerWithGdiTest, givenRestartPeriodicTrimWhenTrimCallbackCalledThenLastPeriodicTrimFenceIsSetToCurrentFenceValue) { + D3DKMT_TRIMNOTIFICATION trimNotification = {0}; + trimNotification.Flags.RestartPeriodicTrim = 1; + trimNotification.NumBytesToTrim = 0; + + // Set last periodic fence value + residencyController->setLastTrimFenceValue(10); + // Set current fence value to greater value + *residencyController->getMonitoredFence().cpuAddress = 20; + + residencyController->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); + + EXPECT_EQ(20u, residencyController->getLastTrimFenceValue()); +} + +TEST_F(WddmResidencyControllerWithGdiTest, trimToBudgetWithZeroSizeReturnsTrue) { + bool status = residencyController->trimResidencyToBudget(0); + + EXPECT_TRUE(status); +} + +TEST_F(WddmResidencyControllerWithGdiTest, trimToBudgetAllDoneAllocations) { + gdi->setNonZeroNumBytesToTrimInEvict(); + + MockWddmAllocation allocation1, allocation2, allocation3; + + allocation1.getResidencyData().resident = true; + allocation1.getResidencyData().updateCompletionData(0, osContextId); + + allocation2.getResidencyData().updateCompletionData(1, osContextId); + allocation2.getResidencyData().resident = true; + + allocation3.getResidencyData().updateCompletionData(2, osContextId); + allocation3.getResidencyData().resident = true; + + *residencyController->getMonitoredFence().cpuAddress = 1; + residencyController->getMonitoredFence().lastSubmittedFence = 1; + residencyController->getMonitoredFence().currentFenceValue = 1; + + wddm->makeNonResidentResult.called = 0; + + residencyController->addToTrimCandidateList(&allocation1); + residencyController->addToTrimCandidateList(&allocation2); + residencyController->addToTrimCandidateList(&allocation3); + + residencyController->trimResidencyToBudget(3 * 4096); + + EXPECT_EQ(2u, wddm->makeNonResidentResult.called); + + EXPECT_EQ(1u, residencyController->peekTrimCandidatesCount()); + residencyController->compactTrimCandidateList(); + EXPECT_EQ(1u, residencyController->peekTrimCandidateList().size()); + + EXPECT_EQ(trimListUnusedPosition, allocation1.getTrimCandidateListPosition(osContextId)); + EXPECT_EQ(trimListUnusedPosition, allocation2.getTrimCandidateListPosition(osContextId)); + EXPECT_NE(trimListUnusedPosition, allocation3.getTrimCandidateListPosition(osContextId)); +} + +TEST_F(WddmResidencyControllerWithGdiTest, trimToBudgetReturnsFalseWhenNumBytesToTrimIsNotZero) { + gdi->setNonZeroNumBytesToTrimInEvict(); + + MockWddmAllocation allocation1; + + allocation1.getResidencyData().resident = true; + allocation1.getResidencyData().updateCompletionData(0, osContextId); + + *residencyController->getMonitoredFence().cpuAddress = 1; + residencyController->getMonitoredFence().lastSubmittedFence = 1; + + wddm->makeNonResidentResult.called = 0; + + residencyController->addToTrimCandidateList(&allocation1); + + bool status = residencyController->trimResidencyToBudget(3 * 4096); + + EXPECT_EQ(1u, wddm->makeNonResidentResult.called); + EXPECT_EQ(0u, residencyController->peekTrimCandidateList().size()); + + EXPECT_FALSE(status); +} + +TEST_F(WddmResidencyControllerWithGdiTest, trimToBudgetStopsEvictingWhenNumBytesToTrimIsZero) { + WddmAllocation allocation1(reinterpret_cast(0x1000), 0x1000, reinterpret_cast(0x1000), 0x1000, nullptr, MemoryPool::MemoryNull, 1u), + allocation2(reinterpret_cast(0x1000), 0x3000, reinterpret_cast(0x1000), 0x3000, nullptr, MemoryPool::MemoryNull, 1u), + allocation3(reinterpret_cast(0x1000), 0x1000, reinterpret_cast(0x1000), 0x1000, nullptr, MemoryPool::MemoryNull, 1u); + + allocation1.getResidencyData().resident = true; + allocation1.getResidencyData().updateCompletionData(0, osContextId); + + allocation2.getResidencyData().updateCompletionData(1, osContextId); + allocation2.getResidencyData().resident = true; + + allocation3.getResidencyData().updateCompletionData(2, osContextId); + allocation3.getResidencyData().resident = true; + + *residencyController->getMonitoredFence().cpuAddress = 1; + residencyController->getMonitoredFence().lastSubmittedFence = 1; + residencyController->getMonitoredFence().currentFenceValue = 1; + + wddm->makeNonResidentResult.called = 0; + + residencyController->addToTrimCandidateList(&allocation1); + residencyController->addToTrimCandidateList(&allocation2); + residencyController->addToTrimCandidateList(&allocation3); + + bool status = residencyController->trimResidencyToBudget(3 * 4096); + + EXPECT_TRUE(status); + EXPECT_EQ(2u, wddm->makeNonResidentResult.called); + EXPECT_EQ(1u, residencyController->peekTrimCandidateList().size()); + + EXPECT_EQ(trimListUnusedPosition, allocation1.getTrimCandidateListPosition(osContextId)); + EXPECT_EQ(trimListUnusedPosition, allocation2.getTrimCandidateListPosition(osContextId)); + EXPECT_NE(trimListUnusedPosition, allocation3.getTrimCandidateListPosition(osContextId)); +} + +TEST_F(WddmResidencyControllerWithGdiTest, trimToBudgetMarksEvictedAllocationNonResident) { + gdi->setNonZeroNumBytesToTrimInEvict(); + + MockWddmAllocation allocation1, allocation2, allocation3; + + allocation1.getResidencyData().resident = true; + allocation1.getResidencyData().updateCompletionData(0, osContextId); + + allocation2.getResidencyData().updateCompletionData(1, osContextId); + allocation2.getResidencyData().resident = true; + + allocation3.getResidencyData().updateCompletionData(2, osContextId); + allocation3.getResidencyData().resident = true; + + *residencyController->getMonitoredFence().cpuAddress = 1; + residencyController->getMonitoredFence().lastSubmittedFence = 1; + residencyController->getMonitoredFence().currentFenceValue = 1; + + wddm->makeNonResidentResult.called = 0; + + residencyController->addToTrimCandidateList(&allocation1); + residencyController->addToTrimCandidateList(&allocation2); + residencyController->addToTrimCandidateList(&allocation3); + + bool status = residencyController->trimResidencyToBudget(3 * 4096); + + EXPECT_FALSE(allocation1.getResidencyData().resident); + EXPECT_FALSE(allocation2.getResidencyData().resident); + EXPECT_TRUE(allocation3.getResidencyData().resident); +} + +TEST_F(WddmResidencyControllerWithGdiTest, trimToBudgetWaitsFromCpuWhenLastFenceIsGreaterThanMonitored) { + gdi->setNonZeroNumBytesToTrimInEvict(); + + MockWddmAllocation allocation1; + + allocation1.getResidencyData().resident = true; + allocation1.getResidencyData().updateCompletionData(2, osContextId); + + *residencyController->getMonitoredFence().cpuAddress = 1; + residencyController->getMonitoredFence().lastSubmittedFence = 2; + residencyController->getMonitoredFence().currentFenceValue = 3; + + wddm->makeNonResidentResult.called = 0; + wddm->waitFromCpuResult.called = 0; + + residencyController->addToTrimCandidateList(&allocation1); + + gdi->getWaitFromCpuArg().hDevice = (D3DKMT_HANDLE)0; + + bool status = residencyController->trimResidencyToBudget(3 * 4096); + + EXPECT_EQ(1u, wddm->makeNonResidentResult.called); + EXPECT_FALSE(allocation1.getResidencyData().resident); + + EXPECT_EQ(wddm->getDevice(), gdi->getWaitFromCpuArg().hDevice); +} + +TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, trimToBudgetEvictsDoneFragmentsOnly) { + gdi->setNonZeroNumBytesToTrimInEvict(); + void *ptr = reinterpret_cast(wddm->virtualAllocAddress + 0x1000); + WddmAllocation allocation1(ptr, 0x1000, ptr, 0x1000, nullptr, MemoryPool::MemoryNull, 1u); + WddmAllocation allocation2(ptr, 0x1000, ptr, 0x1000, nullptr, MemoryPool::MemoryNull, 1u); + + allocation1.getResidencyData().resident = true; + allocation1.getResidencyData().updateCompletionData(0, osContextId); + + allocation2.getResidencyData().updateCompletionData(1, osContextId); + allocation2.getResidencyData().resident = true; + + void *ptrTriple = reinterpret_cast(reinterpret_cast(ptr) + 0x500); + WddmAllocation *allocationTriple = static_cast(memoryManager->allocateGraphicsMemory(8196, ptrTriple)); + + allocationTriple->getResidencyData().updateCompletionData(1, osContextId); + allocationTriple->getResidencyData().resident = true; + + EXPECT_EQ(3u, allocationTriple->fragmentsStorage.fragmentCount); + + for (uint32_t i = 0; i < 3; i++) { + allocationTriple->fragmentsStorage.fragmentStorageData[i].residency->updateCompletionData(1, osContextId); + allocationTriple->fragmentsStorage.fragmentStorageData[i].residency->resident = true; + } + + // This should not be evicted + allocationTriple->fragmentsStorage.fragmentStorageData[1].residency->updateCompletionData(2, osContextId); + + residencyController->addToTrimCandidateList(&allocation1); + residencyController->addToTrimCandidateList(allocationTriple); + residencyController->addToTrimCandidateList(&allocation2); + + *residencyController->getMonitoredFence().cpuAddress = 1; + residencyController->getMonitoredFence().lastSubmittedFence = 1; + residencyController->getMonitoredFence().currentFenceValue = 2; + + wddm->makeNonResidentResult.called = 0; + + bool status = residencyController->trimResidencyToBudget(3 * 4096); + + EXPECT_EQ(2u, wddm->makeNonResidentResult.called); + + EXPECT_FALSE(allocationTriple->fragmentsStorage.fragmentStorageData[0].residency->resident); + EXPECT_TRUE(allocationTriple->fragmentsStorage.fragmentStorageData[1].residency->resident); + EXPECT_FALSE(allocationTriple->fragmentsStorage.fragmentStorageData[2].residency->resident); + + memoryManager->freeGraphicsMemory(allocationTriple); +} + +TEST_F(WddmResidencyControllerWithGdiTest, givenThreeAllocationsAlignedSizeBiggerThanAllocSizeWhenBudgetEqualTwoAlignedAllocationThenEvictOnlyTwo) { + gdi->setNonZeroNumBytesToTrimInEvict(); + size_t underlyingSize = 0xF00; + size_t alignedSize = 0x1000; + size_t budget = 2 * alignedSize; + + //trim budget should consider aligned size, not underlying, so if function considers underlying, it should evict three, not two + EXPECT_GT((3 * underlyingSize), budget); + EXPECT_LT((2 * underlyingSize), budget); + void *ptr1 = reinterpret_cast(wddm->virtualAllocAddress + 0x1000); + void *ptr2 = reinterpret_cast(wddm->virtualAllocAddress + 0x3000); + void *ptr3 = reinterpret_cast(wddm->virtualAllocAddress + 0x5000); + + WddmAllocation allocation1(ptr1, underlyingSize, ptr1, alignedSize, nullptr, MemoryPool::MemoryNull, 1u); + WddmAllocation allocation2(ptr2, underlyingSize, ptr2, alignedSize, nullptr, MemoryPool::MemoryNull, 1u); + WddmAllocation allocation3(ptr3, underlyingSize, ptr3, alignedSize, nullptr, MemoryPool::MemoryNull, 1u); + + allocation1.getResidencyData().resident = true; + allocation1.getResidencyData().updateCompletionData(0, osContextId); + + allocation2.getResidencyData().updateCompletionData(1, osContextId); + allocation2.getResidencyData().resident = true; + + allocation3.getResidencyData().updateCompletionData(1, osContextId); + allocation3.getResidencyData().resident = true; + + *residencyController->getMonitoredFence().cpuAddress = 1; + residencyController->getMonitoredFence().lastSubmittedFence = 1; + residencyController->getMonitoredFence().currentFenceValue = 1; + + wddm->makeNonResidentResult.called = 0; + + residencyController->addToTrimCandidateList(&allocation1); + residencyController->addToTrimCandidateList(&allocation2); + residencyController->addToTrimCandidateList(&allocation3); + + bool status = residencyController->trimResidencyToBudget(budget); + EXPECT_TRUE(status); + + EXPECT_FALSE(allocation1.getResidencyData().resident); + EXPECT_FALSE(allocation2.getResidencyData().resident); + EXPECT_TRUE(allocation3.getResidencyData().resident); +}