diff --git a/runtime/os_interface/windows/wddm_memory_manager.cpp b/runtime/os_interface/windows/wddm_memory_manager.cpp index 1e0b14eed0..0ab4fe066f 100644 --- a/runtime/os_interface/windows/wddm_memory_manager.cpp +++ b/runtime/os_interface/windows/wddm_memory_manager.cpp @@ -297,15 +297,9 @@ void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation for (const auto &residencyController : this->residencyControllers) { if (residencyController) { residencyController->acquireLock(); - } - } - - if (input->getTrimCandidateListPosition() != trimListUnusedPosition) { - removeFromTrimCandidateList(gfxAllocation, true); - } - - for (const auto &residencyController : this->residencyControllers) { - if (residencyController) { + if (input->getTrimCandidateListPosition() != trimListUnusedPosition) { + residencyController->removeFromTrimCandidateList(gfxAllocation, true); + } residencyController->releaseLock(); } } @@ -491,7 +485,7 @@ bool WddmMemoryManager::makeResidentResidencyAllocations(ResidencyContainer &all if (allocation->getTrimCandidateListPosition() != trimListUnusedPosition) { DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation =", allocation, "on trimCandidateList"); - removeFromTrimCandidateList(allocation); + residencyControllers[osContext.getContextId()]->removeFromTrimCandidateList(allocation, false); } else { for (uint32_t allocationId = 0; allocationId < allocation->fragmentsStorage.fragmentCount; allocationId++) { @@ -558,105 +552,12 @@ void WddmMemoryManager::makeNonResidentEvictionAllocations(ResidencyContainer &e for (uint32_t i = 0; i < residencyCount; i++) { WddmAllocation *allocation = reinterpret_cast(evictionAllocations[i]); - - addToTrimCandidateList(allocation); + residencyControllers[osContext.getContextId()]->addToTrimCandidateList(allocation); } this->residencyControllers[osContext.getContextId()]->releaseLock(); } -void WddmMemoryManager::removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList) { - WddmAllocation *wddmAllocation = (WddmAllocation *)allocation; - size_t position = wddmAllocation->getTrimCandidateListPosition(); - - DEBUG_BREAK_IF(!(trimCandidatesCount > (trimCandidatesCount - 1))); - DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size()); - - trimCandidatesCount--; - - trimCandidateList[position] = nullptr; - - checkTrimCandidateCount(); - - if (position == trimCandidateList.size() - 1) { - size_t erasePosition = position; - - if (position == 0) { - trimCandidateList.resize(0); - } else { - while (trimCandidateList[erasePosition] == nullptr && erasePosition > 0) { - erasePosition--; - } - - size_t sizeRemaining = erasePosition + 1; - if (erasePosition == 0 && trimCandidateList[erasePosition] == nullptr) { - sizeRemaining = 0; - } - - trimCandidateList.resize(sizeRemaining); - } - } - wddmAllocation->setTrimCandidateListPosition(trimListUnusedPosition); - - if (compactList && checkTrimCandidateListCompaction()) { - compactTrimCandidateList(); - } - - checkTrimCandidateCount(); -} - -void WddmMemoryManager::addToTrimCandidateList(GraphicsAllocation *allocation) { - WddmAllocation *wddmAllocation = (WddmAllocation *)allocation; - size_t position = trimCandidateList.size(); - - DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size()); - - if (wddmAllocation->getTrimCandidateListPosition() == trimListUnusedPosition) { - trimCandidatesCount++; - trimCandidateList.push_back(allocation); - wddmAllocation->setTrimCandidateListPosition(position); - } - - checkTrimCandidateCount(); -} - -void WddmMemoryManager::compactTrimCandidateList() { - - size_t size = trimCandidateList.size(); - size_t freePosition = 0; - - if (size == 0 || size == trimCandidatesCount) { - return; - } - - DEBUG_BREAK_IF(!(trimCandidateList[size - 1] != nullptr)); - - uint32_t previousCount = trimCandidatesCount; - DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size()); - - while (freePosition < trimCandidatesCount && trimCandidateList[freePosition] != nullptr) - freePosition++; - - for (uint32_t i = 1; i < size; i++) { - - if (trimCandidateList[i] != nullptr && freePosition < i) { - trimCandidateList[freePosition] = trimCandidateList[i]; - trimCandidateList[i] = nullptr; - ((WddmAllocation *)trimCandidateList[freePosition])->setTrimCandidateListPosition(freePosition); - freePosition++; - - // Last element was moved, erase elements from freePosition - if (i == size - 1) { - trimCandidateList.resize(freePosition); - } - } - } - DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size()); - DEBUG_BREAK_IF(trimCandidatesCount != previousCount); - - checkTrimCandidateCount(); -} - void WddmMemoryManager::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes) { OsContext &osContext = *getRegisteredOsContext(0); if (flags.PeriodicTrim) { @@ -667,7 +568,7 @@ void WddmMemoryManager::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint6 this->residencyControllers[osContext.getContextId()]->acquireLock(); WddmAllocation *wddmAllocation = nullptr; - while ((wddmAllocation = getTrimCandidateHead()) != nullptr) { + while ((wddmAllocation = this->residencyControllers[osContext.getContextId()]->getTrimCandidateHead()) != nullptr) { DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", this->residencyControllers[osContext.getContextId()]->getLastTrimFenceValue()); @@ -698,15 +599,16 @@ void WddmMemoryManager::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint6 } wddmAllocation->getResidencyData().resident = false; - removeFromTrimCandidateList(wddmAllocation); + + residencyControllers[osContext.getContextId()]->removeFromTrimCandidateList(wddmAllocation, false); } else { periodicTrimDone = true; break; } } - if (checkTrimCandidateListCompaction()) { - compactTrimCandidateList(); + if (residencyControllers[osContext.getContextId()]->checkTrimCandidateListCompaction()) { + residencyControllers[osContext.getContextId()]->compactTrimCandidateList(); } this->residencyControllers[osContext.getContextId()]->releaseLock(); @@ -728,25 +630,6 @@ void WddmMemoryManager::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint6 } } -void WddmMemoryManager::checkTrimCandidateCount() { - if (DebugManager.flags.ResidencyDebugEnable.get()) { - uint32_t sum = 0; - for (size_t i = 0; i < trimCandidateList.size(); i++) { - if (trimCandidateList[i] != nullptr) { - sum++; - } - } - DEBUG_BREAK_IF(sum != trimCandidatesCount); - } -} - -bool WddmMemoryManager::checkTrimCandidateListCompaction() { - if (2 * trimCandidatesCount <= trimCandidateList.size()) { - return true; - } - return false; -} - bool WddmMemoryManager::trimResidencyToBudget(uint64_t bytes) { bool trimToBudgetDone = false; D3DKMT_HANDLE fragmentEvictHandles[3] = {0}; @@ -757,7 +640,7 @@ bool WddmMemoryManager::trimResidencyToBudget(uint64_t bytes) { while (!trimToBudgetDone) { uint64_t lastFence = 0; - wddmAllocation = getTrimCandidateHead(); + wddmAllocation = this->residencyControllers[0]->getTrimCandidateHead(); if (wddmAllocation == nullptr) { break; @@ -766,7 +649,7 @@ bool WddmMemoryManager::trimResidencyToBudget(uint64_t bytes) { lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(0); auto osContext = wddmAllocation->getResidencyData().getOsContextFromId(0); if (!osContext) { - removeFromTrimCandidateList(wddmAllocation); + residencyControllers[0]->removeFromTrimCandidateList(wddmAllocation, false); continue; } auto &monitoredFence = osContext->get()->getMonitoredFence(); @@ -811,15 +694,15 @@ bool WddmMemoryManager::trimResidencyToBudget(uint64_t bytes) { } wddmAllocation->getResidencyData().resident = false; - removeFromTrimCandidateList(wddmAllocation); + residencyControllers[0]->removeFromTrimCandidateList(wddmAllocation, false); trimToBudgetDone = (numberOfBytesToTrim == 0); } else { trimToBudgetDone = true; } } - if (bytes > numberOfBytesToTrim && checkTrimCandidateListCompaction()) { - compactTrimCandidateList(); + if (bytes > numberOfBytesToTrim && residencyControllers[0]->checkTrimCandidateListCompaction()) { + residencyControllers[0]->compactTrimCandidateList(); } return numberOfBytesToTrim == 0; diff --git a/runtime/os_interface/windows/wddm_memory_manager.h b/runtime/os_interface/windows/wddm_memory_manager.h index f893013e66..0bc6adf108 100644 --- a/runtime/os_interface/windows/wddm_memory_manager.h +++ b/runtime/os_interface/windows/wddm_memory_manager.h @@ -82,30 +82,10 @@ class WddmMemoryManager : public MemoryManager { protected: GraphicsAllocation *createAllocationFromHandle(osHandle handle, bool requireSpecificBitness, bool ntHandle); - WddmAllocation *getTrimCandidateHead() { - uint32_t i = 0; - size_t size = trimCandidateList.size(); - - if (size == 0) { - return nullptr; - } - while ((trimCandidateList[i] == nullptr) && (i < size)) - i++; - - return (WddmAllocation *)trimCandidateList[i]; - } - void removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList = false); - void addToTrimCandidateList(GraphicsAllocation *allocation); - void compactTrimCandidateList(); void trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes); bool trimResidencyToBudget(uint64_t bytes); static bool validateAllocation(WddmAllocation *alloc); - bool checkTrimCandidateListCompaction(); - void checkTrimCandidateCount(); bool createWddmAllocation(WddmAllocation *allocation, AllocationOrigin origin); - ResidencyContainer trimCandidateList; - std::mutex trimCandidateListMutex; - uint32_t trimCandidatesCount = 0; std::vector> residencyControllers; bool memoryBudgetExhausted = false; AlignedMallocRestrictions mallocRestrictions; diff --git a/runtime/os_interface/windows/wddm_residency_controller.cpp b/runtime/os_interface/windows/wddm_residency_controller.cpp index d56e6584de..df861a3039 100644 --- a/runtime/os_interface/windows/wddm_residency_controller.cpp +++ b/runtime/os_interface/windows/wddm_residency_controller.cpp @@ -6,6 +6,8 @@ */ #include "wddm_residency_controller.h" +#include "runtime/os_interface/windows/wddm_allocation.h" +#include "runtime/os_interface/debug_settings_manager.h" namespace OCLRT { @@ -20,4 +22,128 @@ void WddmResidencyController::acquireLock() { void WddmResidencyController::releaseLock() { lock = false; } + +WddmAllocation *WddmResidencyController::getTrimCandidateHead() { + uint32_t i = 0; + const size_t size = trimCandidateList.size(); + + if (size == 0) { + return nullptr; + } + while ((trimCandidateList[i] == nullptr) && (i < size)) + i++; + + return reinterpret_cast(trimCandidateList[i]); +} + +void WddmResidencyController::addToTrimCandidateList(GraphicsAllocation *allocation) { + WddmAllocation *wddmAllocation = (WddmAllocation *)allocation; + size_t position = trimCandidateList.size(); + + DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size()); + + if (wddmAllocation->getTrimCandidateListPosition() == trimListUnusedPosition) { + trimCandidatesCount++; + trimCandidateList.push_back(allocation); + wddmAllocation->setTrimCandidateListPosition(position); + } + + checkTrimCandidateCount(); +} + +void WddmResidencyController::removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList) { + WddmAllocation *wddmAllocation = (WddmAllocation *)allocation; + size_t position = wddmAllocation->getTrimCandidateListPosition(); + + DEBUG_BREAK_IF(!(trimCandidatesCount > (trimCandidatesCount - 1))); + DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size()); + + trimCandidatesCount--; + + trimCandidateList[position] = nullptr; + + checkTrimCandidateCount(); + + if (position == trimCandidateList.size() - 1) { + size_t erasePosition = position; + + if (position == 0) { + trimCandidateList.resize(0); + } else { + while (trimCandidateList[erasePosition] == nullptr && erasePosition > 0) { + erasePosition--; + } + + size_t sizeRemaining = erasePosition + 1; + if (erasePosition == 0 && trimCandidateList[erasePosition] == nullptr) { + sizeRemaining = 0; + } + + trimCandidateList.resize(sizeRemaining); + } + } + wddmAllocation->setTrimCandidateListPosition(trimListUnusedPosition); + + if (compactList && checkTrimCandidateListCompaction()) { + compactTrimCandidateList(); + } + + checkTrimCandidateCount(); +} + +void WddmResidencyController::checkTrimCandidateCount() { + if (DebugManager.flags.ResidencyDebugEnable.get()) { + uint32_t sum = 0; + for (size_t i = 0; i < trimCandidateList.size(); i++) { + if (trimCandidateList[i] != nullptr) { + sum++; + } + } + DEBUG_BREAK_IF(sum != trimCandidatesCount); + } +} + +bool WddmResidencyController::checkTrimCandidateListCompaction() { + if (2 * trimCandidatesCount <= trimCandidateList.size()) { + return true; + } + return false; +} + +void WddmResidencyController::compactTrimCandidateList() { + size_t size = trimCandidateList.size(); + size_t freePosition = 0; + + if (size == 0 || size == trimCandidatesCount) { + return; + } + + DEBUG_BREAK_IF(!(trimCandidateList[size - 1] != nullptr)); + + uint32_t previousCount = trimCandidatesCount; + DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size()); + + while (freePosition < trimCandidatesCount && trimCandidateList[freePosition] != nullptr) + freePosition++; + + for (uint32_t i = 1; i < size; i++) { + + if (trimCandidateList[i] != nullptr && freePosition < i) { + trimCandidateList[freePosition] = trimCandidateList[i]; + trimCandidateList[i] = nullptr; + ((WddmAllocation *)trimCandidateList[freePosition])->setTrimCandidateListPosition(freePosition); + freePosition++; + + // Last element was moved, erase elements from freePosition + if (i == size - 1) { + trimCandidateList.resize(freePosition); + } + } + } + DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size()); + DEBUG_BREAK_IF(trimCandidatesCount != previousCount); + + checkTrimCandidateCount(); +} + } // namespace OCLRT diff --git a/runtime/os_interface/windows/wddm_residency_controller.h b/runtime/os_interface/windows/wddm_residency_controller.h index bfa26b85d1..298b3a1571 100644 --- a/runtime/os_interface/windows/wddm_residency_controller.h +++ b/runtime/os_interface/windows/wddm_residency_controller.h @@ -7,10 +7,15 @@ #pragma once +#include "runtime/memory_manager/residency_container.h" + #include namespace OCLRT { +class GraphicsAllocation; +class WddmAllocation; + class WddmResidencyController { public: WddmResidencyController(); @@ -18,11 +23,23 @@ class WddmResidencyController { void acquireLock(); void releaseLock(); + WddmAllocation *getTrimCandidateHead(); + void addToTrimCandidateList(GraphicsAllocation *allocation); + void removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList); + void checkTrimCandidateCount(); + + bool checkTrimCandidateListCompaction(); + void compactTrimCandidateList(); + uint64_t getLastTrimFenceValue() { return lastTrimFenceValue; } void setLastTrimFenceValue(uint64_t value) { lastTrimFenceValue = value; } + const ResidencyContainer &peekTrimCandidateList() const { return trimCandidateList; } + uint32_t peekTrimCandidatesCount() const { return trimCandidatesCount; } protected: std::atomic lock; uint64_t lastTrimFenceValue; + ResidencyContainer trimCandidateList; + uint32_t trimCandidatesCount = 0; }; } // namespace OCLRT diff --git a/unit_tests/os_interface/windows/CMakeLists.txt b/unit_tests/os_interface/windows/CMakeLists.txt index 58f299bfc2..80bd42fd69 100644 --- a/unit_tests/os_interface/windows/CMakeLists.txt +++ b/unit_tests/os_interface/windows/CMakeLists.txt @@ -36,6 +36,7 @@ set(IGDRCL_SRCS_tests_os_interface_windows ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_memory_manager_allocate_in_device_pool_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_manager_allocate_in_device_pool_tests.inl ${CMAKE_CURRENT_SOURCE_DIR}/wddm_preemption_tests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_controller_tests.cpp ) if(WIN32) file(GLOB IGDRCL_SRC_tests_wddm_interface "${CMAKE_CURRENT_SOURCE_DIR}/wddm2[0-9]_tests\.cpp") diff --git a/unit_tests/os_interface/windows/device_command_stream_tests.cpp b/unit_tests/os_interface/windows/device_command_stream_tests.cpp index a4e23ea220..8c29fc94ce 100644 --- a/unit_tests/os_interface/windows/device_command_stream_tests.cpp +++ b/unit_tests/os_interface/windows/device_command_stream_tests.cpp @@ -21,6 +21,7 @@ #include "runtime/os_interface/windows/os_interface.h" #include "runtime/os_interface/windows/wddm_device_command_stream.h" #include "runtime/os_interface/windows/wddm_memory_manager.h" +#include "runtime/os_interface/windows/wddm_residency_controller.h" #include "unit_tests/fixtures/device_fixture.h" #include "unit_tests/fixtures/gmm_environment_fixture.h" @@ -557,7 +558,7 @@ TEST_F(WddmCommandStreamTest, processEvictionPlacesAllAllocationsOnTrimCandidate csr->processEviction(*device->getOsContext()); - EXPECT_EQ(2u, mockWddmMM->trimCandidateList.size()); + EXPECT_EQ(2u, mockWddmMM->residencyControllers[0]->peekTrimCandidateList().size()); memManager->freeGraphicsMemory(allocation); memManager->freeGraphicsMemory(allocation2); diff --git a/unit_tests/os_interface/windows/mock_wddm_memory_manager.h b/unit_tests/os_interface/windows/mock_wddm_memory_manager.h index da841d963a..fdc1af68ff 100644 --- a/unit_tests/os_interface/windows/mock_wddm_memory_manager.h +++ b/unit_tests/os_interface/windows/mock_wddm_memory_manager.h @@ -14,14 +14,8 @@ class MockWddmMemoryManager : public WddmMemoryManager { using BaseClass = WddmMemoryManager; public: - using BaseClass::addToTrimCandidateList; - using BaseClass::checkTrimCandidateListCompaction; - using BaseClass::compactTrimCandidateList; using BaseClass::createWddmAllocation; - using BaseClass::removeFromTrimCandidateList; using BaseClass::residencyControllers; - using BaseClass::trimCandidateList; - using BaseClass::trimCandidatesCount; using BaseClass::trimResidency; using BaseClass::trimResidencyToBudget; using BaseClass::WddmMemoryManager; diff --git a/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp b/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp index 9d97f0cb98..5781aed785 100644 --- a/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp +++ b/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp @@ -874,179 +874,6 @@ TEST_F(WddmMemoryManagerTest, givenPtrAndSizePassedToCreateInternalAllocationWhe memoryManager->freeGraphicsMemory(wddmAllocation); } -TEST_F(WddmMemoryManagerResidencyTest, addToTrimCandidateListPlacesAllocationInContainerAndAssignsPosition) { - MockWddmAllocation allocation; - - memoryManager->addToTrimCandidateList(&allocation); - - EXPECT_NE(0u, memoryManager->trimCandidateList.size()); - EXPECT_NE(trimListUnusedPosition, allocation.getTrimCandidateListPosition()); - - size_t position = allocation.getTrimCandidateListPosition(); - ASSERT_LT(position, memoryManager->trimCandidateList.size()); - - EXPECT_EQ(&allocation, memoryManager->trimCandidateList[position]); -} - -TEST_F(WddmMemoryManagerResidencyTest, addToTrimCandidateListDoesNotInsertAllocationAlreadyOnTheList) { - MockWddmAllocation allocation; - - memoryManager->trimCandidateList.resize(0); - - memoryManager->addToTrimCandidateList(&allocation); - - EXPECT_NE(trimListUnusedPosition, allocation.getTrimCandidateListPosition()); - - size_t position = allocation.getTrimCandidateListPosition(); - ASSERT_LT(position, memoryManager->trimCandidateList.size()); - - EXPECT_EQ(&allocation, memoryManager->trimCandidateList[position]); - - size_t previousSize = memoryManager->trimCandidateList.size(); - memoryManager->addToTrimCandidateList(&allocation); - - EXPECT_EQ(previousSize, memoryManager->trimCandidateList.size()); - EXPECT_EQ(position, allocation.getTrimCandidateListPosition()); -} - -TEST_F(WddmMemoryManagerResidencyTest, removeFromTrimCandidateListAssignsUnusedPosition) { - MockWddmAllocation allocation; - - memoryManager->addToTrimCandidateList(&allocation); - memoryManager->removeFromTrimCandidateList(&allocation); - - EXPECT_EQ(trimListUnusedPosition, allocation.getTrimCandidateListPosition()); -} - -TEST_F(WddmMemoryManagerResidencyTest, removeFromTrimCandidateListRemovesAllocationInAssignedPosition) { - MockWddmAllocation allocation; - - memoryManager->addToTrimCandidateList(&allocation); - size_t position = allocation.getTrimCandidateListPosition(); - - memoryManager->removeFromTrimCandidateList(&allocation); - - if (memoryManager->trimCandidateList.size() > position) { - EXPECT_NE(&allocation, memoryManager->trimCandidateList[position]); - } -} - -TEST_F(WddmMemoryManagerResidencyTest, removeFromTrimCandidateListRemovesLastAllocation) { - MockWddmAllocation allocation; - - memoryManager->trimCandidateList.resize(0); - - memoryManager->addToTrimCandidateList(&allocation); - - memoryManager->removeFromTrimCandidateList(&allocation); - - EXPECT_EQ(0u, memoryManager->trimCandidateList.size()); -} - -TEST_F(WddmMemoryManagerResidencyTest, removeFromTrimCandidateListRemovesLastAllocationAndAllPreviousEmptyEntries) { - MockWddmAllocation allocation1, allocation2; - - memoryManager->trimCandidateList.resize(0); - - memoryManager->addToTrimCandidateList(&allocation1); - - memoryManager->trimCandidateList.push_back(nullptr); - memoryManager->trimCandidateList.push_back(nullptr); - memoryManager->trimCandidateList.push_back(nullptr); - - memoryManager->addToTrimCandidateList(&allocation2); - - EXPECT_EQ(5u, memoryManager->trimCandidateList.size()); - - memoryManager->removeFromTrimCandidateList(&allocation2); - - EXPECT_EQ(1u, memoryManager->trimCandidateList.size()); -} - -TEST_F(WddmMemoryManagerResidencyTest, successiveAddingToTrimCandidateListAssignsNewPositions) { - MockWddmAllocation allocation1, allocation2, allocation3; - - memoryManager->addToTrimCandidateList(&allocation1); - memoryManager->addToTrimCandidateList(&allocation2); - memoryManager->addToTrimCandidateList(&allocation3); - - EXPECT_EQ(3u, memoryManager->trimCandidateList.size()); - EXPECT_NE(allocation1.getTrimCandidateListPosition(), allocation2.getTrimCandidateListPosition()); - EXPECT_NE(allocation2.getTrimCandidateListPosition(), allocation3.getTrimCandidateListPosition()); -} - -TEST_F(WddmMemoryManagerResidencyTest, DISABLED_removingNotLastAllocationFromTrimCandidateListSubstituesLastPositionAllocation) { - MockWddmAllocation allocation1, allocation2, allocation3; - - memoryManager->addToTrimCandidateList(&allocation1); - memoryManager->addToTrimCandidateList(&allocation2); - memoryManager->addToTrimCandidateList(&allocation3); - - memoryManager->removeFromTrimCandidateList(&allocation2); - - EXPECT_EQ(2u, memoryManager->trimCandidateList.size()); - - EXPECT_EQ(2u, allocation3.getTrimCandidateListPosition()); - EXPECT_NE(allocation2.getTrimCandidateListPosition(), allocation3.getTrimCandidateListPosition()); -} - -TEST_F(WddmMemoryManagerResidencyTest, removingNotLastAllocationFromTrimCandidateListPutsNullEntry) { - MockWddmAllocation allocation1, allocation2, allocation3; - - memoryManager->addToTrimCandidateList(&allocation1); - memoryManager->addToTrimCandidateList(&allocation2); - memoryManager->addToTrimCandidateList(&allocation3); - size_t position2 = allocation2.getTrimCandidateListPosition(); - size_t position3 = allocation3.getTrimCandidateListPosition(); - - memoryManager->removeFromTrimCandidateList(&allocation2); - - EXPECT_EQ(3u, memoryManager->trimCandidateList.size()); - EXPECT_EQ(2u, position3); - EXPECT_EQ(nullptr, memoryManager->trimCandidateList[position2]); -} - -TEST_F(WddmMemoryManagerResidencyTest, compactTrimCandidateListRemovesInitialNullEntriesAndUpdatesPositions) { - MockWddmAllocation allocation1, allocation2, allocation3, allocation4; - - memoryManager->addToTrimCandidateList(&allocation1); - memoryManager->addToTrimCandidateList(&allocation2); - memoryManager->addToTrimCandidateList(&allocation3); - memoryManager->addToTrimCandidateList(&allocation4); - - size_t position3 = allocation3.getTrimCandidateListPosition(); - size_t position4 = allocation4.getTrimCandidateListPosition(); - - memoryManager->removeFromTrimCandidateList(&allocation2); - memoryManager->removeFromTrimCandidateList(&allocation1); - - EXPECT_EQ(4u, memoryManager->trimCandidateList.size()); - - memoryManager->compactTrimCandidateList(); - - EXPECT_EQ(2u, memoryManager->trimCandidateList.size()); - - EXPECT_EQ(memoryManager->trimCandidateList[0], &allocation3); - EXPECT_EQ(0u, allocation3.getTrimCandidateListPosition()); - - EXPECT_EQ(memoryManager->trimCandidateList[1], &allocation4); - EXPECT_EQ(1u, allocation4.getTrimCandidateListPosition()); -} - -TEST_F(WddmMemoryManagerResidencyTest, compactTrimCandidateListWithNonNullEntries) { - MockWddmAllocation allocation1, allocation2, allocation3, allocation4; - - memoryManager->addToTrimCandidateList(&allocation1); - memoryManager->addToTrimCandidateList(&allocation2); - memoryManager->addToTrimCandidateList(&allocation3); - memoryManager->addToTrimCandidateList(&allocation4); - - EXPECT_EQ(4u, memoryManager->trimCandidateList.size()); - - memoryManager->compactTrimCandidateList(); - - EXPECT_EQ(4u, memoryManager->trimCandidateList.size()); -} TEST_F(WddmMemoryManagerResidencyTest, makeResidentResidencyAllocationsMarksAllocationsResident) { MockWddmAllocation allocation1, allocation2, allocation3, allocation4; @@ -1144,17 +971,15 @@ TEST_F(WddmMemoryManagerResidencyTest, givenNotUsedAllocationsFromPreviousPeriod wddm->makeNonResidentResult.called = 0; - memoryManager->trimCandidateList.resize(0); - - memoryManager->addToTrimCandidateList(&allocation1); - memoryManager->addToTrimCandidateList(&allocation2); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation1); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation2); memoryManager->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); // 2 allocations evicted EXPECT_EQ(2u, wddm->makeNonResidentResult.called); // removed from trim candidate list - EXPECT_EQ(0u, memoryManager->trimCandidateList.size()); + EXPECT_EQ(0u, memoryManager->residencyControllers[0]->peekTrimCandidateList().size()); // marked nonresident EXPECT_FALSE(allocation1.getResidencyData().resident); EXPECT_FALSE(allocation2.getResidencyData().resident); @@ -1180,10 +1005,8 @@ TEST_F(WddmMemoryManagerResidencyTest, givenOneUsedAllocationFromPreviousPeriodi wddm->makeNonResidentResult.called = 0; - memoryManager->trimCandidateList.resize(0); - - memoryManager->addToTrimCandidateList(&allocation1); - memoryManager->addToTrimCandidateList(&allocation2); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation1); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation2); memoryManager->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); @@ -1226,9 +1049,7 @@ TEST_F(WddmMemoryManagerResidencyTest, givenTripleAllocationWithUsedAndUnusedFra wddm->makeNonResidentResult.called = 0; - memoryManager->trimCandidateList.resize(0); - - memoryManager->addToTrimCandidateList(allocationTriple); + memoryManager->residencyControllers[0]->addToTrimCandidateList(allocationTriple); memoryManager->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); @@ -1253,7 +1074,6 @@ TEST_F(WddmMemoryManagerResidencyTest, givenPeriodicTrimWhenTrimCallbackCalledTh // Set current fence value to greater value *osContext->get()->getMonitoredFence().cpuAddress = 20; - memoryManager->trimCandidateList.resize(0); memoryManager->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); EXPECT_EQ(20u, memoryManager->residencyControllers[0]->getLastTrimFenceValue()); @@ -1271,7 +1091,6 @@ TEST_F(WddmMemoryManagerResidencyTest, givenRestartPeriodicTrimWhenTrimCallbackC // Set current fence value to greater value *osContext->get()->getMonitoredFence().cpuAddress = 20; - memoryManager->trimCandidateList.resize(0); memoryManager->trimResidency(trimNotification.Flags, trimNotification.NumBytesToTrim); EXPECT_EQ(20u, memoryManager->residencyControllers[0]->getLastTrimFenceValue()); @@ -1303,19 +1122,17 @@ TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetAllDoneAllocations) { wddm->makeNonResidentResult.called = 0; - memoryManager->trimCandidateList.resize(0); - - memoryManager->addToTrimCandidateList(&allocation1); - memoryManager->addToTrimCandidateList(&allocation2); - memoryManager->addToTrimCandidateList(&allocation3); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation1); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation2); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation3); memoryManager->trimResidencyToBudget(3 * 4096); EXPECT_EQ(2u, wddm->makeNonResidentResult.called); - EXPECT_EQ(1u, memoryManager->trimCandidatesCount); - memoryManager->compactTrimCandidateList(); - EXPECT_EQ(1u, memoryManager->trimCandidateList.size()); + EXPECT_EQ(1u, memoryManager->residencyControllers[0]->peekTrimCandidatesCount()); + memoryManager->residencyControllers[0]->compactTrimCandidateList(); + EXPECT_EQ(1u, memoryManager->residencyControllers[0]->peekTrimCandidateList().size()); EXPECT_EQ(trimListUnusedPosition, allocation1.getTrimCandidateListPosition()); EXPECT_EQ(trimListUnusedPosition, allocation2.getTrimCandidateListPosition()); @@ -1334,14 +1151,13 @@ TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetReturnsFalseWhenNumBytesToTri osContext->get()->getMonitoredFence().lastSubmittedFence = 1; wddm->makeNonResidentResult.called = 0; - memoryManager->trimCandidateList.resize(0); - memoryManager->addToTrimCandidateList(&allocation1); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation1); bool status = memoryManager->trimResidencyToBudget(3 * 4096); EXPECT_EQ(1u, wddm->makeNonResidentResult.called); - EXPECT_EQ(0u, memoryManager->trimCandidateList.size()); + EXPECT_EQ(0u, memoryManager->residencyControllers[0]->peekTrimCandidateList().size()); EXPECT_FALSE(status); } @@ -1366,17 +1182,15 @@ TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetStopsEvictingWhenNumBytesToTr wddm->makeNonResidentResult.called = 0; - memoryManager->trimCandidateList.resize(0); - - memoryManager->addToTrimCandidateList(&allocation1); - memoryManager->addToTrimCandidateList(&allocation2); - memoryManager->addToTrimCandidateList(&allocation3); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation1); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation2); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation3); bool status = memoryManager->trimResidencyToBudget(3 * 4096); EXPECT_TRUE(status); EXPECT_EQ(2u, wddm->makeNonResidentResult.called); - EXPECT_EQ(1u, memoryManager->trimCandidateList.size()); + EXPECT_EQ(1u, memoryManager->residencyControllers[0]->peekTrimCandidateList().size()); EXPECT_EQ(trimListUnusedPosition, allocation1.getTrimCandidateListPosition()); EXPECT_EQ(trimListUnusedPosition, allocation2.getTrimCandidateListPosition()); @@ -1403,11 +1217,9 @@ TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetMarksEvictedAllocationNonResi wddm->makeNonResidentResult.called = 0; - memoryManager->trimCandidateList.resize(0); - - memoryManager->addToTrimCandidateList(&allocation1); - memoryManager->addToTrimCandidateList(&allocation2); - memoryManager->addToTrimCandidateList(&allocation3); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation1); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation2); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation3); bool status = memoryManager->trimResidencyToBudget(3 * 4096); @@ -1431,9 +1243,7 @@ TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetWaitsFromCpuWhenLastFenceIsGr wddm->makeNonResidentResult.called = 0; wddm->waitFromCpuResult.called = 0; - memoryManager->trimCandidateList.resize(0); - - memoryManager->addToTrimCandidateList(&allocation1); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation1); gdi->getWaitFromCpuArg().hDevice = (D3DKMT_HANDLE)0; @@ -1473,11 +1283,9 @@ TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetEvictsDoneFragmentsOnly) { // This should not be evicted allocationTriple->fragmentsStorage.fragmentStorageData[1].residency->updateCompletionData(2, osContext); - memoryManager->trimCandidateList.resize(0); - - memoryManager->addToTrimCandidateList(&allocation1); - memoryManager->addToTrimCandidateList(allocationTriple); - memoryManager->addToTrimCandidateList(&allocation2); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation1); + memoryManager->residencyControllers[0]->addToTrimCandidateList(allocationTriple); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation2); *osContext->get()->getMonitoredFence().cpuAddress = 1; osContext->get()->getMonitoredFence().lastSubmittedFence = 1; @@ -1496,29 +1304,6 @@ TEST_F(WddmMemoryManagerResidencyTest, trimToBudgetEvictsDoneFragmentsOnly) { memoryManager->freeGraphicsMemory(allocationTriple); } -TEST_F(WddmMemoryManagerResidencyTest, checkTrimCandidateListCompaction) { - memoryManager->trimCandidatesCount = 10; - memoryManager->trimCandidateList.resize(20); - - bool comapctionRequired = memoryManager->checkTrimCandidateListCompaction(); - - EXPECT_TRUE(comapctionRequired); - - memoryManager->trimCandidatesCount = 5; - memoryManager->trimCandidateList.resize(20); - - comapctionRequired = memoryManager->checkTrimCandidateListCompaction(); - - EXPECT_TRUE(comapctionRequired); - - memoryManager->trimCandidatesCount = 18; - memoryManager->trimCandidateList.resize(20); - - comapctionRequired = memoryManager->checkTrimCandidateListCompaction(); - - EXPECT_FALSE(comapctionRequired); -} - TEST_F(WddmMemoryManagerResidencyTest, givenThreeAllocationsAlignedSizeBiggerThanAllocSizeWhenBudgetEqualTwoAlignedAllocationThenEvictOnlyTwo) { gdi->setNonZeroNumBytesToTrimInEvict(); size_t underlyingSize = 0xF00; @@ -1551,11 +1336,9 @@ TEST_F(WddmMemoryManagerResidencyTest, givenThreeAllocationsAlignedSizeBiggerTha wddm->makeNonResidentResult.called = 0; - memoryManager->trimCandidateList.resize(0); - - memoryManager->addToTrimCandidateList(&allocation1); - memoryManager->addToTrimCandidateList(&allocation2); - memoryManager->addToTrimCandidateList(&allocation3); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation1); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation2); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocation3); bool status = memoryManager->trimResidencyToBudget(budget); EXPECT_TRUE(status); @@ -1833,7 +1616,7 @@ TEST_F(WddmMemoryManagerTest2, makeResidentResidencyAllocationsSucceedsWhenMakeR EXPECT_CALL(*wddm, makeResident(::testing::_, ::testing::_, ::testing::_, ::testing::_)).Times(2).WillOnce(::testing::Invoke(makeResidentWithOutBytesToTrim)).WillOnce(::testing::Return(true)); - memoryManager->addToTrimCandidateList(&allocationToTrim); + memoryManager->residencyControllers[0]->addToTrimCandidateList(&allocationToTrim); ResidencyContainer residencyPack{&allocation1}; bool result = memoryManager->makeResidentResidencyAllocations(residencyPack, *osContext); diff --git a/unit_tests/os_interface/windows/wddm_residency_controller_tests.cpp b/unit_tests/os_interface/windows/wddm_residency_controller_tests.cpp new file mode 100644 index 0000000000..6fa3cfdabe --- /dev/null +++ b/unit_tests/os_interface/windows/wddm_residency_controller_tests.cpp @@ -0,0 +1,220 @@ +/* + * Copyright (C) 2018 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "runtime/os_interface/windows/wddm_residency_controller.h" +#include "unit_tests/os_interface/windows/mock_wddm_allocation.h" + +#include "test.h" +#include + +using namespace OCLRT; + +class MockWddmResidencyController : public WddmResidencyController { + public: + using WddmResidencyController::trimCandidateList; + using WddmResidencyController::trimCandidatesCount; +}; + +struct WddmResidencyControllerTest : ::testing::Test { + void SetUp() { + residencyController = std::make_unique(); + } + + std::unique_ptr residencyController; +}; + +TEST_F(WddmResidencyControllerTest, addToTrimCandidateListPlacesAllocationInContainerAndAssignsPosition) { + MockWddmAllocation allocation; + residencyController->addToTrimCandidateList(&allocation); + + EXPECT_NE(0u, residencyController->trimCandidateList.size()); + EXPECT_NE(trimListUnusedPosition, allocation.getTrimCandidateListPosition()); + + size_t position = allocation.getTrimCandidateListPosition(); + ASSERT_LT(position, residencyController->trimCandidateList.size()); + + EXPECT_EQ(&allocation, residencyController->trimCandidateList[position]); +} + +TEST_F(WddmResidencyControllerTest, addToTrimCandidateListDoesNotInsertAllocationAlreadyOnTheList) { + MockWddmAllocation allocation; + + residencyController->trimCandidateList.resize(0); + + residencyController->addToTrimCandidateList(&allocation); + + EXPECT_NE(trimListUnusedPosition, allocation.getTrimCandidateListPosition()); + + size_t position = allocation.getTrimCandidateListPosition(); + ASSERT_LT(position, residencyController->trimCandidateList.size()); + + EXPECT_EQ(&allocation, residencyController->trimCandidateList[position]); + + size_t previousSize = residencyController->trimCandidateList.size(); + residencyController->addToTrimCandidateList(&allocation); + + EXPECT_EQ(previousSize, residencyController->trimCandidateList.size()); + EXPECT_EQ(position, allocation.getTrimCandidateListPosition()); +} + +TEST_F(WddmResidencyControllerTest, removeFromTrimCandidateListAssignsUnusedPosition) { + MockWddmAllocation allocation; + + residencyController->addToTrimCandidateList(&allocation); + residencyController->removeFromTrimCandidateList(&allocation, false); + + EXPECT_EQ(trimListUnusedPosition, allocation.getTrimCandidateListPosition()); +} + +TEST_F(WddmResidencyControllerTest, removeFromTrimCandidateListRemovesAllocationInAssignedPosition) { + MockWddmAllocation allocation; + + residencyController->addToTrimCandidateList(&allocation); + size_t position = allocation.getTrimCandidateListPosition(); + + residencyController->removeFromTrimCandidateList(&allocation, false); + + if (residencyController->trimCandidateList.size() > position) { + EXPECT_NE(&allocation, residencyController->trimCandidateList[position]); + } +} + +TEST_F(WddmResidencyControllerTest, removeFromTrimCandidateListRemovesLastAllocation) { + MockWddmAllocation allocation; + + residencyController->trimCandidateList.resize(0); + + residencyController->addToTrimCandidateList(&allocation); + + residencyController->removeFromTrimCandidateList(&allocation, false); + + EXPECT_EQ(0u, residencyController->trimCandidateList.size()); +} + +TEST_F(WddmResidencyControllerTest, removeFromTrimCandidateListRemovesLastAllocationAndAllPreviousEmptyEntries) { + MockWddmAllocation allocation1, allocation2; + + residencyController->trimCandidateList.resize(0); + + residencyController->addToTrimCandidateList(&allocation1); + + residencyController->trimCandidateList.push_back(nullptr); + residencyController->trimCandidateList.push_back(nullptr); + residencyController->trimCandidateList.push_back(nullptr); + + residencyController->addToTrimCandidateList(&allocation2); + + EXPECT_EQ(5u, residencyController->trimCandidateList.size()); + + residencyController->removeFromTrimCandidateList(&allocation2, false); + + EXPECT_EQ(1u, residencyController->trimCandidateList.size()); +} + +TEST_F(WddmResidencyControllerTest, successiveAddingToTrimCandidateListAssignsNewPositions) { + MockWddmAllocation allocation1, allocation2, allocation3; + + residencyController->addToTrimCandidateList(&allocation1); + residencyController->addToTrimCandidateList(&allocation2); + residencyController->addToTrimCandidateList(&allocation3); + + EXPECT_EQ(3u, residencyController->trimCandidateList.size()); + EXPECT_NE(allocation1.getTrimCandidateListPosition(), allocation2.getTrimCandidateListPosition()); + EXPECT_NE(allocation2.getTrimCandidateListPosition(), allocation3.getTrimCandidateListPosition()); +} + +TEST_F(WddmResidencyControllerTest, DISABLED_removingNotLastAllocationFromTrimCandidateListSubstituesLastPositionAllocation) { + MockWddmAllocation allocation1, allocation2, allocation3; + + residencyController->addToTrimCandidateList(&allocation1); + residencyController->addToTrimCandidateList(&allocation2); + residencyController->addToTrimCandidateList(&allocation3); + + residencyController->removeFromTrimCandidateList(&allocation2, false); + + EXPECT_EQ(2u, residencyController->trimCandidateList.size()); + + EXPECT_EQ(2u, allocation3.getTrimCandidateListPosition()); + EXPECT_NE(allocation2.getTrimCandidateListPosition(), allocation3.getTrimCandidateListPosition()); +} + +TEST_F(WddmResidencyControllerTest, removingNotLastAllocationFromTrimCandidateListPutsNullEntry) { + MockWddmAllocation allocation1, allocation2, allocation3; + + residencyController->addToTrimCandidateList(&allocation1); + residencyController->addToTrimCandidateList(&allocation2); + residencyController->addToTrimCandidateList(&allocation3); + size_t position2 = allocation2.getTrimCandidateListPosition(); + size_t position3 = allocation3.getTrimCandidateListPosition(); + + residencyController->removeFromTrimCandidateList(&allocation2, false); + + EXPECT_EQ(3u, residencyController->trimCandidateList.size()); + EXPECT_EQ(2u, position3); + EXPECT_EQ(nullptr, residencyController->trimCandidateList[position2]); +} + +TEST_F(WddmResidencyControllerTest, compactTrimCandidateListRemovesInitialNullEntriesAndUpdatesPositions) { + MockWddmAllocation allocation1, allocation2, allocation3, allocation4; + + residencyController->addToTrimCandidateList(&allocation1); + residencyController->addToTrimCandidateList(&allocation2); + residencyController->addToTrimCandidateList(&allocation3); + residencyController->addToTrimCandidateList(&allocation4); + + size_t position3 = allocation3.getTrimCandidateListPosition(); + size_t position4 = allocation4.getTrimCandidateListPosition(); + + residencyController->removeFromTrimCandidateList(&allocation2, false); + residencyController->removeFromTrimCandidateList(&allocation1, false); + + EXPECT_EQ(4u, residencyController->trimCandidateList.size()); + + residencyController->compactTrimCandidateList(); + + EXPECT_EQ(2u, residencyController->trimCandidateList.size()); + + EXPECT_EQ(residencyController->trimCandidateList[0], &allocation3); + EXPECT_EQ(0u, allocation3.getTrimCandidateListPosition()); + + EXPECT_EQ(residencyController->trimCandidateList[1], &allocation4); + EXPECT_EQ(1u, allocation4.getTrimCandidateListPosition()); +} + +TEST_F(WddmResidencyControllerTest, compactTrimCandidateListWithNonNullEntries) { + MockWddmAllocation allocation1, allocation2, allocation3, allocation4; + + residencyController->addToTrimCandidateList(&allocation1); + residencyController->addToTrimCandidateList(&allocation2); + residencyController->addToTrimCandidateList(&allocation3); + residencyController->addToTrimCandidateList(&allocation4); + + EXPECT_EQ(4u, residencyController->trimCandidateList.size()); + + residencyController->compactTrimCandidateList(); + + EXPECT_EQ(4u, residencyController->trimCandidateList.size()); +} + +TEST_F(WddmResidencyControllerTest, checkTrimCandidateListCompaction) { + bool comapactionRequired; + + residencyController->trimCandidatesCount = 10; + residencyController->trimCandidateList.resize(20); + comapactionRequired = residencyController->checkTrimCandidateListCompaction(); + EXPECT_TRUE(comapactionRequired); + + residencyController->trimCandidatesCount = 5; + residencyController->trimCandidateList.resize(20); + comapactionRequired = residencyController->checkTrimCandidateListCompaction(); + EXPECT_TRUE(comapactionRequired); + + residencyController->trimCandidatesCount = 18; + residencyController->trimCandidateList.resize(20); + comapactionRequired = residencyController->checkTrimCandidateListCompaction(); + EXPECT_FALSE(comapactionRequired); +}