diff --git a/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp index 2c23e4f544..86b0aad6c6 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp @@ -56,45 +56,109 @@ struct WddmMemoryOperationsHandlerTest : public WddmTest { StackVec allocationData; }; -TEST_F(WddmMemoryOperationsHandlerTest, givenRegularAllocationWhenMakingResidentAllocationThenMakeResidentCalled) { +TEST_F(WddmMemoryOperationsHandlerTest, givenRegularAllocationWhenMakingResidentAllocationThenMakeResidentIsCalledAndAllocationIsMarkedAsExplicitlyResident) { + wddmAllocation->setExplicitlyMadeResident(false); EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmAllocation), MemoryOperationsStatus::success); + EXPECT_TRUE(wddmAllocation->isExplicitlyMadeResident()); } -TEST_F(WddmMemoryOperationsHandlerTest, givenFragmentedAllocationWhenMakingResidentAllocationThenMakeResidentCalled) { +TEST_F(WddmMemoryOperationsHandlerTest, givenFragmentedAllocationWhenMakingResidentAllocationThenMakeResidentIsCalledAndAllocationIsMarkedAsExplicitlyResident) { allocationPtr = wddmFragmentedAllocation.get(); + allocationPtr->setExplicitlyMadeResident(false); EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success); + EXPECT_TRUE(allocationPtr->isExplicitlyMadeResident()); } TEST_F(WddmMemoryOperationsHandlerTest, givenVariousAllocationsWhenMakingResidentAllocationThenMakeResidentCalled) { + + for (auto &allocation : allocationData) { + allocation->setExplicitlyMadeResident(false); + } + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(allocationData), false), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmAllocation), MemoryOperationsStatus::success); EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success); + + for (auto &allocation : allocationData) { + EXPECT_TRUE(allocation->isExplicitlyMadeResident()); + } } TEST_F(WddmMemoryOperationsHandlerTest, givenRegularAllocationWhenEvictingResidentAllocationThenEvictCalled) { wddm->callBaseEvict = true; + allocationPtr->setExplicitlyMadeResident(false); + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_TRUE(allocationPtr->isExplicitlyMadeResident()); + EXPECT_EQ(wddmMemoryOperationsHandler->evict(nullptr, *wddmAllocation), MemoryOperationsStatus::success); + EXPECT_FALSE(allocationPtr->isExplicitlyMadeResident()); + EXPECT_EQ(0u, gdi->getEvictArg().Flags.EvictOnlyIfNecessary); EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmAllocation), MemoryOperationsStatus::memoryNotFound); } TEST_F(WddmMemoryOperationsHandlerTest, givenFragmentedAllocationWhenEvictingResidentAllocationThenEvictCalled) { allocationPtr = wddmFragmentedAllocation.get(); + allocationPtr->setExplicitlyMadeResident(false); EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1), false), MemoryOperationsStatus::success); + EXPECT_TRUE(allocationPtr->isExplicitlyMadeResident()); + EXPECT_EQ(wddmMemoryOperationsHandler->evict(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success); + EXPECT_FALSE(allocationPtr->isExplicitlyMadeResident()); + EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::memoryNotFound); } TEST_F(WddmMemoryOperationsHandlerTest, givenVariousAllocationsWhenEvictingResidentAllocationThenEvictCalled) { + for (auto &allocation : allocationData) { + allocation->setExplicitlyMadeResident(false); + } + wddm->evictResult.called = 0; EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(allocationData), false), MemoryOperationsStatus::success); + EXPECT_TRUE(wddmAllocation->isExplicitlyMadeResident()); + EXPECT_TRUE(wddmFragmentedAllocation->isExplicitlyMadeResident()); + EXPECT_EQ(wddmMemoryOperationsHandler->evict(nullptr, *wddmAllocation), MemoryOperationsStatus::success); + EXPECT_FALSE(wddmAllocation->isExplicitlyMadeResident()); + EXPECT_TRUE(wddmFragmentedAllocation->isExplicitlyMadeResident()); + EXPECT_EQ(1u, wddm->evictResult.called); + EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmAllocation), MemoryOperationsStatus::memoryNotFound); + EXPECT_EQ(wddmMemoryOperationsHandler->evict(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success); + EXPECT_FALSE(wddmAllocation->isExplicitlyMadeResident()); + EXPECT_FALSE(wddmFragmentedAllocation->isExplicitlyMadeResident()); + EXPECT_EQ(2u, wddm->evictResult.called); + + EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::memoryNotFound); +} + +TEST_F(WddmMemoryOperationsHandlerTest, givenVariousAllocationsWhenFreeResidentAllocationThenAllocationIsRemovedButEvictIsNotCalled) { + for (auto &allocation : allocationData) { + allocation->setExplicitlyMadeResident(false); + } + wddm->evictResult.called = 0; + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(allocationData), false), MemoryOperationsStatus::success); + EXPECT_TRUE(wddmAllocation->isExplicitlyMadeResident()); + EXPECT_TRUE(wddmFragmentedAllocation->isExplicitlyMadeResident()); + + EXPECT_EQ(wddmMemoryOperationsHandler->free(nullptr, *wddmAllocation), MemoryOperationsStatus::success); + EXPECT_TRUE(wddmAllocation->isExplicitlyMadeResident()); + EXPECT_TRUE(wddmFragmentedAllocation->isExplicitlyMadeResident()); + EXPECT_EQ(0u, wddm->evictResult.called); + + EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmAllocation), MemoryOperationsStatus::memoryNotFound); + + EXPECT_EQ(wddmMemoryOperationsHandler->free(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::success); + EXPECT_TRUE(wddmAllocation->isExplicitlyMadeResident()); + EXPECT_TRUE(wddmFragmentedAllocation->isExplicitlyMadeResident()); + EXPECT_EQ(0u, wddm->evictResult.called); + EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmFragmentedAllocation), MemoryOperationsStatus::memoryNotFound); } diff --git a/shared/source/memory_manager/graphics_allocation.h b/shared/source/memory_manager/graphics_allocation.h index 1436c9e74c..645a686c12 100644 --- a/shared/source/memory_manager/graphics_allocation.h +++ b/shared/source/memory_manager/graphics_allocation.h @@ -339,6 +339,13 @@ class GraphicsAllocation : public IDNode { std::atomic hostPtrTaskCountAssignment{0}; + bool isExplicitlyMadeResident() const { + return this->explicitlyMadeResident; + } + void setExplicitlyMadeResident(bool explicitlyMadeResident) { + this->explicitlyMadeResident = explicitlyMadeResident; + } + protected: struct UsageInfo { TaskCountType taskCount = objectNotUsed; @@ -404,5 +411,6 @@ class GraphicsAllocation : public IDNode { std::atomic registeredContextsNum{0}; bool shareableHostMemory = false; bool cantBeReadOnly = false; + bool explicitlyMadeResident = false; }; } // namespace NEO diff --git a/shared/source/os_interface/windows/wddm_memory_manager.cpp b/shared/source/os_interface/windows/wddm_memory_manager.cpp index b1cc9492f7..0150b1bfe8 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.cpp +++ b/shared/source/os_interface/windows/wddm_memory_manager.cpp @@ -725,6 +725,10 @@ void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation WddmAllocation *input = static_cast(gfxAllocation); DEBUG_BREAK_IF(!validateAllocation(input)); + if (gfxAllocation->isExplicitlyMadeResident()) { + freeAssociatedResourceImpl(*gfxAllocation); + } + auto ®isteredEngines = getRegisteredEngines(gfxAllocation->getRootDeviceIndex()); for (auto &engine : registeredEngines) { auto &residencyController = static_cast(engine.osContext)->getResidencyController(); @@ -1237,6 +1241,7 @@ bool WddmMemoryManager::copyMemoryToAllocationBanks(GraphicsAllocation *graphics memcpy_s(ptrOffset(ptr, destinationOffset), graphicsAllocation->getUnderlyingBufferSize() - destinationOffset, memoryToCopy, sizeToCopy); wddm.unlockResource(wddmAllocation->getHandles()[handleId]); } + wddmAllocation->setExplicitlyMadeResident(wddmAllocation->needsMakeResidentBeforeLock()); return true; } diff --git a/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp b/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp index 3ccd99ead0..972be56c3c 100644 --- a/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp +++ b/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp @@ -28,6 +28,7 @@ MemoryOperationsStatus WddmMemoryOperationsHandler::makeResident(Device *device, size_t totalSize = 0; for (const auto &allocation : gfxAllocations) { + allocation->setExplicitlyMadeResident(true); WddmAllocation *wddmAllocation = reinterpret_cast(allocation); totalSize += wddmAllocation->getAlignedSize(); @@ -48,6 +49,7 @@ MemoryOperationsStatus WddmMemoryOperationsHandler::makeResident(Device *device, } MemoryOperationsStatus WddmMemoryOperationsHandler::evict(Device *device, GraphicsAllocation &gfxAllocation) { + gfxAllocation.setExplicitlyMadeResident(false); constexpr uint32_t stackHandlesCount = NEO::maxFragmentsCount * EngineLimits::maxHandleCount; StackVec handlesForEviction; WddmAllocation &wddmAllocation = reinterpret_cast(gfxAllocation); @@ -81,4 +83,25 @@ MemoryOperationsStatus WddmMemoryOperationsHandler::isResident(Device *device, G return residentAllocations->isAllocationResident(defaultHandle); } +MemoryOperationsStatus WddmMemoryOperationsHandler::free(Device *device, GraphicsAllocation &gfxAllocation) { + if (gfxAllocation.isExplicitlyMadeResident()) { + + WddmAllocation &wddmAllocation = reinterpret_cast(gfxAllocation); + + if (wddmAllocation.fragmentsStorage.fragmentCount > 0) { + OsHandleStorage &fragmentStorage = wddmAllocation.fragmentsStorage; + + for (uint32_t allocId = 0; allocId < fragmentStorage.fragmentCount; allocId++) { + residentAllocations->removeResource(static_cast(fragmentStorage.fragmentStorageData[allocId].osHandleStorage)->handle); + } + } else { + const auto &handles = wddmAllocation.getHandles(); + size_t handleCount = wddmAllocation.getNumGmms(); + for (uint32_t i = 0; i < handleCount; i++) { + residentAllocations->removeResource(handles[i]); + } + } + } + return MemoryOperationsStatus::success; +} } // namespace NEO diff --git a/shared/source/os_interface/windows/wddm_memory_operations_handler.h b/shared/source/os_interface/windows/wddm_memory_operations_handler.h index d50ba8ce11..d44caa244e 100644 --- a/shared/source/os_interface/windows/wddm_memory_operations_handler.h +++ b/shared/source/os_interface/windows/wddm_memory_operations_handler.h @@ -36,6 +36,7 @@ class WddmMemoryOperationsHandler : public MemoryOperationsHandler { MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override { return evict(nullptr, gfxAllocation); } + MemoryOperationsStatus free(Device *device, GraphicsAllocation &gfxAllocation) override; protected: Wddm *wddm; diff --git a/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp index e8cf8f532d..14b766a5c7 100644 --- a/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp @@ -1400,6 +1400,29 @@ TEST_F(WddmMemoryManagerSimpleTest, whenDestroyingLockedAllocationIfDeviceRequir } EXPECT_EQ(0u, mockTemporaryResources->evictResourceResult.called); } + +TEST_F(WddmMemoryManagerSimpleTest, givenLocalMemoryKernelIsaWithMemoryCopiedWhenDestroyingAllocationIfDeviceRequiresMakeResidentPriorToLockThenRemoveFromTemporaryResources) { + DebugManagerStateRestore restorer; + debugManager.flags.EnableLocalMemory.set(1); + char data{}; + AllocationProperties properties{0, true, sizeof(data), AllocationType::kernelIsa, false, false, 0}; + properties.subDevicesBitfield.set(0); + memoryManager->localMemorySupported[properties.rootDeviceIndex] = true; + auto allocation = static_cast(memoryManager->allocateGraphicsMemoryWithProperties(properties)); + ASSERT_NE(nullptr, allocation); + memoryManager->copyMemoryToAllocation(allocation, 0, &data, sizeof(data)); + + auto makeResidentPriorToLockRequired = memoryManager->peekExecutionEnvironment().rootDeviceEnvironments[0u]->getHelper().makeResidentBeforeLockNeeded(allocation->needsMakeResidentBeforeLock()); + EXPECT_EQ(makeResidentPriorToLockRequired, allocation->needsMakeResidentBeforeLock()); + EXPECT_EQ(makeResidentPriorToLockRequired, allocation->isExplicitlyMadeResident()); + memoryManager->freeGraphicsMemory(allocation); + if (makeResidentPriorToLockRequired) { + EXPECT_EQ(1u, mockTemporaryResources->removeResourceResult.called); + } else { + EXPECT_EQ(0u, mockTemporaryResources->removeResourceResult.called); + } + EXPECT_EQ(0u, mockTemporaryResources->evictResourceResult.called); +} TEST_F(WddmMemoryManagerSimpleTest, whenDestroyingNotLockedAllocationThatDoesntNeedMakeResidentBeforeLockThenDontEvictAllocationFromWddmTemporaryResources) { DebugManagerStateRestore restorer; debugManager.flags.ForcePreferredAllocationMethod.set(static_cast(GfxMemoryAllocationMethod::useUmdSystemPtr));