diff --git a/runtime/os_interface/windows/wddm/wddm.cpp b/runtime/os_interface/windows/wddm/wddm.cpp index faf9809b03..f733ba165e 100644 --- a/runtime/os_interface/windows/wddm/wddm.cpp +++ b/runtime/os_interface/windows/wddm/wddm.cpp @@ -826,6 +826,18 @@ void Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, Wdd } } +void Wddm::unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback) { + if (callback == nullptr) { + return; + } + D3DKMT_UNREGISTERTRIMNOTIFICATION unregisterTrimNotification; + unregisterTrimNotification.Callback = callback; + unregisterTrimNotification.Handle = this->trimCallbackHandle; + + NTSTATUS status = gdi->unregisterTrimNotification(&unregisterTrimNotification); + DEBUG_BREAK_IF(status != STATUS_SUCCESS); +} + void Wddm::releaseReservedAddress(void *reservedAddress) { if (reservedAddress) { auto status = virtualFree(reservedAddress, 0, MEM_RELEASE); diff --git a/runtime/os_interface/windows/wddm/wddm.h b/runtime/os_interface/windows/wddm/wddm.h index ed792dafdc..02b20baf4e 100644 --- a/runtime/os_interface/windows/wddm/wddm.h +++ b/runtime/os_interface/windows/wddm/wddm.h @@ -80,6 +80,7 @@ class Wddm { NTSTATUS escape(D3DKMT_ESCAPE &escapeCommand); void registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmMemoryManager *memoryManager); + void unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback); MOCKABLE_VIRTUAL void releaseReservedAddress(void *reservedAddress); MOCKABLE_VIRTUAL bool reserveValidAddressRange(size_t size, void *&reservedMem); diff --git a/runtime/os_interface/windows/wddm_memory_manager.cpp b/runtime/os_interface/windows/wddm_memory_manager.cpp index 0ab4fe066f..c948661bd9 100644 --- a/runtime/os_interface/windows/wddm_memory_manager.cpp +++ b/runtime/os_interface/windows/wddm_memory_manager.cpp @@ -27,6 +27,16 @@ namespace OCLRT { WddmMemoryManager::~WddmMemoryManager() { applyCommonCleanup(); + + for (auto &residencyController : this->residencyControllers) { + residencyController->acquireTrimCallbackLock(); + wddm->unregisterTrimCallback(trimCallback); + residencyController->releaseTrimCallbackLock(); + + // Wait for lock to ensure trimCallback ended + residencyController->acquireTrimCallbackLock(); + residencyController->releaseTrimCallbackLock(); + } } WddmMemoryManager::WddmMemoryManager(bool enable64kbPages, bool enableLocalMemory, Wddm *wddm, ExecutionEnvironment &executionEnvironment) : MemoryManager(enable64kbPages, enableLocalMemory, executionEnvironment) { @@ -44,7 +54,9 @@ void APIENTRY WddmMemoryManager::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *t WddmMemoryManager *wddmMemMngr = (WddmMemoryManager *)trimNotification->Context; DEBUG_BREAK_IF(wddmMemMngr == nullptr); + wddmMemMngr->residencyControllers[0]->acquireTrimCallbackLock(); wddmMemMngr->trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim); + wddmMemMngr->residencyControllers[0]->releaseTrimCallbackLock(); } GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryForImage(ImageInfo &imgInfo, Gmm *gmm) { diff --git a/runtime/os_interface/windows/wddm_memory_manager.h b/runtime/os_interface/windows/wddm_memory_manager.h index 0bc6adf108..48e58be499 100644 --- a/runtime/os_interface/windows/wddm_memory_manager.h +++ b/runtime/os_interface/windows/wddm_memory_manager.h @@ -81,9 +81,10 @@ class WddmMemoryManager : public MemoryManager { AlignedMallocRestrictions *getAlignedMallocRestrictions() override; protected: - GraphicsAllocation *createAllocationFromHandle(osHandle handle, bool requireSpecificBitness, bool ntHandle); void trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes); bool trimResidencyToBudget(uint64_t bytes); + + GraphicsAllocation *createAllocationFromHandle(osHandle handle, bool requireSpecificBitness, bool ntHandle); static bool validateAllocation(WddmAllocation *alloc); bool createWddmAllocation(WddmAllocation *allocation, AllocationOrigin origin); std::vector> residencyControllers; diff --git a/runtime/os_interface/windows/wddm_residency_controller.cpp b/runtime/os_interface/windows/wddm_residency_controller.cpp index df861a3039..c3f6ed9525 100644 --- a/runtime/os_interface/windows/wddm_residency_controller.cpp +++ b/runtime/os_interface/windows/wddm_residency_controller.cpp @@ -8,6 +8,7 @@ #include "wddm_residency_controller.h" #include "runtime/os_interface/windows/wddm_allocation.h" #include "runtime/os_interface/debug_settings_manager.h" +#include "runtime/utilities/spinlock.h" namespace OCLRT { @@ -23,6 +24,16 @@ void WddmResidencyController::releaseLock() { lock = false; } +void WddmResidencyController::acquireTrimCallbackLock() { + SpinLock spinLock; + spinLock.enter(this->trimCallbackLock); +} + +void WddmResidencyController::releaseTrimCallbackLock() { + SpinLock spinLock; + spinLock.leave(this->trimCallbackLock); +} + WddmAllocation *WddmResidencyController::getTrimCandidateHead() { uint32_t i = 0; const size_t size = trimCandidateList.size(); diff --git a/runtime/os_interface/windows/wddm_residency_controller.h b/runtime/os_interface/windows/wddm_residency_controller.h index 298b3a1571..6887c42917 100644 --- a/runtime/os_interface/windows/wddm_residency_controller.h +++ b/runtime/os_interface/windows/wddm_residency_controller.h @@ -23,6 +23,9 @@ class WddmResidencyController { void acquireLock(); void releaseLock(); + void acquireTrimCallbackLock(); + void releaseTrimCallbackLock(); + WddmAllocation *getTrimCandidateHead(); void addToTrimCandidateList(GraphicsAllocation *allocation); void removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList); @@ -38,6 +41,7 @@ class WddmResidencyController { protected: std::atomic lock; + std::atomic_flag trimCallbackLock = ATOMIC_FLAG_INIT; uint64_t lastTrimFenceValue; ResidencyContainer trimCandidateList; uint32_t trimCandidatesCount = 0; diff --git a/unit_tests/os_interface/windows/mock_gdi_interface.h b/unit_tests/os_interface/windows/mock_gdi_interface.h index 78b40fd391..b5017a6a9f 100644 --- a/unit_tests/os_interface/windows/mock_gdi_interface.h +++ b/unit_tests/os_interface/windows/mock_gdi_interface.h @@ -46,6 +46,11 @@ class MockGdi : public Gdi { return 0; } + static NTSTATUS __stdcall unregisterTrimNotificationMock(IN D3DKMT_UNREGISTERTRIMNOTIFICATION *arg) { + getUnregisterTrimNotificationArg() = *arg; + return 0; + } + static NTSTATUS __stdcall destroyAllocation2Mock(IN D3DKMT_DESTROYALLOCATION2 *arg) { getDestroyArg() = *arg; return 0; @@ -87,6 +92,7 @@ class MockGdi : public Gdi { evict = reinterpret_cast(evictMock); } registerTrimNotification = reinterpret_cast(registerTimNotificationMock); + unregisterTrimNotification = reinterpret_cast(unregisterTrimNotificationMock); destroyAllocation2 = reinterpret_cast(destroyAllocation2Mock); waitForSynchronizationObjectFromCpu = reinterpret_cast(waitFromCpuMock); queryResourceInfo = reinterpret_cast(queryResourceInfoMock); @@ -109,6 +115,11 @@ class MockGdi : public Gdi { return registerTrimArg; } + static D3DKMT_UNREGISTERTRIMNOTIFICATION &getUnregisterTrimNotificationArg() { + static D3DKMT_UNREGISTERTRIMNOTIFICATION unregisterTrimArg; + return unregisterTrimArg; + } + static D3DKMT_DESTROYALLOCATION2 &getDestroyArg() { static D3DKMT_DESTROYALLOCATION2 destroyArg; return destroyArg; diff --git a/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp b/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp index 5781aed785..4a58efd9ba 100644 --- a/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp +++ b/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp @@ -874,7 +874,6 @@ TEST_F(WddmMemoryManagerTest, givenPtrAndSizePassedToCreateInternalAllocationWhe memoryManager->freeGraphicsMemory(wddmAllocation); } - TEST_F(WddmMemoryManagerResidencyTest, makeResidentResidencyAllocationsMarksAllocationsResident) { MockWddmAllocation allocation1, allocation2, allocation3, allocation4; ResidencyContainer residencyPack{&allocation1, &allocation2, &allocation3, &allocation4}; @@ -942,6 +941,16 @@ TEST_F(WddmMemoryManagerResidencyTest, trimCallbackIsRegisteredInWddmMemoryManag EXPECT_EQ(wddm->getDevice(), gdi->getRegisterTrimNotificationArg().hDevice); } +TEST_F(WddmMemoryManagerResidencyTest, givenWddmMemoryManagerWhenCallingDestructorThenUnregisterTrimCallback) { + auto trimCallbackHandle = wddm->trimCallbackHandle; + auto trimCallbackAddress = reinterpret_cast(memoryManager->trimCallback); + memoryManager.reset(); + + auto &unregisterNotification = gdi->getUnregisterTrimNotificationArg(); + EXPECT_EQ(trimCallbackAddress, unregisterNotification.Callback); + EXPECT_EQ(trimCallbackHandle, unregisterNotification.Handle); +} + TEST(WddmDebugModesTests, givenDebugModeWhenItIsActiveThenTrimCallbackIsNotRegistred) { DebugManagerStateRestore stateRestore; DebugManager.flags.DoNotRegisterTrimCallback.set(true); @@ -1642,15 +1651,16 @@ TEST_F(WddmMemoryManagerTest2, givenMemoryManagerWhenMakeResidentFailsThenMemory struct WddmMemoryManagerWithAsyncDeleterTest : ::testing::Test { void SetUp() { wddm = std::make_unique(); + wddm->gdi.reset(new MockGdi()); wddm->callBaseDestroyAllocations = false; deleter = new MockDeferredDeleter; memoryManager = std::make_unique(wddm.get(), executionEnvironment); memoryManager->setDeferredDeleter(deleter); } MockDeferredDeleter *deleter = nullptr; + std::unique_ptr wddm; std::unique_ptr memoryManager; ExecutionEnvironment executionEnvironment; - std::unique_ptr wddm; }; TEST_F(WddmMemoryManagerWithAsyncDeleterTest, givenWddmWhenAsyncDeleterIsEnabledThenCanDeferDeletions) {