diff --git a/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp index 8dbcb84776..1867b82a69 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp @@ -520,7 +520,7 @@ TEST_F(Wddm20Tests, WhenMakingResidentAndEvictingThenReturnIsCorrect) { EXPECT_TRUE(error); uint64_t sizeToTrim; - error = wddm->evict(&allocation.getHandles()[0], allocation.getNumGmms(), sizeToTrim); + error = wddm->evict(&allocation.getHandles()[0], allocation.getNumGmms(), sizeToTrim, true); EXPECT_TRUE(error); auto monitoredFence = osContext->getResidencyController().getMonitoredFence(); @@ -814,7 +814,7 @@ TEST_F(Wddm20Tests, GivenMultipleHandlesWhenMakingResidentThenBytesToTrimIsCorre EXPECT_EQ(gdi->getMakeResidentArg().NumBytesToTrim, bytesToTrim); } -TEST_F(Wddm20Tests, WhenMakingNonResidentThenEvictIsCalled) { +TEST_F(Wddm20Tests, WhenMakingNonResidentAndEvictNotNeededThenEvictIsCalledWithProperFlagSet) { D3DKMT_HANDLE handle = (D3DKMT_HANDLE)0x1234; gdi->getEvictArg().AllocationList = nullptr; @@ -825,12 +825,33 @@ TEST_F(Wddm20Tests, WhenMakingNonResidentThenEvictIsCalled) { wddm->callBaseEvict = true; uint64_t sizeToTrim = 10; - wddm->evict(&handle, 1, sizeToTrim); + wddm->evict(&handle, 1, sizeToTrim, false); EXPECT_EQ(1u, gdi->getEvictArg().NumAllocations); EXPECT_EQ(&handle, gdi->getEvictArg().AllocationList); EXPECT_EQ(wddm->getDeviceHandle(), gdi->getEvictArg().hDevice); EXPECT_EQ(0u, gdi->getEvictArg().NumBytesToTrim); + EXPECT_EQ(1u, gdi->getEvictArg().Flags.EvictOnlyIfNecessary); +} + +TEST_F(Wddm20Tests, WhenMakingNonResidentAndEvictNeededThenEvictIsCalledWithProperFlagSet) { + D3DKMT_HANDLE handle = (D3DKMT_HANDLE)0x1234; + + gdi->getEvictArg().AllocationList = nullptr; + gdi->getEvictArg().Flags.Value = 0; + gdi->getEvictArg().hDevice = 0; + gdi->getEvictArg().NumAllocations = 0; + gdi->getEvictArg().NumBytesToTrim = 20; + wddm->callBaseEvict = true; + + uint64_t sizeToTrim = 10; + wddm->evict(&handle, 1, sizeToTrim, true); + + EXPECT_EQ(1u, gdi->getEvictArg().NumAllocations); + EXPECT_EQ(&handle, gdi->getEvictArg().AllocationList); + EXPECT_EQ(wddm->getDeviceHandle(), gdi->getEvictArg().hDevice); + EXPECT_EQ(0u, gdi->getEvictArg().NumBytesToTrim); + EXPECT_EQ(0u, gdi->getEvictArg().Flags.EvictOnlyIfNecessary); } TEST_F(Wddm20Tests, givenDestroyAllocationWhenItIsCalledThenAllocationIsPassedToDestroyAllocation) { @@ -1072,11 +1093,13 @@ TEST_F(WddmLockWithMakeResidentTests, whenApplyBlockingMakeResidentAndTemporaryR allocation.handle = 0x3; WddmMock mockWddm(*executionEnvironment->rootDeviceEnvironments[0]); mockWddm.makeResidentStatus = false; + mockWddm.callBaseEvict = true; auto mockTemporaryResources = static_cast(mockWddm.temporaryResources.get()); mockTemporaryResources->resourceHandles.push_back(allocation.handle); mockWddm.temporaryResources->makeResidentResource(allocation.handle, 0x1000); EXPECT_EQ(2u, mockTemporaryResources->evictAllResourcesResult.called); EXPECT_EQ(1u, mockWddm.evictResult.called); + EXPECT_EQ(0u, gdi->getEvictArg().Flags.EvictOnlyIfNecessary); EXPECT_EQ(allocation.handle, mockWddm.makeResidentResult.handlePack[0]); EXPECT_EQ(3u, mockWddm.makeResidentResult.called); } diff --git a/opencl/test/unit_test/os_interface/windows/wddm_kmdaf_listener_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm_kmdaf_listener_tests.cpp index 547f40eae2..af90e2fdf9 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm_kmdaf_listener_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm_kmdaf_listener_tests.cpp @@ -128,7 +128,7 @@ TEST_F(WddmKmDafListenerTest, givenWddmWhenEvictIsCalledThenKmDafListenerNotifyE MockWddmAllocation allocation(rootDeviceEnvironment->getGmmHelper()); uint64_t sizeToTrim; - wddmWithKmDafMock->evict(&allocation.handle, 1, sizeToTrim); + wddmWithKmDafMock->evict(&allocation.handle, 1, sizeToTrim, true); EXPECT_EQ(wddmWithKmDafMock->featureTable->flags.ftrKmdDaf, wddmWithKmDafMock->getKmDafListenerMock().notifyEvictParametrization.ftrKmdDaf); EXPECT_EQ(wddmWithKmDafMock->getAdapter(), wddmWithKmDafMock->getKmDafListenerMock().notifyEvictParametrization.hAdapter); diff --git a/opencl/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp index 33789710fe..b97c18505c 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp @@ -516,6 +516,7 @@ TEST_F(WddmResidencyControllerWithGdiTest, givenNotUsedAllocationsFromPreviousPe residencyController->getMonitoredFence().currentFenceValue = 20; wddm->evictResult.called = 0; + wddm->callBaseEvict = true; residencyController->addToTrimCandidateList(&allocation1); residencyController->addToTrimCandidateList(&allocation2); @@ -524,6 +525,7 @@ TEST_F(WddmResidencyControllerWithGdiTest, givenNotUsedAllocationsFromPreviousPe // 2 allocations evicted EXPECT_EQ(2u, wddm->evictResult.called); + EXPECT_EQ(1u, gdi->getEvictArg().Flags.EvictOnlyIfNecessary); // removed from trim candidate list EXPECT_EQ(0u, residencyController->peekTrimCandidateList().size()); // marked nonresident @@ -601,6 +603,7 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, givenTripleAllocation residencyController->getMonitoredFence().currentFenceValue = 20; wddm->evictResult.called = 0; + wddm->callBaseEvict = true; residencyController->addToTrimCandidateList(allocationTriple); @@ -608,6 +611,7 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, givenTripleAllocation // 2 fragments evicted with one call EXPECT_EQ(1u, wddm->evictResult.called); + EXPECT_EQ(1u, gdi->getEvictArg().Flags.EvictOnlyIfNecessary); // marked nonresident EXPECT_FALSE(allocationTriple->fragmentsStorage.fragmentStorageData[0].residency->resident[osContextId]); EXPECT_FALSE(allocationTriple->fragmentsStorage.fragmentStorageData[2].residency->resident[osContextId]); @@ -672,6 +676,7 @@ TEST_F(WddmResidencyControllerWithGdiTest, WhenTrimmingToBudgetThenAllDoneAlloca residencyController->getMonitoredFence().currentFenceValue = 1; wddm->evictResult.called = 0; + wddm->callBaseEvict = true; residencyController->addToTrimCandidateList(&allocation1); residencyController->addToTrimCandidateList(&allocation2); @@ -680,6 +685,7 @@ TEST_F(WddmResidencyControllerWithGdiTest, WhenTrimmingToBudgetThenAllDoneAlloca residencyController->trimResidencyToBudget(3 * 4096); EXPECT_EQ(2u, wddm->evictResult.called); + EXPECT_EQ(0u, gdi->getEvictArg().Flags.EvictOnlyIfNecessary); EXPECT_EQ(1u, residencyController->peekTrimCandidatesCount()); residencyController->compactTrimCandidateList(); @@ -853,11 +859,11 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, WhenTrimmingToBudgetT residencyController->getMonitoredFence().currentFenceValue = 2; wddm->evictResult.called = 0; - + wddm->callBaseEvict = true; residencyController->trimResidencyToBudget(3 * 4096); EXPECT_EQ(2u, wddm->evictResult.called); - + EXPECT_EQ(0u, gdi->getEvictArg().Flags.EvictOnlyIfNecessary); EXPECT_FALSE(allocationTriple->fragmentsStorage.fragmentStorageData[0].residency->resident[osContextId]); EXPECT_TRUE(allocationTriple->fragmentsStorage.fragmentStorageData[1].residency->resident[osContextId]); EXPECT_FALSE(allocationTriple->fragmentsStorage.fragmentStorageData[2].residency->resident[osContextId]); diff --git a/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp index b07d5b1dbb..a6f64070d2 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp @@ -75,8 +75,10 @@ TEST_F(WddmMemoryOperationsHandlerTest, givenVariousAllocationsWhenMakingResiden } TEST_F(WddmMemoryOperationsHandlerTest, givenRegularAllocationWhenEvictingResidentAllocationThenEvictCalled) { + wddm->callBaseEvict = true; EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1)), MemoryOperationsStatus::SUCCESS); EXPECT_EQ(wddmMemoryOperationsHandler->evict(nullptr, *wddmAllocation), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(0u, gdi->getEvictArg().Flags.EvictOnlyIfNecessary); EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, *wddmAllocation), MemoryOperationsStatus::MEMORY_NOT_FOUND); } diff --git a/shared/source/os_interface/windows/trim_callback.cpp b/shared/source/os_interface/windows/trim_callback.cpp index 893f5e7c2a..5141d82f74 100644 --- a/shared/source/os_interface/windows/trim_callback.cpp +++ b/shared/source/os_interface/windows/trim_callback.cpp @@ -70,7 +70,7 @@ void WddmResidencyController::trimResidency(const D3DDDI_TRIMRESIDENCYSET_FLAGS if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation: default handle =", wddmAllocation->getDefaultHandle(), "lastFence =", wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId)); - this->wddm.evict(&wddmAllocation->getHandles()[0], wddmAllocation->getNumGmms(), sizeToTrim); + this->wddm.evict(&wddmAllocation->getHandles()[0], wddmAllocation->getNumGmms(), sizeToTrim, false); } for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { @@ -84,7 +84,7 @@ void WddmResidencyController::trimResidency(const D3DDDI_TRIMRESIDENCYSET_FLAGS } } if (fragmentsToEvict != 0) { - this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim); + this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim, false); } wddmAllocation->getResidencyData().resident[osContextId] = false; @@ -129,7 +129,7 @@ bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes) { } if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { - this->wddm.evict(&wddmAllocation->getHandles()[0], wddmAllocation->getNumGmms(), sizeToTrim); + this->wddm.evict(&wddmAllocation->getHandles()[0], wddmAllocation->getNumGmms(), sizeToTrim, true); sizeEvicted = wddmAllocation->getAlignedSize(); } else { auto &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData; @@ -140,7 +140,7 @@ bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes) { } if (fragmentsToEvict != 0) { - this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim); + this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim, true); for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) { diff --git a/shared/source/os_interface/windows/wddm/wddm.cpp b/shared/source/os_interface/windows/wddm/wddm.cpp index 480a8e3049..8ad6370203 100644 --- a/shared/source/os_interface/windows/wddm/wddm.cpp +++ b/shared/source/os_interface/windows/wddm/wddm.cpp @@ -359,13 +359,14 @@ std::vector> Wddm::discoverDevices(ExecutionEnvironm return hwDeviceIds; } -bool Wddm::evict(const D3DKMT_HANDLE *handleList, uint32_t numOfHandles, uint64_t &sizeToTrim) { +bool Wddm::evict(const D3DKMT_HANDLE *handleList, uint32_t numOfHandles, uint64_t &sizeToTrim, bool evictNeeded) { NTSTATUS status = STATUS_SUCCESS; D3DKMT_EVICT evict = {}; evict.AllocationList = handleList; evict.hDevice = device; evict.NumAllocations = numOfHandles; evict.NumBytesToTrim = 0; + evict.Flags.EvictOnlyIfNecessary = evictNeeded ? 0 : 1; status = getGdi()->evict(&evict); diff --git a/shared/source/os_interface/windows/wddm/wddm.h b/shared/source/os_interface/windows/wddm/wddm.h index 1ed3a52473..061dfa2486 100644 --- a/shared/source/os_interface/windows/wddm/wddm.h +++ b/shared/source/os_interface/windows/wddm/wddm.h @@ -70,7 +70,7 @@ class Wddm : public DriverModel { static Wddm *createWddm(std::unique_ptr &&hwDeviceId, RootDeviceEnvironment &rootDeviceEnvironment); bool init(); - MOCKABLE_VIRTUAL bool evict(const D3DKMT_HANDLE *handleList, uint32_t numOfHandles, uint64_t &sizeToTrim); + MOCKABLE_VIRTUAL bool evict(const D3DKMT_HANDLE *handleList, uint32_t numOfHandles, uint64_t &sizeToTrim, bool evictNeeded); MOCKABLE_VIRTUAL bool makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t totalSize); MOCKABLE_VIRTUAL bool mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_VIRTUAL_ADDRESS preferredAddress, D3DGPU_VIRTUAL_ADDRESS &gpuPtr); bool mapGpuVirtualAddress(AllocationStorageData *allocationStorageData); diff --git a/shared/source/os_interface/windows/wddm_residency_allocations_container.cpp b/shared/source/os_interface/windows/wddm_residency_allocations_container.cpp index 922550311a..d97da2fd50 100644 --- a/shared/source/os_interface/windows/wddm_residency_allocations_container.cpp +++ b/shared/source/os_interface/windows/wddm_residency_allocations_container.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2021 Intel Corporation + * Copyright (C) 2019-2022 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -36,7 +36,7 @@ MemoryOperationsStatus WddmResidentAllocationsContainer::evictAllResourcesNoLock } uint64_t sizeToTrim = 0; uint32_t evictedResources = static_cast(resourcesToEvict.size()); - bool success = wddm->evict(resourcesToEvict.data(), evictedResources, sizeToTrim); + bool success = wddm->evict(resourcesToEvict.data(), evictedResources, sizeToTrim, true); return success ? MemoryOperationsStatus::SUCCESS : MemoryOperationsStatus::FAILED; } @@ -54,7 +54,7 @@ MemoryOperationsStatus WddmResidentAllocationsContainer::evictResources(const D3 UNRECOVERABLE_IF(distance + count > resourceHandles.size()); resourceHandles.erase(position, position + count); uint64_t sizeToTrim = 0; - if (!wddm->evict(handles, count, sizeToTrim)) { + if (!wddm->evict(handles, count, sizeToTrim, true)) { return MemoryOperationsStatus::FAILED; } return MemoryOperationsStatus::SUCCESS; diff --git a/shared/test/common/mocks/mock_wddm.cpp b/shared/test/common/mocks/mock_wddm.cpp index 97a7f2c174..681f4478a2 100644 --- a/shared/test/common/mocks/mock_wddm.cpp +++ b/shared/test/common/mocks/mock_wddm.cpp @@ -59,10 +59,10 @@ bool WddmMock::makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool c } return makeResidentStatus; } -bool WddmMock::evict(const D3DKMT_HANDLE *handles, uint32_t num, uint64_t &sizeToTrim) { +bool WddmMock::evict(const D3DKMT_HANDLE *handles, uint32_t num, uint64_t &sizeToTrim, bool evictNeeded) { evictResult.called++; if (callBaseEvict) { - evictStatus = Wddm::evict(handles, num, sizeToTrim); + evictStatus = Wddm::evict(handles, num, sizeToTrim, evictNeeded); } return evictStatus; } diff --git a/shared/test/common/mocks/mock_wddm.h b/shared/test/common/mocks/mock_wddm.h index d8897bfb9a..0b91a1668a 100644 --- a/shared/test/common/mocks/mock_wddm.h +++ b/shared/test/common/mocks/mock_wddm.h @@ -124,7 +124,7 @@ class WddmMock : public Wddm { void resetGdi(Gdi *gdi); bool makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim, size_t totalSize) override; - bool evict(const D3DKMT_HANDLE *handles, uint32_t num, uint64_t &sizeToTrim) override; + bool evict(const D3DKMT_HANDLE *handles, uint32_t num, uint64_t &sizeToTrim, bool evictNeeded) override; NTSTATUS createAllocationsAndMapGpuVa(OsHandleStorage &osHandles) override; WddmMockHelpers::MakeResidentCall makeResidentResult;