diff --git a/level_zero/core/source/memory/cpu_page_fault_memory_manager.cpp b/level_zero/core/source/memory/cpu_page_fault_memory_manager.cpp index d998100823..245c8e4379 100644 --- a/level_zero/core/source/memory/cpu_page_fault_memory_manager.cpp +++ b/level_zero/core/source/memory/cpu_page_fault_memory_manager.cpp @@ -42,7 +42,7 @@ void PageFaultManager::transferToGpu(void *ptr, void *device) { this->evictMemoryAfterImplCopy(allocData->cpuAllocation, deviceImp->getNEODevice()); } -void PageFaultManager::allowCPUMemoryEviction(void *ptr, PageFaultData &pageFaultData) { +void PageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) { L0::DeviceImp *deviceImp = static_cast(pageFaultData.cmdQ); CommandStreamReceiver *csr = nullptr; @@ -54,7 +54,7 @@ void PageFaultManager::allowCPUMemoryEviction(void *ptr, PageFaultData &pageFaul UNRECOVERABLE_IF(csr == nullptr); auto osInterface = deviceImp->getNEODevice()->getRootDeviceEnvironment().osInterface.get(); - allowCPUMemoryEvictionImpl(ptr, *csr, osInterface); + allowCPUMemoryEvictionImpl(evict, ptr, *csr, osInterface); } } // namespace NEO diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp index c56b8c60b2..d5d5fa00b7 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp @@ -841,7 +841,7 @@ TEST_F(CommandListMemAdvisePageFault, givenUnifiedMemoryAllocWhenAllowCPUMemoryE NEO::PageFaultManager::PageFaultData pageData; pageData.cmdQ = deviceImp; - mockPageFaultManager->baseAllowCPUMemoryEviction(ptr, pageData); + mockPageFaultManager->baseAllowCPUMemoryEviction(true, ptr, pageData); EXPECT_EQ(mockPageFaultManager->allowCPUMemoryEvictionImplCalled, 1); CommandStreamReceiver *csr = nullptr; diff --git a/opencl/source/memory_manager/cpu_page_fault_manager_memory_sync.cpp b/opencl/source/memory_manager/cpu_page_fault_manager_memory_sync.cpp index 096dda916b..1a8170a959 100644 --- a/opencl/source/memory_manager/cpu_page_fault_manager_memory_sync.cpp +++ b/opencl/source/memory_manager/cpu_page_fault_manager_memory_sync.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2023 Intel Corporation + * Copyright (C) 2019-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -33,7 +33,7 @@ void PageFaultManager::transferToGpu(void *ptr, void *cmdQ) { UNRECOVERABLE_IF(allocData == nullptr); this->evictMemoryAfterImplCopy(allocData->cpuAllocation, &commandQueue->getDevice()); } -void PageFaultManager::allowCPUMemoryEviction(void *ptr, PageFaultData &pageFaultData) { +void PageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) { auto commandQueue = static_cast(pageFaultData.cmdQ); auto allocData = memoryData[ptr].unifiedMemoryManager->getSVMAlloc(ptr); @@ -42,7 +42,7 @@ void PageFaultManager::allowCPUMemoryEviction(void *ptr, PageFaultData &pageFaul auto &csr = commandQueue->selectCsrForBuiltinOperation(csrSelectionArgs); auto osInterface = commandQueue->getDevice().getRootDeviceEnvironment().osInterface.get(); - allowCPUMemoryEvictionImpl(ptr, csr, osInterface); + allowCPUMemoryEvictionImpl(evict, ptr, csr, osInterface); } } // namespace NEO diff --git a/opencl/test/unit_test/memory_manager/cpu_page_fault_manager_memory_sync_tests.cpp b/opencl/test/unit_test/memory_manager/cpu_page_fault_manager_memory_sync_tests.cpp index 75fd403651..c00181be48 100644 --- a/opencl/test/unit_test/memory_manager/cpu_page_fault_manager_memory_sync_tests.cpp +++ b/opencl/test/unit_test/memory_manager/cpu_page_fault_manager_memory_sync_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2023 Intel Corporation + * Copyright (C) 2019-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -124,7 +124,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenAllowCPUMemoryEvictionIs NEO::PageFaultManager::PageFaultData pageData; pageData.cmdQ = cmdQ.get(); - pageFaultManager->baseAllowCPUMemoryEviction(alloc, pageData); + pageFaultManager->baseAllowCPUMemoryEviction(true, alloc, pageData); EXPECT_EQ(pageFaultManager->allowCPUMemoryEvictionImplCalled, 1); auto allocData = svmAllocsManager->getSVMAlloc(alloc); diff --git a/shared/source/page_fault_manager/cpu_page_fault_manager.cpp b/shared/source/page_fault_manager/cpu_page_fault_manager.cpp index 62229b098b..c7c3c0a211 100644 --- a/shared/source/page_fault_manager/cpu_page_fault_manager.cpp +++ b/shared/source/page_fault_manager/cpu_page_fault_manager.cpp @@ -73,6 +73,7 @@ void PageFaultManager::moveAllocationsWithinUMAllocsManagerToGpuDomain(SVMAllocs inline void PageFaultManager::migrateStorageToGpuDomain(void *ptr, PageFaultData &pageFaultData) { if (pageFaultData.domain == AllocationDomain::cpu) { this->setCpuAllocEvictable(false, ptr, pageFaultData.unifiedMemoryManager); + this->allowCPUMemoryEviction(false, ptr, pageFaultData); std::chrono::steady_clock::time_point start; std::chrono::steady_clock::time_point end; @@ -117,7 +118,7 @@ void PageFaultManager::transferAndUnprotectMemory(PageFaultManager *pageFaultHan pageFaultHandler->migrateStorageToCpuDomain(allocPtr, pageFaultData); pageFaultHandler->allowCPUMemoryAccess(allocPtr, pageFaultData.size); pageFaultHandler->setCpuAllocEvictable(true, allocPtr, pageFaultData.unifiedMemoryManager); - pageFaultHandler->allowCPUMemoryEviction(allocPtr, pageFaultData); + pageFaultHandler->allowCPUMemoryEviction(true, allocPtr, pageFaultData); } void PageFaultManager::unprotectAndTransferMemory(PageFaultManager *pageFaultHandler, void *allocPtr, PageFaultData &pageFaultData) { diff --git a/shared/source/page_fault_manager/cpu_page_fault_manager.h b/shared/source/page_fault_manager/cpu_page_fault_manager.h index 1c478efb95..ba016b695d 100644 --- a/shared/source/page_fault_manager/cpu_page_fault_manager.h +++ b/shared/source/page_fault_manager/cpu_page_fault_manager.h @@ -57,13 +57,13 @@ class PageFaultManager : public NonCopyableOrMovableClass { virtual bool checkFaultHandlerFromPageFaultManager() = 0; virtual void registerFaultHandler() = 0; virtual void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) = 0; - virtual void allowCPUMemoryEvictionImpl(void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) = 0; + virtual void allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) = 0; MOCKABLE_VIRTUAL bool verifyPageFault(void *ptr); MOCKABLE_VIRTUAL void transferToGpu(void *ptr, void *cmdQ); MOCKABLE_VIRTUAL void setAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager); MOCKABLE_VIRTUAL void setCpuAllocEvictable(bool evictable, void *ptr, SVMAllocsManager *unifiedMemoryManager); - MOCKABLE_VIRTUAL void allowCPUMemoryEviction(void *ptr, PageFaultData &pageFaultData); + MOCKABLE_VIRTUAL void allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData); static void transferAndUnprotectMemory(PageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData); static void unprotectAndTransferMemory(PageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData); diff --git a/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.cpp b/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.cpp index 1724ed5b57..329cbaf353 100644 --- a/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.cpp +++ b/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.cpp @@ -97,6 +97,6 @@ void PageFaultManagerLinux::evictMemoryAfterImplCopy(GraphicsAllocation *allocat } } -void PageFaultManagerLinux::allowCPUMemoryEvictionImpl(void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) {} +void PageFaultManagerLinux::allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) {} } // namespace NEO diff --git a/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h b/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h index e540444427..9ff966c6f8 100644 --- a/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h +++ b/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h @@ -25,7 +25,7 @@ class PageFaultManagerLinux : public PageFaultManager { void protectCPUMemoryAccess(void *ptr, size_t size) override; void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override; - void allowCPUMemoryEvictionImpl(void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override; + void allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override; bool checkFaultHandlerFromPageFaultManager() override; void registerFaultHandler() override; diff --git a/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.cpp b/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.cpp index dab525c566..088eb887dd 100644 --- a/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.cpp +++ b/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.cpp @@ -69,7 +69,7 @@ void PageFaultManagerWindows::protectCPUMemoryAccess(void *ptr, size_t size) { void PageFaultManagerWindows::evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) {} -void PageFaultManagerWindows::allowCPUMemoryEvictionImpl(void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) { +void PageFaultManagerWindows::allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) { NEO::SvmAllocationData *allocData = memoryData[ptr].unifiedMemoryManager->getSVMAlloc(ptr); UNRECOVERABLE_IF(allocData == nullptr); @@ -77,7 +77,14 @@ void PageFaultManagerWindows::allowCPUMemoryEvictionImpl(void *ptr, CommandStrea auto &residencyController = static_cast(&csr.getOsContext())->getResidencyController(); auto lock = residencyController.acquireLock(); - csr.getEvictionAllocations().push_back(allocData->cpuAllocation); + auto &evictContainer = csr.getEvictionAllocations(); + auto iter = std::find(evictContainer.begin(), evictContainer.end(), allocData->cpuAllocation); + auto allocInEvictionList = iter != evictContainer.end(); + if (evict && !allocInEvictionList) { + evictContainer.push_back(allocData->cpuAllocation); + } else if (!evict && allocInEvictionList) { + evictContainer.erase(iter); + } } } diff --git a/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h b/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h index 51d9c0e997..5c183397bb 100644 --- a/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h +++ b/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2023 Intel Corporation + * Copyright (C) 2019-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -25,7 +25,7 @@ class PageFaultManagerWindows : public PageFaultManager { void protectCPUMemoryAccess(void *ptr, size_t size) override; void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override; - void allowCPUMemoryEvictionImpl(void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override; + void allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override; bool checkFaultHandlerFromPageFaultManager() override; void registerFaultHandler() override; diff --git a/shared/test/common/mocks/mock_cpu_page_fault_manager.h b/shared/test/common/mocks/mock_cpu_page_fault_manager.h index 2708fa95ca..429602d881 100644 --- a/shared/test/common/mocks/mock_cpu_page_fault_manager.h +++ b/shared/test/common/mocks/mock_cpu_page_fault_manager.h @@ -58,7 +58,7 @@ class MockPageFaultManager : public PageFaultManager { setCpuAllocEvictableCalled++; isCpuAllocEvictable = evictable; } - void allowCPUMemoryEviction(void *ptr, PageFaultData &pageFaultData) override { + void allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) override { allowCPUMemoryEvictionCalled++; } void baseAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager) { @@ -73,12 +73,12 @@ class MockPageFaultManager : public PageFaultManager { void baseCpuAllocEvictable(bool evictable, void *ptr, SVMAllocsManager *unifiedMemoryManager) { PageFaultManager::setCpuAllocEvictable(evictable, ptr, unifiedMemoryManager); } - void baseAllowCPUMemoryEviction(void *ptr, PageFaultData &pageFaultData) { - PageFaultManager::allowCPUMemoryEviction(ptr, pageFaultData); + void baseAllowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) { + PageFaultManager::allowCPUMemoryEviction(evict, ptr, pageFaultData); } void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override {} - void allowCPUMemoryEvictionImpl(void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override { + void allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override { allowCPUMemoryEvictionImplCalled++; engineType = csr.getOsContext().getEngineType(); engineUsage = csr.getOsContext().getEngineUsage(); @@ -123,6 +123,7 @@ template class MockPageFaultManagerHandlerInvoke : public T { public: using T::allowCPUMemoryAccess; + using T::allowCPUMemoryEvictionImpl; using T::checkFaultHandlerFromPageFaultManager; using T::evictMemoryAfterImplCopy; using T::protectCPUMemoryAccess; diff --git a/shared/test/unit_test/page_fault_manager/cpu_page_fault_manager_tests.cpp b/shared/test/unit_test/page_fault_manager/cpu_page_fault_manager_tests.cpp index 2ebd232f76..1e98548875 100644 --- a/shared/test/unit_test/page_fault_manager/cpu_page_fault_manager_tests.cpp +++ b/shared/test/unit_test/page_fault_manager/cpu_page_fault_manager_tests.cpp @@ -719,7 +719,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenMigratedBetweenCpuAndGpu pageFaultManager->moveAllocationToGpuDomain(ptr); EXPECT_EQ(pageFaultManager->moveAllocationToGpuDomainCalled, 1); EXPECT_EQ(pageFaultManager->setCpuAllocEvictableCalled, 1); - EXPECT_EQ(pageFaultManager->allowCPUMemoryEvictionCalled, 0); + EXPECT_EQ(pageFaultManager->allowCPUMemoryEvictionCalled, 1); EXPECT_EQ(pageFaultManager->transferToGpuCalled, 1); EXPECT_EQ(pageFaultManager->protectMemoryCalled, 1); EXPECT_EQ(pageFaultManager->isCpuAllocEvictable, 0); @@ -731,7 +731,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenMigratedBetweenCpuAndGpu EXPECT_EQ(pageFaultManager->transferToCpuCalled, 1); EXPECT_EQ(pageFaultManager->allowMemoryAccessCalled, 1); EXPECT_EQ(pageFaultManager->setCpuAllocEvictableCalled, 2); - EXPECT_EQ(pageFaultManager->allowCPUMemoryEvictionCalled, 1); + EXPECT_EQ(pageFaultManager->allowCPUMemoryEvictionCalled, 2); EXPECT_EQ(pageFaultManager->allowedMemoryAccessAddress, ptr); EXPECT_EQ(pageFaultManager->accessAllowedSize, 10u); EXPECT_EQ(pageFaultManager->isCpuAllocEvictable, 1); diff --git a/shared/test/unit_test/page_fault_manager/windows/cpu_page_fault_manager_windows_tests.cpp b/shared/test/unit_test/page_fault_manager/windows/cpu_page_fault_manager_windows_tests.cpp index 89a7a494ae..f0b77aa7aa 100644 --- a/shared/test/unit_test/page_fault_manager/windows/cpu_page_fault_manager_windows_tests.cpp +++ b/shared/test/unit_test/page_fault_manager/windows/cpu_page_fault_manager_windows_tests.cpp @@ -5,9 +5,15 @@ * */ +#include "shared/source/os_interface/os_interface.h" +#include "shared/source/os_interface/windows/os_context_win.h" +#include "shared/source/os_interface/windows/wddm/wddm.h" #include "shared/source/os_interface/windows/windows_wrapper.h" #include "shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h" #include "shared/test/common/fixtures/cpu_page_fault_manager_tests_fixture.h" +#include "shared/test/common/helpers/debug_manager_state_restore.h" +#include "shared/test/common/helpers/engine_descriptor_helper.h" +#include "shared/test/common/mocks/mock_command_stream_receiver.h" #include "shared/test/common/mocks/mock_cpu_page_fault_manager.h" #include "gtest/gtest.h" @@ -117,3 +123,37 @@ TEST_F(PageFaultManagerWindowsTest, RemoveVectoredExceptionHandler(previousHandler); } + +TEST_F(PageFaultManagerTest, + givenDefaultSaHandlerWhenCPUMemoryEvictionIsCalledThenAllocAddedToEvictionListOnlyOnce) { + DebugManagerStateRestore restore; + debugManager.flags.AllocateSharedAllocationsWithCpuAndGpuStorage.set(true); + executionEnvironment.memoryManager.reset(memoryManager.release()); + auto rootDeviceEnvironment = executionEnvironment.rootDeviceEnvironments[0].get(); + auto wddm = std::unique_ptr(Wddm::createWddm(nullptr, *rootDeviceEnvironment)); + auto osContext = std::make_unique(*wddm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor()); + auto csr = std::make_unique(executionEnvironment, 0, 1); + csr->setupContext(*osContext); + auto unifiedMemoryManager = std::make_unique(executionEnvironment.memoryManager.get(), false); + auto pageFaultManager = std::make_unique(); + + OSInterface osInterface; + RootDeviceIndicesContainer rootDeviceIndices = {0}; + std::map deviceBitfields{{0, 0b1}}; + SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::sharedUnifiedMemory, 1, rootDeviceIndices, deviceBitfields); + auto ptr = unifiedMemoryManager->createUnifiedAllocationWithDeviceStorage(4096u, {}, unifiedMemoryProperties); + void *cmdQ = reinterpret_cast(0xFFFF); + pageFaultManager->insertAllocation(ptr, 10, unifiedMemoryManager.get(), cmdQ, {}); + + EXPECT_EQ(0u, csr->getEvictionAllocations().size()); + pageFaultManager->allowCPUMemoryEvictionImpl(true, ptr, *csr, &osInterface); + EXPECT_EQ(1u, csr->getEvictionAllocations().size()); + + pageFaultManager->allowCPUMemoryEvictionImpl(true, ptr, *csr, &osInterface); + EXPECT_EQ(1u, csr->getEvictionAllocations().size()); + + pageFaultManager->allowCPUMemoryEvictionImpl(false, ptr, *csr, &osInterface); + EXPECT_EQ(0u, csr->getEvictionAllocations().size()); + + unifiedMemoryManager->freeSVMAlloc(ptr); +} \ No newline at end of file diff --git a/shared/test/unit_test/ult_specific_config.cpp b/shared/test/unit_test/ult_specific_config.cpp index a70d2b7dfc..b6d93c1ca1 100644 --- a/shared/test/unit_test/ult_specific_config.cpp +++ b/shared/test/unit_test/ult_specific_config.cpp @@ -33,7 +33,7 @@ void PageFaultManager::transferToCpu(void *ptr, size_t size, void *cmdQ) { } void PageFaultManager::transferToGpu(void *ptr, void *cmdQ) { } -void PageFaultManager::allowCPUMemoryEviction(void *ptr, PageFaultData &pageFaultData) { +void PageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) { } void RootDeviceEnvironment::initApiGfxCoreHelper() {