From b81a78d0e99a0b6ab6f564cab6be157b2c802a55 Mon Sep 17 00:00:00 2001 From: Lukasz Jobczyk Date: Tue, 15 Sep 2020 12:35:17 +0200 Subject: [PATCH] Evict USM cpu allocation after migration Related-To: NEO-5007 Change-Id: I3c91af3ca22cb6233d530b252cc0c75d8fc2f8b5 Signed-off-by: Lukasz Jobczyk --- .../memory/cpu_page_fault_memory_manager.cpp | 2 + .../cpu_page_fault_manager_memory_sync.cpp | 3 + .../aub_tests/fixtures/aub_fixture.h | 2 + ...u_page_fault_manager_memory_sync_tests.cpp | 7 ++ .../test/unit_test/test_files/igdrcl.config | 1 + .../debug_settings/debug_variables_base.inl | 1 + .../cpu_page_fault_manager.h | 3 + .../linux/cpu_page_fault_manager_linux.cpp | 12 +++ .../linux/cpu_page_fault_manager_linux.h | 3 + .../cpu_page_fault_manager_windows.cpp | 2 + .../windows/cpu_page_fault_manager_windows.h | 1 + .../cpu_page_fault_manager_linux_tests.cpp | 77 +++++++++++++++++++ .../mock_cpu_page_fault_manager.h | 2 + 13 files changed, 116 insertions(+) diff --git a/level_zero/core/source/memory/cpu_page_fault_memory_manager.cpp b/level_zero/core/source/memory/cpu_page_fault_memory_manager.cpp index 59682457aa..93961ca21a 100644 --- a/level_zero/core/source/memory/cpu_page_fault_memory_manager.cpp +++ b/level_zero/core/source/memory/cpu_page_fault_memory_manager.cpp @@ -36,5 +36,7 @@ void PageFaultManager::transferToGpu(void *ptr, void *device) { allocData->cpuAllocation, allocData->size, false); UNRECOVERABLE_IF(ret); + + this->evictMemoryAfterImplCopy(allocData->cpuAllocation, deviceImp->getNEODevice()); } } // namespace NEO diff --git a/opencl/source/memory_manager/cpu_page_fault_manager_memory_sync.cpp b/opencl/source/memory_manager/cpu_page_fault_manager_memory_sync.cpp index 8a76194642..2e4c61b9ae 100644 --- a/opencl/source/memory_manager/cpu_page_fault_manager_memory_sync.cpp +++ b/opencl/source/memory_manager/cpu_page_fault_manager_memory_sync.cpp @@ -24,5 +24,8 @@ void PageFaultManager::transferToGpu(void *ptr, void *cmdQ) { UNRECOVERABLE_IF(retVal); retVal = commandQueue->finish(); UNRECOVERABLE_IF(retVal); + + auto allocData = memoryData[ptr].unifiedMemoryManager->getSVMAlloc(ptr); + this->evictMemoryAfterImplCopy(allocData->cpuAllocation, &commandQueue->getDevice()); } } // namespace NEO diff --git a/opencl/test/unit_test/aub_tests/fixtures/aub_fixture.h b/opencl/test/unit_test/aub_tests/fixtures/aub_fixture.h index 450934c00d..d92c264e5b 100644 --- a/opencl/test/unit_test/aub_tests/fixtures/aub_fixture.h +++ b/opencl/test/unit_test/aub_tests/fixtures/aub_fixture.h @@ -19,6 +19,7 @@ #include "opencl/source/platform/platform.h" #include "opencl/test/unit_test/command_queue/command_queue_fixture.h" #include "opencl/test/unit_test/mocks/mock_cl_device.h" +#include "opencl/test/unit_test/mocks/mock_memory_operations_handler.h" #include "opencl/test/unit_test/mocks/mock_platform.h" #include "gtest/gtest.h" @@ -42,6 +43,7 @@ class AUBFixture : public CommandQueueHwFixture { executionEnvironment = platform()->peekExecutionEnvironment(); executionEnvironment->prepareRootDeviceEnvironments(1u); executionEnvironment->rootDeviceEnvironments[0]->setHwInfo(&hwInfo); + executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface = std::make_unique(); device = std::make_unique(MockDevice::create(executionEnvironment, rootDeviceIndex)); diff --git a/opencl/test/unit_test/memory_manager/cpu_page_fault_manager_memory_sync_tests.cpp b/opencl/test/unit_test/memory_manager/cpu_page_fault_manager_memory_sync_tests.cpp index c9a62b1d72..d685fc004e 100644 --- a/opencl/test/unit_test/memory_manager/cpu_page_fault_manager_memory_sync_tests.cpp +++ b/opencl/test/unit_test/memory_manager/cpu_page_fault_manager_memory_sync_tests.cpp @@ -10,6 +10,7 @@ #include "shared/test/unit_test/test_macros/test_checks_shared.h" #include "opencl/source/command_queue/command_queue.h" +#include "opencl/test/unit_test/mocks/mock_cl_device.h" #include "opencl/test/unit_test/mocks/mock_command_queue.h" #include "opencl/test/unit_test/mocks/mock_graphics_allocation.h" #include "opencl/test/unit_test/mocks/mock_memory_manager.h" @@ -52,7 +53,9 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenSynchronizeMemoryThenEnq auto svmAllocsManager = std::make_unique(memoryManager.get()); void *alloc = svmAllocsManager->createSVMAlloc(mockRootDeviceIndex, 256, {}, mockDeviceBitfield); + auto device = std::unique_ptr(new MockClDevice{MockDevice::createWithNewExecutionEnvironment(nullptr)}); auto cmdQ = std::make_unique(); + cmdQ->device = device.get(); pageFaultManager->insertAllocation(alloc, 256, svmAllocsManager.get(), cmdQ.get(), {}); pageFaultManager->baseCpuTransfer(alloc, 10, cmdQ.get()); @@ -66,6 +69,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenSynchronizeMemoryThenEnq EXPECT_EQ(cmdQ->finishCalled, 1); svmAllocsManager->freeSVMAlloc(alloc); + cmdQ->device = nullptr; } TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenGpuTransferIsInvokedThenInsertMapOperation) { @@ -83,7 +87,9 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenGpuTransferIsInvokedThen auto memoryManager = std::make_unique(executionEnvironment); auto svmAllocsManager = std::make_unique(memoryManager.get()); void *alloc = svmAllocsManager->createSVMAlloc(mockRootDeviceIndex, 256, {}, mockDeviceBitfield); + auto device = std::unique_ptr(new MockClDevice{MockDevice::createWithNewExecutionEnvironment(nullptr)}); auto cmdQ = std::make_unique(); + cmdQ->device = device.get(); pageFaultManager->insertAllocation(alloc, 256, svmAllocsManager.get(), cmdQ.get(), {}); EXPECT_EQ(svmAllocsManager->insertSvmMapOperationCalled, 0); @@ -91,4 +97,5 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenGpuTransferIsInvokedThen EXPECT_EQ(svmAllocsManager->insertSvmMapOperationCalled, 1); svmAllocsManager->freeSVMAlloc(alloc); + cmdQ->device = nullptr; } diff --git a/opencl/test/unit_test/test_files/igdrcl.config b/opencl/test/unit_test/test_files/igdrcl.config index d133817810..88709e6dcf 100644 --- a/opencl/test/unit_test/test_files/igdrcl.config +++ b/opencl/test/unit_test/test_files/igdrcl.config @@ -85,6 +85,7 @@ DirectSubmissionEnableDebugBuffer = 0 DirectSubmissionDiagnosticExecutionCount = 30 DirectSubmissionDisableCacheFlush = 0 DirectSubmissionDisableMonitorFence = 0 +USMEvictAfterMigration = 1 EnableNullHardware = 0 ForceLinearImages = 0 ForceSLML3Config = 0 diff --git a/shared/source/debug_settings/debug_variables_base.inl b/shared/source/debug_settings/debug_variables_base.inl index bd9d781785..28f9de8756 100644 --- a/shared/source/debug_settings/debug_variables_base.inl +++ b/shared/source/debug_settings/debug_variables_base.inl @@ -137,6 +137,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionDiagnosticExecutionCount, 30, "N DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionOverrideBlitterSupport, -1, "Overrides default blitter support: -1: do not override, 0: disable engine support, 1: enable engine support with init start, 2: enable engine support without init start") DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionOverrideRenderSupport, -1, "Overrides default render support: -1: do not override, 0: disable engine support, 1: enable engine support with init start, 2: enable engine support without init start") DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionOverrideComputeSupport, -1, "Overrides default compute support: -1: do not override, 0: disable engine support, 1: enable engine support with init start, 2: enable engine support without init start") +DECLARE_DEBUG_VARIABLE(bool, USMEvictAfterMigration, true, "Evict USM allocation after implicit migration to GPU") DECLARE_DEBUG_VARIABLE(bool, DirectSubmissionDisableCacheFlush, false, "Disable dispatching cache flush commands") DECLARE_DEBUG_VARIABLE(bool, DirectSubmissionDisableMonitorFence, false, "Disable dispatching monitor fence commands") diff --git a/shared/source/page_fault_manager/cpu_page_fault_manager.h b/shared/source/page_fault_manager/cpu_page_fault_manager.h index 896f66e2aa..4140584dcc 100644 --- a/shared/source/page_fault_manager/cpu_page_fault_manager.h +++ b/shared/source/page_fault_manager/cpu_page_fault_manager.h @@ -16,6 +16,8 @@ #include namespace NEO { +class GraphicsAllocation; +class Device; class SVMAllocsManager; class PageFaultManager : public NonCopyableOrMovableClass { @@ -46,6 +48,7 @@ class PageFaultManager : public NonCopyableOrMovableClass { virtual void allowCPUMemoryAccess(void *ptr, size_t size) = 0; virtual void protectCPUMemoryAccess(void *ptr, size_t size) = 0; + virtual void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) = 0; virtual void broadcastWaitSignal() = 0; MOCKABLE_VIRTUAL void waitForCopy(); diff --git a/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.cpp b/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.cpp index 8b53063278..2bab71ffe7 100644 --- a/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.cpp +++ b/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.cpp @@ -7,7 +7,10 @@ #include "shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h" +#include "shared/source/debug_settings/debug_settings_manager.h" +#include "shared/source/device/device.h" #include "shared/source/helpers/debug_helpers.h" +#include "shared/source/memory_manager/memory_operations_handler.h" #include #include @@ -40,6 +43,9 @@ PageFaultManagerLinux::PageFaultManagerLinux() { retVal = sigaction(SIGUSR1, &pageFaultManagerHandler, &previousUserSignalHandler); UNRECOVERABLE_IF(retVal != 0); + + this->evictMemoryAfterCopy = DebugManager.flags.EnableDirectSubmission.get() && + DebugManager.flags.USMEvictAfterMigration.get(); } PageFaultManagerLinux::~PageFaultManagerLinux() { @@ -113,4 +119,10 @@ void PageFaultManagerLinux::sendSignalToThread(int threadId) { syscall(SYS_tkill, threadId, SIGUSR1); } +void PageFaultManagerLinux::evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) { + if (evictMemoryAfterCopy) { + device->getRootDeviceEnvironment().memoryOperationsInterface->evict(device, *allocation); + } +}; + } // namespace NEO diff --git a/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h b/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h index 8d43176cdf..77c522e14f 100644 --- a/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h +++ b/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h @@ -24,6 +24,7 @@ class PageFaultManagerLinux : public PageFaultManager { void allowCPUMemoryAccess(void *ptr, size_t size) override; void protectCPUMemoryAccess(void *ptr, size_t size) override; + void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override; void broadcastWaitSignal() override; MOCKABLE_VIRTUAL void sendSignalToThread(int threadId); @@ -34,5 +35,7 @@ class PageFaultManagerLinux : public PageFaultManager { struct sigaction previousPageFaultHandler = {}; struct sigaction previousUserSignalHandler = {}; + + bool evictMemoryAfterCopy = false; }; } // namespace NEO diff --git a/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.cpp b/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.cpp index 9da6200018..601f020cf9 100644 --- a/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.cpp +++ b/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.cpp @@ -51,6 +51,8 @@ void PageFaultManagerWindows::protectCPUMemoryAccess(void *ptr, size_t size) { UNRECOVERABLE_IF(!retVal); } +void PageFaultManagerWindows::evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) {} + void PageFaultManagerWindows::broadcastWaitSignal() {} } // namespace NEO diff --git a/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h b/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h index 76b2bee4a6..1db51e89da 100644 --- a/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h +++ b/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h @@ -25,6 +25,7 @@ class PageFaultManagerWindows : public PageFaultManager { void allowCPUMemoryAccess(void *ptr, size_t size) override; void protectCPUMemoryAccess(void *ptr, size_t size) override; + void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override; void broadcastWaitSignal() override; static std::function pageFaultHandler; diff --git a/shared/test/unit_test/page_fault_manager/linux/cpu_page_fault_manager_linux_tests.cpp b/shared/test/unit_test/page_fault_manager/linux/cpu_page_fault_manager_linux_tests.cpp index e4dc2bdb43..07e893a8e9 100644 --- a/shared/test/unit_test/page_fault_manager/linux/cpu_page_fault_manager_linux_tests.cpp +++ b/shared/test/unit_test/page_fault_manager/linux/cpu_page_fault_manager_linux_tests.cpp @@ -6,9 +6,14 @@ */ #include "shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h" +#include "shared/test/unit_test/helpers/debug_manager_state_restore.h" +#include "shared/test/unit_test/mocks/mock_device.h" #include "shared/test/unit_test/page_fault_manager/cpu_page_fault_manager_tests_fixture.h" #include "shared/test/unit_test/page_fault_manager/mock_cpu_page_fault_manager.h" +#include "opencl/test/unit_test/mocks/mock_graphics_allocation.h" +#include "opencl/test/unit_test/mocks/mock_memory_operations_handler.h" + #include "gtest/gtest.h" #include @@ -73,6 +78,78 @@ TEST_F(PageFaultManagerLinuxTest, whenPageFaultIsRaisedThenHandlerIsInvoked) { EXPECT_TRUE(pageFaultManager->handlerInvoked); } +struct MockOperationsInterface : public MockMemoryOperationsHandler { + bool evictCalled = false; + MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override { + this->evictCalled = true; + return MemoryOperationsStatus::UNSUPPORTED; + } +}; + +TEST_F(PageFaultManagerLinuxTest, givenDirectSubmissionAndUSMEvictWaEnabledWhenEvitMemoryAfterCopyThenMemoryOperationsHandlerEvictMethodIsCalled) { + DebugManagerStateRestore restorer; + DebugManager.flags.EnableDirectSubmission.set(true); + DebugManager.flags.USMEvictAfterMigration.set(true); + + auto pageFaultManager = std::make_unique(); + std::unique_ptr device(MockDevice::createWithNewExecutionEnvironment(nullptr)); + device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface = std::make_unique(); + auto operationInterface = static_cast(device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface.get()); + MockGraphicsAllocation allocation; + + EXPECT_FALSE(operationInterface->evictCalled); + pageFaultManager->evictMemoryAfterImplCopy(&allocation, device.get()); + EXPECT_TRUE(operationInterface->evictCalled); +} + +TEST_F(PageFaultManagerLinuxTest, givenDirectSubmissionEnabledAndUSMEvictWaDisabledWhenEvitMemoryAfterCopyThenMemoryOperationsHandlerEvictMethodIsNotCalled) { + DebugManagerStateRestore restorer; + DebugManager.flags.EnableDirectSubmission.set(true); + DebugManager.flags.USMEvictAfterMigration.set(false); + + auto pageFaultManager = std::make_unique(); + std::unique_ptr device(MockDevice::createWithNewExecutionEnvironment(nullptr)); + device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface = std::make_unique(); + auto operationInterface = static_cast(device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface.get()); + MockGraphicsAllocation allocation; + + EXPECT_FALSE(operationInterface->evictCalled); + pageFaultManager->evictMemoryAfterImplCopy(&allocation, device.get()); + EXPECT_FALSE(operationInterface->evictCalled); +} + +TEST_F(PageFaultManagerLinuxTest, givenDirectSubmissionAndUSMEvictWaDisabledWhenEvitMemoryAfterCopyThenMemoryOperationsHandlerEvictMethodIsNotCalled) { + DebugManagerStateRestore restorer; + DebugManager.flags.EnableDirectSubmission.set(false); + DebugManager.flags.USMEvictAfterMigration.set(false); + + auto pageFaultManager = std::make_unique(); + std::unique_ptr device(MockDevice::createWithNewExecutionEnvironment(nullptr)); + device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface = std::make_unique(); + auto operationInterface = static_cast(device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface.get()); + MockGraphicsAllocation allocation; + + EXPECT_FALSE(operationInterface->evictCalled); + pageFaultManager->evictMemoryAfterImplCopy(&allocation, device.get()); + EXPECT_FALSE(operationInterface->evictCalled); +} + +TEST_F(PageFaultManagerLinuxTest, givenDirectSubmissionDisabledAndUSMEvictWaEnabledWhenEvitMemoryAfterCopyThenMemoryOperationsHandlerEvictMethodIsNotCalled) { + DebugManagerStateRestore restorer; + DebugManager.flags.EnableDirectSubmission.set(false); + DebugManager.flags.USMEvictAfterMigration.set(true); + + auto pageFaultManager = std::make_unique(); + std::unique_ptr device(MockDevice::createWithNewExecutionEnvironment(nullptr)); + device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface = std::make_unique(); + auto operationInterface = static_cast(device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface.get()); + MockGraphicsAllocation allocation; + + EXPECT_FALSE(operationInterface->evictCalled); + pageFaultManager->evictMemoryAfterImplCopy(&allocation, device.get()); + EXPECT_FALSE(operationInterface->evictCalled); +} + TEST_F(PageFaultManagerLinuxTest, givenProtectedMemoryWhenTryingToAccessThenPageFaultIsRaisedAndMemoryIsAccessibleAfterHandling) { auto pageFaultManager = std::make_unique(); pageFaultManager->allowCPUMemoryAccessOnPageFault = true; diff --git a/shared/test/unit_test/page_fault_manager/mock_cpu_page_fault_manager.h b/shared/test/unit_test/page_fault_manager/mock_cpu_page_fault_manager.h index 7e6e8a93a1..401da6641e 100644 --- a/shared/test/unit_test/page_fault_manager/mock_cpu_page_fault_manager.h +++ b/shared/test/unit_test/page_fault_manager/mock_cpu_page_fault_manager.h @@ -50,6 +50,7 @@ class MockPageFaultManager : public PageFaultManager { PageFaultManager::transferToGpu(ptr, cmdQ); } void broadcastWaitSignal() override {} + void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override {} int allowMemoryAccessCalled = 0; int protectMemoryCalled = 0; @@ -69,6 +70,7 @@ template class MockPageFaultManagerHandlerInvoke : public T { public: using T::allowCPUMemoryAccess; + using T::evictMemoryAfterImplCopy; using T::protectCPUMemoryAccess; using T::T;