intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-12-26 15:03:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: add cpu alloc to eviction list only once

Browse Source 
Related-To: NEO-12572

Also, before migration to GPU domain, remove it from this list

Signed-off-by: Szymon Morek <szymon.morek@intel.com>
This commit is contained in:
Szymon Morek
2024-09-30 12:34:23 +00:00
committed by Compute-Runtime-Automation
parent 9db83b8231
commit b2fd1972a4
14 changed files with 73 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
level_zero/core/source/memory/cpu_page_fault_memory_manager.cpp
View File

@@ -42,7 +42,7 @@ void PageFaultManager::transferToGpu(void *ptr, void *device) {
this->evictMemoryAfterImplCopy(allocData->cpuAllocation, deviceImp->getNEODevice());
}
void PageFaultManager::allowCPUMemoryEviction(void *ptr, PageFaultData &pageFaultData) {
void PageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
L0::DeviceImp *deviceImp = static_cast<L0::DeviceImp *>(pageFaultData.cmdQ);
CommandStreamReceiver *csr = nullptr;
@@ -54,7 +54,7 @@ void PageFaultManager::allowCPUMemoryEviction(void *ptr, PageFaultData &pageFaul
UNRECOVERABLE_IF(csr == nullptr);
auto osInterface = deviceImp->getNEODevice()->getRootDeviceEnvironment().osInterface.get();
allowCPUMemoryEvictionImpl(ptr, *csr, osInterface);
allowCPUMemoryEvictionImpl(evict, ptr, *csr, osInterface);
}
} // namespace NEO

2
level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp
View File

@@ -841,7 +841,7 @@ TEST_F(CommandListMemAdvisePageFault, givenUnifiedMemoryAllocWhenAllowCPUMemoryE
NEO::PageFaultManager::PageFaultData pageData;
pageData.cmdQ = deviceImp;
mockPageFaultManager->baseAllowCPUMemoryEviction(ptr, pageData);
mockPageFaultManager->baseAllowCPUMemoryEviction(true, ptr, pageData);
EXPECT_EQ(mockPageFaultManager->allowCPUMemoryEvictionImplCalled, 1);
CommandStreamReceiver *csr = nullptr;

6
opencl/source/memory_manager/cpu_page_fault_manager_memory_sync.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2023 Intel Corporation
* Copyright (C) 2019-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -33,7 +33,7 @@ void PageFaultManager::transferToGpu(void *ptr, void *cmdQ) {
UNRECOVERABLE_IF(allocData == nullptr);
this->evictMemoryAfterImplCopy(allocData->cpuAllocation, &commandQueue->getDevice());
}
void PageFaultManager::allowCPUMemoryEviction(void *ptr, PageFaultData &pageFaultData) {
void PageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
auto commandQueue = static_cast<CommandQueue *>(pageFaultData.cmdQ);
auto allocData = memoryData[ptr].unifiedMemoryManager->getSVMAlloc(ptr);
@@ -42,7 +42,7 @@ void PageFaultManager::allowCPUMemoryEviction(void *ptr, PageFaultData &pageFaul
auto &csr = commandQueue->selectCsrForBuiltinOperation(csrSelectionArgs);
auto osInterface = commandQueue->getDevice().getRootDeviceEnvironment().osInterface.get();
allowCPUMemoryEvictionImpl(ptr, csr, osInterface);
allowCPUMemoryEvictionImpl(evict, ptr, csr, osInterface);
}
} // namespace NEO

4
opencl/test/unit_test/memory_manager/cpu_page_fault_manager_memory_sync_tests.cpp
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2023 Intel Corporation
* Copyright (C) 2019-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -124,7 +124,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenAllowCPUMemoryEvictionIs
NEO::PageFaultManager::PageFaultData pageData;
pageData.cmdQ = cmdQ.get();
pageFaultManager->baseAllowCPUMemoryEviction(alloc, pageData);
pageFaultManager->baseAllowCPUMemoryEviction(true, alloc, pageData);
EXPECT_EQ(pageFaultManager->allowCPUMemoryEvictionImplCalled, 1);
auto allocData = svmAllocsManager->getSVMAlloc(alloc);

3
shared/source/page_fault_manager/cpu_page_fault_manager.cpp
View File

@@ -73,6 +73,7 @@ void PageFaultManager::moveAllocationsWithinUMAllocsManagerToGpuDomain(SVMAllocs
inline void PageFaultManager::migrateStorageToGpuDomain(void *ptr, PageFaultData &pageFaultData) {
if (pageFaultData.domain == AllocationDomain::cpu) {
this->setCpuAllocEvictable(false, ptr, pageFaultData.unifiedMemoryManager);
this->allowCPUMemoryEviction(false, ptr, pageFaultData);
std::chrono::steady_clock::time_point start;
std::chrono::steady_clock::time_point end;
@@ -117,7 +118,7 @@ void PageFaultManager::transferAndUnprotectMemory(PageFaultManager *pageFaultHan
pageFaultHandler->migrateStorageToCpuDomain(allocPtr, pageFaultData);
pageFaultHandler->allowCPUMemoryAccess(allocPtr, pageFaultData.size);
pageFaultHandler->setCpuAllocEvictable(true, allocPtr, pageFaultData.unifiedMemoryManager);
pageFaultHandler->allowCPUMemoryEviction(allocPtr, pageFaultData);
pageFaultHandler->allowCPUMemoryEviction(true, allocPtr, pageFaultData);
}
void PageFaultManager::unprotectAndTransferMemory(PageFaultManager *pageFaultHandler, void *allocPtr, PageFaultData &pageFaultData) {

4
shared/source/page_fault_manager/cpu_page_fault_manager.h
View File

@@ -57,13 +57,13 @@ class PageFaultManager : public NonCopyableOrMovableClass {
virtual bool checkFaultHandlerFromPageFaultManager() = 0;
virtual void registerFaultHandler() = 0;
virtual void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) = 0;
virtual void allowCPUMemoryEvictionImpl(void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) = 0;
virtual void allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) = 0;
MOCKABLE_VIRTUAL bool verifyPageFault(void *ptr);
MOCKABLE_VIRTUAL void transferToGpu(void *ptr, void *cmdQ);
MOCKABLE_VIRTUAL void setAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager);
MOCKABLE_VIRTUAL void setCpuAllocEvictable(bool evictable, void *ptr, SVMAllocsManager *unifiedMemoryManager);
MOCKABLE_VIRTUAL void allowCPUMemoryEviction(void *ptr, PageFaultData &pageFaultData);
MOCKABLE_VIRTUAL void allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData);
static void transferAndUnprotectMemory(PageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData);
static void unprotectAndTransferMemory(PageFaultManager *pageFaultHandler, void *alloc, PageFaultData &pageFaultData);

2
shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.cpp
View File

@@ -97,6 +97,6 @@ void PageFaultManagerLinux::evictMemoryAfterImplCopy(GraphicsAllocation *allocat
}
}
void PageFaultManagerLinux::allowCPUMemoryEvictionImpl(void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) {}
void PageFaultManagerLinux::allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) {}
} // namespace NEO

2
shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h
View File

@@ -25,7 +25,7 @@ class PageFaultManagerLinux : public PageFaultManager {
void protectCPUMemoryAccess(void *ptr, size_t size) override;
void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override;
void allowCPUMemoryEvictionImpl(void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override;
void allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override;
bool checkFaultHandlerFromPageFaultManager() override;
void registerFaultHandler() override;

11
shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.cpp
View File

@@ -69,7 +69,7 @@ void PageFaultManagerWindows::protectCPUMemoryAccess(void *ptr, size_t size) {
void PageFaultManagerWindows::evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) {}
void PageFaultManagerWindows::allowCPUMemoryEvictionImpl(void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) {
void PageFaultManagerWindows::allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) {
NEO::SvmAllocationData *allocData = memoryData[ptr].unifiedMemoryManager->getSVMAlloc(ptr);
UNRECOVERABLE_IF(allocData == nullptr);
@@ -77,7 +77,14 @@ void PageFaultManagerWindows::allowCPUMemoryEvictionImpl(void *ptr, CommandStrea
auto &residencyController = static_cast<OsContextWin *>(&csr.getOsContext())->getResidencyController();
auto lock = residencyController.acquireLock();
csr.getEvictionAllocations().push_back(allocData->cpuAllocation);
auto &evictContainer = csr.getEvictionAllocations();
auto iter = std::find(evictContainer.begin(), evictContainer.end(), allocData->cpuAllocation);
auto allocInEvictionList = iter != evictContainer.end();
if (evict && !allocInEvictionList) {
evictContainer.push_back(allocData->cpuAllocation);
} else if (!evict && allocInEvictionList) {
evictContainer.erase(iter);
}
}
}

4
shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2023 Intel Corporation
* Copyright (C) 2019-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@@ -25,7 +25,7 @@ class PageFaultManagerWindows : public PageFaultManager {
void protectCPUMemoryAccess(void *ptr, size_t size) override;
void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override;
void allowCPUMemoryEvictionImpl(void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override;
void allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override;
bool checkFaultHandlerFromPageFaultManager() override;
void registerFaultHandler() override;

9
shared/test/common/mocks/mock_cpu_page_fault_manager.h
View File

@@ -58,7 +58,7 @@ class MockPageFaultManager : public PageFaultManager {
setCpuAllocEvictableCalled++;
isCpuAllocEvictable = evictable;
}
void allowCPUMemoryEviction(void *ptr, PageFaultData &pageFaultData) override {
void allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) override {
allowCPUMemoryEvictionCalled++;
}
void baseAubWritable(bool writable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
@@ -73,12 +73,12 @@ class MockPageFaultManager : public PageFaultManager {
void baseCpuAllocEvictable(bool evictable, void *ptr, SVMAllocsManager *unifiedMemoryManager) {
PageFaultManager::setCpuAllocEvictable(evictable, ptr, unifiedMemoryManager);
}
void baseAllowCPUMemoryEviction(void *ptr, PageFaultData &pageFaultData) {
PageFaultManager::allowCPUMemoryEviction(ptr, pageFaultData);
void baseAllowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
PageFaultManager::allowCPUMemoryEviction(evict, ptr, pageFaultData);
}
void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override {}
void allowCPUMemoryEvictionImpl(void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override {
void allowCPUMemoryEvictionImpl(bool evict, void *ptr, CommandStreamReceiver &csr, OSInterface *osInterface) override {
allowCPUMemoryEvictionImplCalled++;
engineType = csr.getOsContext().getEngineType();
engineUsage = csr.getOsContext().getEngineUsage();
@@ -123,6 +123,7 @@ template <class T>
class MockPageFaultManagerHandlerInvoke : public T {
public:
using T::allowCPUMemoryAccess;
using T::allowCPUMemoryEvictionImpl;
using T::checkFaultHandlerFromPageFaultManager;
using T::evictMemoryAfterImplCopy;
using T::protectCPUMemoryAccess;

4
shared/test/unit_test/page_fault_manager/cpu_page_fault_manager_tests.cpp
View File

@@ -719,7 +719,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenMigratedBetweenCpuAndGpu
pageFaultManager->moveAllocationToGpuDomain(ptr);
EXPECT_EQ(pageFaultManager->moveAllocationToGpuDomainCalled, 1);
EXPECT_EQ(pageFaultManager->setCpuAllocEvictableCalled, 1);
EXPECT_EQ(pageFaultManager->allowCPUMemoryEvictionCalled, 0);
EXPECT_EQ(pageFaultManager->allowCPUMemoryEvictionCalled, 1);
EXPECT_EQ(pageFaultManager->transferToGpuCalled, 1);
EXPECT_EQ(pageFaultManager->protectMemoryCalled, 1);
EXPECT_EQ(pageFaultManager->isCpuAllocEvictable, 0);
@@ -731,7 +731,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenMigratedBetweenCpuAndGpu
EXPECT_EQ(pageFaultManager->transferToCpuCalled, 1);
EXPECT_EQ(pageFaultManager->allowMemoryAccessCalled, 1);
EXPECT_EQ(pageFaultManager->setCpuAllocEvictableCalled, 2);
EXPECT_EQ(pageFaultManager->allowCPUMemoryEvictionCalled, 1);
EXPECT_EQ(pageFaultManager->allowCPUMemoryEvictionCalled, 2);
EXPECT_EQ(pageFaultManager->allowedMemoryAccessAddress, ptr);
EXPECT_EQ(pageFaultManager->accessAllowedSize, 10u);
EXPECT_EQ(pageFaultManager->isCpuAllocEvictable, 1);

40
shared/test/unit_test/page_fault_manager/windows/cpu_page_fault_manager_windows_tests.cpp
View File

@@ -5,9 +5,15 @@
*
*/
#include "shared/source/os_interface/os_interface.h"
#include "shared/source/os_interface/windows/os_context_win.h"
#include "shared/source/os_interface/windows/wddm/wddm.h"
#include "shared/source/os_interface/windows/windows_wrapper.h"
#include "shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h"
#include "shared/test/common/fixtures/cpu_page_fault_manager_tests_fixture.h"
#include "shared/test/common/helpers/debug_manager_state_restore.h"
#include "shared/test/common/helpers/engine_descriptor_helper.h"
#include "shared/test/common/mocks/mock_command_stream_receiver.h"
#include "shared/test/common/mocks/mock_cpu_page_fault_manager.h"
#include "gtest/gtest.h"
@@ -117,3 +123,37 @@ TEST_F(PageFaultManagerWindowsTest,
RemoveVectoredExceptionHandler(previousHandler);
}
TEST_F(PageFaultManagerTest,
givenDefaultSaHandlerWhenCPUMemoryEvictionIsCalledThenAllocAddedToEvictionListOnlyOnce) {
DebugManagerStateRestore restore;
debugManager.flags.AllocateSharedAllocationsWithCpuAndGpuStorage.set(true);
executionEnvironment.memoryManager.reset(memoryManager.release());
auto rootDeviceEnvironment = executionEnvironment.rootDeviceEnvironments[0].get();
auto wddm = std::unique_ptr<Wddm>(Wddm::createWddm(nullptr, *rootDeviceEnvironment));
auto osContext = std::make_unique<OsContextWin>(*wddm, 0, 0u, EngineDescriptorHelper::getDefaultDescriptor());
auto csr = std::make_unique<MockCommandStreamReceiver>(executionEnvironment, 0, 1);
csr->setupContext(*osContext);
auto unifiedMemoryManager = std::make_unique<SVMAllocsManager>(executionEnvironment.memoryManager.get(), false);
auto pageFaultManager = std::make_unique<MockPageFaultManagerWindows>();
OSInterface osInterface;
RootDeviceIndicesContainer rootDeviceIndices = {0};
std::map<uint32_t, DeviceBitfield> deviceBitfields{{0, 0b1}};
SVMAllocsManager::UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::sharedUnifiedMemory, 1, rootDeviceIndices, deviceBitfields);
auto ptr = unifiedMemoryManager->createUnifiedAllocationWithDeviceStorage(4096u, {}, unifiedMemoryProperties);
void *cmdQ = reinterpret_cast<void *>(0xFFFF);
pageFaultManager->insertAllocation(ptr, 10, unifiedMemoryManager.get(), cmdQ, {});
EXPECT_EQ(0u, csr->getEvictionAllocations().size());
pageFaultManager->allowCPUMemoryEvictionImpl(true, ptr, *csr, &osInterface);
EXPECT_EQ(1u, csr->getEvictionAllocations().size());
pageFaultManager->allowCPUMemoryEvictionImpl(true, ptr, *csr, &osInterface);
EXPECT_EQ(1u, csr->getEvictionAllocations().size());
pageFaultManager->allowCPUMemoryEvictionImpl(false, ptr, *csr, &osInterface);
EXPECT_EQ(0u, csr->getEvictionAllocations().size());
unifiedMemoryManager->freeSVMAlloc(ptr);
}

2
shared/test/unit_test/ult_specific_config.cpp
View File

@@ -33,7 +33,7 @@ void PageFaultManager::transferToCpu(void *ptr, size_t size, void *cmdQ) {
}
void PageFaultManager::transferToGpu(void *ptr, void *cmdQ) {
}
void PageFaultManager::allowCPUMemoryEviction(void *ptr, PageFaultData &pageFaultData) {
void PageFaultManager::allowCPUMemoryEviction(bool evict, void *ptr, PageFaultData &pageFaultData) {
}
void RootDeviceEnvironment::initApiGfxCoreHelper() {