From 5aeae0cf9909e514d8fa166c5e62a3b397f122fc Mon Sep 17 00:00:00 2001 From: "Milczarek, Slawomir" Date: Mon, 11 Oct 2021 09:27:26 +0000 Subject: [PATCH] Add mem advise control flags to drm alocation Signed-off-by: Milczarek, Slawomir --- level_zero/core/source/cmdlist/cmdlist_hw.inl | 6 ++-- .../sources/cmdlist/test_cmdlist_1.cpp | 30 +++++++++++++++++-- .../mocks/linux/mock_drm_allocation.h | 1 + .../linux/drm_memory_manager_tests.cpp | 28 ++++++++++++++++- .../source/memory_manager/memadvise_flags.h | 6 +++- shared/source/memory_manager/memory_manager.h | 2 +- .../os_interface/linux/drm_allocation.h | 4 +++ .../linux/drm_allocation_extended.cpp | 11 +++++++ .../os_interface/linux/drm_memory_manager.cpp | 7 ++--- .../os_interface/linux/drm_memory_manager.h | 2 +- .../test/common/mocks/mock_memory_manager.h | 8 +++-- 11 files changed, 91 insertions(+), 14 deletions(-) diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.inl b/level_zero/core/source/cmdlist/cmdlist_hw.inl index b7486123fc..458695a37a 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.inl +++ b/level_zero/core/source/cmdlist/cmdlist_hw.inl @@ -792,12 +792,14 @@ ze_result_t CommandListCoreFamily::appendMemAdvise(ze_device_hand } auto alloc = allocData->gpuAllocations.getGraphicsAllocation(deviceImp->getRootDeviceIndex()); - memoryManager->setMemAdvise(alloc, flags); + if (!memoryManager->setMemAdvise(alloc, flags, deviceImp->getRootDeviceIndex())) { + return ZE_RESULT_ERROR_UNKNOWN; + } deviceImp->memAdviseSharedAllocations[allocData] = flags; return ZE_RESULT_SUCCESS; } - return ZE_RESULT_ERROR_UNKNOWN; + return ZE_RESULT_ERROR_INVALID_ARGUMENT; } template diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp index e03d246e1d..9f322af1ed 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp @@ -106,7 +106,7 @@ TEST_F(CommandListCreate, givenNonExistingPtrThenAppendMemAdviseReturnsError) { ASSERT_NE(nullptr, commandList); auto res = commandList->appendMemAdvise(device, nullptr, 0, ZE_MEMORY_ADVICE_SET_READ_MOSTLY); - EXPECT_EQ(ZE_RESULT_ERROR_UNKNOWN, res); + EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res); } TEST_F(CommandListCreate, givenNonExistingPtrThenAppendMemoryPrefetchReturnsError) { @@ -118,7 +118,33 @@ TEST_F(CommandListCreate, givenNonExistingPtrThenAppendMemoryPrefetchReturnsErro EXPECT_EQ(ZE_RESULT_ERROR_INVALID_ARGUMENT, res); } -TEST_F(CommandListCreate, givenValidPtrThenAppendMemAdviseReturnsSuccess) { +TEST_F(CommandListCreate, givenValidPtrWhenAppendMemAdviseFailsThenReturnError) { + size_t size = 10; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_device_mem_alloc_desc_t deviceDesc = {}; + auto res = context->allocDeviceMem(device->toHandle(), + &deviceDesc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + EXPECT_NE(nullptr, ptr); + + ze_result_t returnValue; + std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue)); + ASSERT_NE(nullptr, commandList); + + auto memoryManager = static_cast(device->getDriverHandle()->getMemoryManager()); + memoryManager->failSetMemAdvise = true; + + res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION); + EXPECT_EQ(ZE_RESULT_ERROR_UNKNOWN, res); + + res = context->freeMem(ptr); + ASSERT_EQ(res, ZE_RESULT_SUCCESS); +} + +TEST_F(CommandListCreate, givenValidPtrWhenAppendMemAdviseSucceedsThenReturnSuccess) { size_t size = 10; size_t alignment = 1u; void *ptr = nullptr; diff --git a/opencl/test/unit_test/mocks/linux/mock_drm_allocation.h b/opencl/test/unit_test/mocks/linux/mock_drm_allocation.h index 741034896c..06cf0ad75c 100644 --- a/opencl/test/unit_test/mocks/linux/mock_drm_allocation.h +++ b/opencl/test/unit_test/mocks/linux/mock_drm_allocation.h @@ -25,6 +25,7 @@ class MockBufferObject : public BufferObject { class MockDrmAllocation : public DrmAllocation { public: using DrmAllocation::bufferObjects; + using DrmAllocation::enabledMemAdviseFlags; using DrmAllocation::memoryPool; using DrmAllocation::registeredBoBindHandles; diff --git a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index cade040ab0..7b41534113 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -4389,6 +4389,32 @@ TEST(DrmAllocationTest, givenDrmAllocationWhenSetCachePolicyIsCalledThenUpdatePo } } +TEST(DrmAllocationTest, givenDrmAllocationWhenSetMemAdviseWithCachePolicyIsCalledThenUpdatePolicyInBufferObject) { + auto executionEnvironment = std::make_unique(); + executionEnvironment->prepareRootDeviceEnvironments(1); + + DrmMock drm(*executionEnvironment->rootDeviceEnvironments[0]); + + MockBufferObject bo(&drm, 0, 0, 1); + MockDrmAllocation allocation(GraphicsAllocation::AllocationType::BUFFER, MemoryPool::LocalMemory); + allocation.bufferObjects[0] = &bo; + + EXPECT_EQ(CachePolicy::WriteBack, bo.peekCachePolicy()); + + MemAdviseFlags memAdviseFlags{}; + EXPECT_TRUE(memAdviseFlags.cached_memory); + + for (auto cached : {true, false, true}) { + memAdviseFlags.cached_memory = cached; + + EXPECT_TRUE(allocation.setMemAdvise(&drm, memAdviseFlags)); + + EXPECT_EQ(cached ? CachePolicy::WriteBack : CachePolicy::Uncached, bo.peekCachePolicy()); + + EXPECT_EQ(memAdviseFlags.memadvise_flags, allocation.enabledMemAdviseFlags.memadvise_flags); + } +} + TEST(DrmAllocationTest, givenBoWhenMarkingForCaptureThenBosAreMarked) { auto executionEnvironment = std::make_unique(); executionEnvironment->prepareRootDeviceEnvironments(1); @@ -4726,7 +4752,7 @@ TEST_F(DrmMemoryManagerTest, givenDrmMemoryManagerWhenSetMemAdviseIsCalledThenUp MemAdviseFlags flags{}; flags.cached_memory = isCached; - memoryManager.setMemAdvise(&drmAllocation, flags); + EXPECT_TRUE(memoryManager.setMemAdvise(&drmAllocation, flags, rootDeviceIndex)); EXPECT_EQ(isCached ? CachePolicy::WriteBack : CachePolicy::Uncached, bo.peekCachePolicy()); } } diff --git a/shared/source/memory_manager/memadvise_flags.h b/shared/source/memory_manager/memadvise_flags.h index e86fc623a2..cc358322da 100644 --- a/shared/source/memory_manager/memadvise_flags.h +++ b/shared/source/memory_manager/memadvise_flags.h @@ -11,7 +11,7 @@ namespace NEO { -typedef union { +typedef union MemAdviseFlagsTag { uint8_t memadvise_flags; /* all memadvise_flags */ struct { @@ -24,6 +24,10 @@ typedef union { reserved1 : 1, reserved0 : 1; }; + MemAdviseFlagsTag() { + memadvise_flags = 0; + cached_memory = 1; + } } MemAdviseFlags; } // namespace NEO diff --git a/shared/source/memory_manager/memory_manager.h b/shared/source/memory_manager/memory_manager.h index 167fb4f3c8..6eb810fcf2 100644 --- a/shared/source/memory_manager/memory_manager.h +++ b/shared/source/memory_manager/memory_manager.h @@ -209,7 +209,7 @@ class MemoryManager { virtual void registerSysMemAlloc(GraphicsAllocation *allocation){}; virtual void registerLocalMemAlloc(GraphicsAllocation *allocation, uint32_t rootDeviceIndex){}; - virtual void setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags){}; + virtual bool setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags, uint32_t rootDeviceIndex) { return true; } bool isExternalAllocation(GraphicsAllocation::AllocationType allocationType); LocalMemoryUsageBankSelector *getLocalMemoryUsageBankSelector(GraphicsAllocation::AllocationType allocationType, uint32_t rootDeviceIndex); diff --git a/shared/source/os_interface/linux/drm_allocation.h b/shared/source/os_interface/linux/drm_allocation.h index 95aa9b11d2..bc072222c3 100644 --- a/shared/source/os_interface/linux/drm_allocation.h +++ b/shared/source/os_interface/linux/drm_allocation.h @@ -7,6 +7,7 @@ #pragma once #include "shared/source/memory_manager/graphics_allocation.h" +#include "shared/source/memory_manager/memadvise_flags.h" #include "shared/source/memory_manager/memory_manager.h" namespace NEO { @@ -73,6 +74,8 @@ class DrmAllocation : public GraphicsAllocation { bool setCacheAdvice(Drm *drm, size_t regionSize, CacheRegion regionIndex); void setCachePolicy(CachePolicy memType); + bool setMemAdvise(Drm *drm, MemAdviseFlags flags); + void *getMmapPtr() { return this->mmapPtr; } void setMmapPtr(void *ptr) { this->mmapPtr = ptr; } size_t getMmapSize() { return this->mmapSize; } @@ -89,6 +92,7 @@ class DrmAllocation : public GraphicsAllocation { protected: BufferObjects bufferObjects{}; StackVec registeredBoBindHandles; + MemAdviseFlags enabledMemAdviseFlags{}; void *mmapPtr = nullptr; size_t mmapSize = 0u; diff --git a/shared/source/os_interface/linux/drm_allocation_extended.cpp b/shared/source/os_interface/linux/drm_allocation_extended.cpp index f66f165d95..2b5399a8e7 100644 --- a/shared/source/os_interface/linux/drm_allocation_extended.cpp +++ b/shared/source/os_interface/linux/drm_allocation_extended.cpp @@ -31,4 +31,15 @@ bool DrmAllocation::setCacheRegion(Drm *drm, CacheRegion regionIndex) { return setCacheAdvice(drm, 0, regionIndex); } +bool DrmAllocation::setMemAdvise(Drm *drm, MemAdviseFlags flags) { + if (flags.cached_memory != enabledMemAdviseFlags.cached_memory) { + CachePolicy memType = flags.cached_memory ? CachePolicy::WriteBack : CachePolicy::Uncached; + setCachePolicy(memType); + } + + enabledMemAdviseFlags = flags; + + return true; +} + } // namespace NEO diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index 92b44fbe91..9cb1fd8221 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -201,11 +201,10 @@ bool DrmMemoryManager::isKmdMigrationAvailable(uint32_t rootDeviceIndex) { return useKmdMigration; } -void DrmMemoryManager::setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags) { - DrmAllocation *drmAllocation = static_cast(gfxAllocation); +bool DrmMemoryManager::setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags, uint32_t rootDeviceIndex) { + auto drmAllocation = static_cast(gfxAllocation); - CachePolicy memType = flags.cached_memory ? CachePolicy::WriteBack : CachePolicy::Uncached; - drmAllocation->setCachePolicy(memType); + return drmAllocation->setMemAdvise(&this->getDrm(rootDeviceIndex), flags); } NEO::BufferObject *DrmMemoryManager::allocUserptr(uintptr_t address, size_t size, uint64_t flags, uint32_t rootDeviceIndex) { diff --git a/shared/source/os_interface/linux/drm_memory_manager.h b/shared/source/os_interface/linux/drm_memory_manager.h index d8591e8f0f..f846a8e912 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.h +++ b/shared/source/os_interface/linux/drm_memory_manager.h @@ -65,7 +65,7 @@ class DrmMemoryManager : public MemoryManager { bool isKmdMigrationAvailable(uint32_t rootDeviceIndex) override; - void setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags) override; + bool setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags, uint32_t rootDeviceIndex) override; std::unique_lock acquireAllocLock(); std::vector &getSysMemAllocs(); diff --git a/shared/test/common/mocks/mock_memory_manager.h b/shared/test/common/mocks/mock_memory_manager.h index 4408359a4b..6a5ce31898 100644 --- a/shared/test/common/mocks/mock_memory_manager.h +++ b/shared/test/common/mocks/mock_memory_manager.h @@ -141,9 +141,12 @@ class MockMemoryManager : public MemoryManagerCreate { } void forceLimitedRangeAllocator(uint32_t rootDeviceIndex, uint64_t range) { getGfxPartition(rootDeviceIndex)->init(range, 0, 0, gfxPartitions.size()); } - void setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags) override { + bool setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags, uint32_t rootDeviceIndex) override { memAdviseFlags = flags; - MemoryManager::setMemAdvise(gfxAllocation, flags); + if (failSetMemAdvise) { + return false; + } + return MemoryManager::setMemAdvise(gfxAllocation, flags, rootDeviceIndex); } uint32_t freeGraphicsMemoryCalled = 0u; @@ -172,6 +175,7 @@ class MockMemoryManager : public MemoryManagerCreate { bool failReserveAddress = false; bool failAllocateSystemMemory = false; bool failAllocate32Bit = false; + bool failSetMemAdvise = false; bool cpuCopyRequired = false; bool forceRenderCompressed = false; bool forceFailureInPrimaryAllocation = false;