Media
/
compute-runtime
mirror of https://github.com/intel/compute-runtime.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for mem advise to set cache policy in buffer object 

Signed-off-by: Milczarek, Slawomir <slawomir.milczarek@intel.com>
This commit is contained in:
Milczarek, Slawomir 2021-10-06 23:22:22 +00:00 committed by Compute-Runtime-Automation
parent 45de13233a
commit afa45bd9e7
13 changed files with 145 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
level_zero/core/source/cmdlist/cmdlist_hw.inl
View File

@ -23,6 +23,7 @@
#include "shared/source/indirect_heap/indirect_heap.h"
#include "shared/source/memory_manager/allocation_properties.h"
#include "shared/source/memory_manager/graphics_allocation.h"
#include "shared/source/memory_manager/memadvise_flags.h"
#include "shared/source/memory_manager/memory_manager.h"
#include "shared/source/os_interface/hw_info_config.h"
#include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
@ -736,7 +737,7 @@ template <GFXCORE_FAMILY gfxCoreFamily>
ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendMemAdvise(ze_device_handle_t hDevice,
const void *ptr, size_t size,
ze_memory_advice_t advice) {
MemAdviseFlags flags;
NEO::MemAdviseFlags flags;
flags.memadvise_flags = 0;
auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr);
@ -776,8 +777,8 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendMemAdvise(ze_device_hand
break;
}
NEO::PageFaultManager *pageFaultManager = nullptr;
pageFaultManager = device->getDriverHandle()->getMemoryManager()->getPageFaultManager();
auto memoryManager = device->getDriverHandle()->getMemoryManager();
auto pageFaultManager = memoryManager->getPageFaultManager();
if (pageFaultManager) {
/* If Read Only and Device Preferred Hints have been cleared, then cpu_migration of Shared memory can be re-enabled*/
if (flags.cpu_migration_blocked) {
@ -789,6 +790,10 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendMemAdvise(ze_device_hand
/* Given MemAdvise hints, use different gpu Domain Handler for the Page Fault Handling */
pageFaultManager->setGpuDomainHandler(L0::handleGpuDomainTransferForHwWithHints);
}
auto alloc = allocData->gpuAllocations.getGraphicsAllocation(deviceImp->getRootDeviceIndex());
memoryManager->setMemAdvise(alloc, flags);
deviceImp->memAdviseSharedAllocations[allocData] = flags;
return ZE_RESULT_SUCCESS;
}

18
level_zero/core/source/device/device_imp.h
View File

@ -9,6 +9,7 @@
#include "shared/source/helpers/topology_map.h"
#include "shared/source/memory_manager/allocations_list.h"
#include "shared/source/memory_manager/memadvise_flags.h"
#include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
#include "shared/source/utilities/spinlock.h"
@ -27,21 +28,6 @@
namespace L0 {
struct SysmanDevice;
typedef union {
uint8_t memadvise_flags; /* all memadvise_flags */
struct
{
uint8_t read_only : 1, /* ZE_MEMORY_ADVICE_SET_READ_MOSTLY or ZE_MEMORY_ADVICE_CLEAR_READ_MOSTLY */
device_preferred_location : 1, /* ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION or ZE_MEMORY_ADVICE_CLEAR_PREFERRED_LOCATION */
non_atomic : 1, /* ZE_MEMORY_ADVICE_SET_NON_ATOMIC_MOSTLY or ZE_MEMORY_ADVICE_CLEAR_NON_ATOMIC_MOSTLY */
cached_memory : 1, /* ZE_MEMORY_ADVICE_BIAS_CACHED or ZE_MEMORY_ADVICE_BIAS_UNCACHED */
cpu_migration_blocked : 1, /* ZE_MEMORY_ADVICE_SET_READ_MOSTLY and ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION */
reserved2 : 1,
reserved1 : 1,
reserved0 : 1;
};
} MemAdviseFlags;
struct DeviceImp : public Device {
uint32_t getRootDeviceIndex() override;
ze_result_t canAccessPeer(ze_device_handle_t hPeerDevice, ze_bool_t *value) override;
@ -135,7 +121,7 @@ struct DeviceImp : public Device {
NEO::SVMAllocsManager::MapBasedAllocationTracker peerAllocations;
NEO::SpinLock peerAllocationsMutex;
std::map<NEO::SvmAllocationData *, MemAdviseFlags> memAdviseSharedAllocations;
std::map<NEO::SvmAllocationData *, NEO::MemAdviseFlags> memAdviseSharedAllocations;
NEO::AllocationsList allocationsForReuse;
void createSysmanHandle(bool isSubDevice);

27
level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp
View File

@ -177,7 +177,7 @@ TEST_F(CommandListCreate, givenValidPtrThenAppendMemAdviseSetAndClearReadMostlyT
EXPECT_NE(nullptr, ptr);
ze_result_t returnValue;
L0::MemAdviseFlags flags;
NEO::MemAdviseFlags flags;
std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
ASSERT_NE(nullptr, commandList);
@ -209,7 +209,7 @@ TEST_F(CommandListCreate, givenValidPtrThenAppendMemAdviseSetAndClearPreferredLo
EXPECT_NE(nullptr, ptr);
ze_result_t returnValue;
L0::MemAdviseFlags flags;
NEO::MemAdviseFlags flags;
std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
ASSERT_NE(nullptr, commandList);
@ -241,7 +241,7 @@ TEST_F(CommandListCreate, givenValidPtrThenAppendMemAdviseSetAndClearNonAtomicTh
EXPECT_NE(nullptr, ptr);
ze_result_t returnValue;
L0::MemAdviseFlags flags;
NEO::MemAdviseFlags flags;
std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
ASSERT_NE(nullptr, commandList);
@ -273,7 +273,7 @@ TEST_F(CommandListCreate, givenValidPtrThenAppendMemAdviseSetAndClearCachingThen
EXPECT_NE(nullptr, ptr);
ze_result_t returnValue;
L0::MemAdviseFlags flags;
NEO::MemAdviseFlags flags;
std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
ASSERT_NE(nullptr, commandList);
@ -283,10 +283,13 @@ TEST_F(CommandListCreate, givenValidPtrThenAppendMemAdviseSetAndClearCachingThen
L0::DeviceImp *deviceImp = static_cast<L0::DeviceImp *>((L0::Device::fromHandle(device)));
flags = deviceImp->memAdviseSharedAllocations[allocData];
EXPECT_EQ(1, flags.cached_memory);
auto memoryManager = static_cast<MockMemoryManager *>(device->getDriverHandle()->getMemoryManager());
EXPECT_EQ(1, memoryManager->memAdviseFlags.cached_memory);
res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_BIAS_UNCACHED);
EXPECT_EQ(ZE_RESULT_SUCCESS, res);
flags = deviceImp->memAdviseSharedAllocations[allocData];
EXPECT_EQ(0, flags.cached_memory);
EXPECT_EQ(0, memoryManager->memAdviseFlags.cached_memory);
res = context->freeMem(ptr);
ASSERT_EQ(res, ZE_RESULT_SUCCESS);
@ -307,7 +310,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerThenAppend
EXPECT_NE(nullptr, ptr);
ze_result_t returnValue;
L0::MemAdviseFlags flags;
NEO::MemAdviseFlags flags;
std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
ASSERT_NE(nullptr, commandList);
@ -355,7 +358,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerThenGpuDom
EXPECT_NE(nullptr, ptr);
ze_result_t returnValue;
L0::MemAdviseFlags flags;
NEO::MemAdviseFlags flags;
std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
ASSERT_NE(nullptr, commandList);
@ -396,7 +399,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
EXPECT_NE(nullptr, ptr);
ze_result_t returnValue;
L0::MemAdviseFlags flags;
NEO::MemAdviseFlags flags;
std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
ASSERT_NE(nullptr, commandList);
@ -442,7 +445,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
EXPECT_NE(nullptr, ptr);
ze_result_t returnValue;
L0::MemAdviseFlags flags;
NEO::MemAdviseFlags flags;
std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
ASSERT_NE(nullptr, commandList);
@ -511,7 +514,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
EXPECT_NE(nullptr, ptr);
ze_result_t returnValue;
L0::MemAdviseFlags flags;
NEO::MemAdviseFlags flags;
std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
ASSERT_NE(nullptr, commandList);
@ -566,7 +569,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
EXPECT_NE(nullptr, ptr);
ze_result_t returnValue;
L0::MemAdviseFlags flags;
NEO::MemAdviseFlags flags;
std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
ASSERT_NE(nullptr, commandList);
@ -607,7 +610,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
EXPECT_NE(nullptr, ptr);
ze_result_t returnValue;
L0::MemAdviseFlags flags;
NEO::MemAdviseFlags flags;
std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
ASSERT_NE(nullptr, commandList);
@ -653,7 +656,7 @@ TEST_F(CommandListMemAdvisePageFault, givenInvalidPtrAndPageFaultHandlerAndGpuDo
EXPECT_NE(nullptr, ptr);
ze_result_t returnValue;
L0::MemAdviseFlags flags;
NEO::MemAdviseFlags flags;
std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
ASSERT_NE(nullptr, commandList);

52
opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp
View File

@ -4352,6 +4352,42 @@ TEST(DrmAllocationTest, givenDrmAllocationWhenCacheRegionIsSetSuccessfullyThenSe
}
}
TEST(DrmAllocationTest, givenDrmAllocationWhenBufferObjectIsCreatedThenApplyDefaultCachePolicy) {
auto executionEnvironment = std::make_unique<ExecutionEnvironment>();
executionEnvironment->prepareRootDeviceEnvironments(1);
DrmMock drm(*executionEnvironment->rootDeviceEnvironments[0]);
MockBufferObject bo(&drm, 0, 0, 1);
MockDrmAllocation allocation(GraphicsAllocation::AllocationType::BUFFER, MemoryPool::LocalMemory);
allocation.bufferObjects[0] = &bo;
for (auto bo : allocation.bufferObjects) {
if (bo != nullptr) {
EXPECT_EQ(CachePolicy::WriteBack, bo->peekCachePolicy());
}
}
}
TEST(DrmAllocationTest, givenDrmAllocationWhenSetCachePolicyIsCalledThenUpdatePolicyInBufferObject) {
auto executionEnvironment = std::make_unique<ExecutionEnvironment>();
executionEnvironment->prepareRootDeviceEnvironments(1);
DrmMock drm(*executionEnvironment->rootDeviceEnvironments[0]);
MockBufferObject bo(&drm, 0, 0, 1);
MockDrmAllocation allocation(GraphicsAllocation::AllocationType::BUFFER, MemoryPool::LocalMemory);
allocation.bufferObjects[0] = &bo;
allocation.setCachePolicy(CachePolicy::Uncached);
for (auto bo : allocation.bufferObjects) {
if (bo != nullptr) {
EXPECT_EQ(CachePolicy::Uncached, bo->peekCachePolicy());
}
}
}
TEST(DrmAllocationTest, givenBoWhenMarkingForCaptureThenBosAreMarked) {
auto executionEnvironment = std::make_unique<ExecutionEnvironment>();
executionEnvironment->prepareRootDeviceEnvironments(1);
@ -4678,4 +4714,20 @@ TEST_F(DrmMemoryManagerTest, givenDrmMemoryManagerWhenGetLocalMemoryIsCalledThen
EXPECT_EQ(0 * GB, memoryManager->getLocalMemorySize(rootDeviceIndex, 0xF));
}
TEST_F(DrmMemoryManagerTest, givenDrmMemoryManagerWhenSetMemAdviseIsCalledThenUpdateCachePolicyInBufferObject) {
TestedDrmMemoryManager memoryManager(false, false, false, *executionEnvironment);
BufferObject bo(mock, 1, 1024, 0);
DrmAllocation drmAllocation(0, GraphicsAllocation::AllocationType::UNIFIED_SHARED_MEMORY, &bo, nullptr, 0u, 0u, MemoryPool::LocalMemory);
EXPECT_EQ(&bo, drmAllocation.getBO());
for (auto isCached : {false, true}) {
MemAdviseFlags flags{};
flags.cached_memory = isCached;
memoryManager.setMemAdvise(&drmAllocation, flags);
EXPECT_EQ(isCached ? CachePolicy::WriteBack : CachePolicy::Uncached, bo.peekCachePolicy());
}
}
} // namespace NEO

29
shared/source/memory_manager/memadvise_flags.h Normal file
View File

@ -0,0 +1,29 @@
/*
* Copyright (C) 2021 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include <stdint.h>
namespace NEO {
typedef union {
uint8_t memadvise_flags; /* all memadvise_flags */
struct
{
uint8_t read_only : 1, /* ZE_MEMORY_ADVICE_SET_READ_MOSTLY or ZE_MEMORY_ADVICE_CLEAR_READ_MOSTLY */
device_preferred_location : 1, /* ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION or ZE_MEMORY_ADVICE_CLEAR_PREFERRED_LOCATION */
non_atomic : 1, /* ZE_MEMORY_ADVICE_SET_NON_ATOMIC_MOSTLY or ZE_MEMORY_ADVICE_CLEAR_NON_ATOMIC_MOSTLY */
cached_memory : 1, /* ZE_MEMORY_ADVICE_BIAS_CACHED or ZE_MEMORY_ADVICE_BIAS_UNCACHED */
cpu_migration_blocked : 1, /* ZE_MEMORY_ADVICE_SET_READ_MOSTLY and ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION */
reserved2 : 1,
reserved1 : 1,
reserved0 : 1;
};
} MemAdviseFlags;
} // namespace NEO

3
shared/source/memory_manager/memory_manager.h
View File

@ -19,6 +19,7 @@
#include "shared/source/memory_manager/graphics_allocation.h"
#include "shared/source/memory_manager/host_ptr_defines.h"
#include "shared/source/memory_manager/local_memory_usage.h"
#include "shared/source/memory_manager/memadvise_flags.h"
#include "shared/source/memory_manager/multi_graphics_allocation.h"
#include "shared/source/os_interface/os_interface.h"
#include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
@ -208,6 +209,8 @@ class MemoryManager {
virtual void registerSysMemAlloc(GraphicsAllocation *allocation){};
virtual void registerLocalMemAlloc(GraphicsAllocation *allocation, uint32_t rootDeviceIndex){};
virtual void setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags){};
bool isExternalAllocation(GraphicsAllocation::AllocationType allocationType);
LocalMemoryUsageBankSelector *getLocalMemoryUsageBankSelector(GraphicsAllocation::AllocationType allocationType, uint32_t rootDeviceIndex);

7
shared/source/os_interface/linux/cache_info.h
View File

@ -12,6 +12,13 @@
namespace NEO {
enum class CachePolicy : uint32_t {
Uncached = 0,
WriteCombined = 1,
WriteThrough = 2,
WriteBack = 3,
};
enum class CacheRegion : uint16_t {
Default = 0,
Region1,

8
shared/source/os_interface/linux/drm_allocation.cpp
View File

@ -29,6 +29,14 @@ uint64_t DrmAllocation::peekInternalHandle(MemoryManager *memoryManager) {
return static_cast<uint64_t>((static_cast<DrmMemoryManager *>(memoryManager))->obtainFdFromHandle(getBO()->peekHandle(), this->rootDeviceIndex));
}
void DrmAllocation::setCachePolicy(CachePolicy memType) {
for (auto bo : bufferObjects) {
if (bo != nullptr) {
bo->setCachePolicy(memType);
}
}
}
bool DrmAllocation::setCacheAdvice(Drm *drm, size_t regionSize, CacheRegion regionIndex) {
if (!drm->getCacheInfo()->getCacheRegion(regionSize, regionIndex)) {
return false;

2
shared/source/os_interface/linux/drm_allocation.h
View File

@ -13,6 +13,7 @@ namespace NEO {
class BufferObject;
class OsContext;
class Drm;
enum class CachePolicy : uint32_t;
enum class CacheRegion : uint16_t;
struct OsHandleLinux : OsHandle {
@ -66,6 +67,7 @@ class DrmAllocation : public GraphicsAllocation {
bool setCacheRegion(Drm *drm, CacheRegion regionIndex);
bool setCacheAdvice(Drm *drm, size_t regionSize, CacheRegion regionIndex);
void setCachePolicy(CachePolicy memType);
void *getMmapPtr() { return this->mmapPtr; }
void setMmapPtr(void *ptr) { this->mmapPtr = ptr; }

4
shared/source/os_interface/linux/drm_buffer_object.h
View File

@ -90,6 +90,9 @@ class BufferObject {
void setCacheRegion(CacheRegion regionIndex) { cacheRegion = regionIndex; }
CacheRegion peekCacheRegion() const { return cacheRegion; }
void setCachePolicy(CachePolicy memType) { cachePolicy = memType; }
CachePolicy peekCachePolicy() const { return cachePolicy; }
protected:
Drm *drm = nullptr;
bool perContextVmsUsed = false;
@ -115,6 +118,7 @@ class BufferObject {
uint64_t unmapSize = 0;
CacheRegion cacheRegion = CacheRegion::Default;
CachePolicy cachePolicy = CachePolicy::WriteBack;
std::vector<std::array<bool, EngineLimits::maxHandleCount>> bindInfo;
StackVec<uint32_t, 2> bindExtHandles;

7
shared/source/os_interface/linux/drm_memory_manager.cpp
View File

@ -201,6 +201,13 @@ bool DrmMemoryManager::isKmdMigrationAvailable(uint32_t rootDeviceIndex) {
return useKmdMigration;
}
void DrmMemoryManager::setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags) {
DrmAllocation *drmAllocation = static_cast<DrmAllocation *>(gfxAllocation);
CachePolicy memType = flags.cached_memory ? CachePolicy::WriteBack : CachePolicy::Uncached;
drmAllocation->setCachePolicy(memType);
}
NEO::BufferObject *DrmMemoryManager::allocUserptr(uintptr_t address, size_t size, uint64_t flags, uint32_t rootDeviceIndex) {
drm_i915_gem_userptr userptr = {};
userptr.user_ptr = address;

2
shared/source/os_interface/linux/drm_memory_manager.h
View File

@ -65,6 +65,8 @@ class DrmMemoryManager : public MemoryManager {
bool isKmdMigrationAvailable(uint32_t rootDeviceIndex) override;
void setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags) override;
std::unique_lock<std::mutex> acquireAllocLock();
std::vector<GraphicsAllocation *> &getSysMemAllocs();
std::vector<GraphicsAllocation *> &getLocalMemAllocs(uint32_t rootDeviceIndex);

6
shared/test/common/mocks/mock_memory_manager.h
View File

@ -141,6 +141,11 @@ class MockMemoryManager : public MemoryManagerCreate<OsAgnosticMemoryManager> {
}
void forceLimitedRangeAllocator(uint32_t rootDeviceIndex, uint64_t range) { getGfxPartition(rootDeviceIndex)->init(range, 0, 0, gfxPartitions.size()); }
void setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags) override {
memAdviseFlags = flags;
MemoryManager::setMemAdvise(gfxAllocation, flags);
}
uint32_t freeGraphicsMemoryCalled = 0u;
uint32_t unlockResourceCalled = 0u;
uint32_t lockResourceCalled = 0u;
@ -178,6 +183,7 @@ class MockMemoryManager : public MemoryManagerCreate<OsAgnosticMemoryManager> {
std::unique_ptr<MockExecutionEnvironment> mockExecutionEnvironment;
DeviceBitfield recentlyPassedDeviceBitfield{};
std::unique_ptr<MultiGraphicsAllocation> waitAllocations = nullptr;
MemAdviseFlags memAdviseFlags{};
};
class GMockMemoryManager : public MockMemoryManager {