Add support for mem advise to set cache policy in buffer object

Signed-off-by: Milczarek, Slawomir <slawomir.milczarek@intel.com>

level_zero/core/source/cmdlist/cmdlist_hw.inl

@@ -23,6 +23,7 @@
 #include "shared/source/indirect_heap/indirect_heap.h"
 #include "shared/source/memory_manager/allocation_properties.h"
 #include "shared/source/memory_manager/graphics_allocation.h"
+#include "shared/source/memory_manager/memadvise_flags.h"
 #include "shared/source/memory_manager/memory_manager.h"
 #include "shared/source/os_interface/hw_info_config.h"
 #include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
@@ -736,7 +737,7 @@ template <GFXCORE_FAMILY gfxCoreFamily>
 ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendMemAdvise(ze_device_handle_t hDevice,
                                                                   const void *ptr, size_t size,
                                                                   ze_memory_advice_t advice) {
-    MemAdviseFlags flags;
+    NEO::MemAdviseFlags flags;
     flags.memadvise_flags = 0;
 
     auto allocData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr);
@@ -776,8 +777,8 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendMemAdvise(ze_device_hand
             break;
         }
 
-        NEO::PageFaultManager *pageFaultManager = nullptr;
+        auto memoryManager = device->getDriverHandle()->getMemoryManager();
-        pageFaultManager = device->getDriverHandle()->getMemoryManager()->getPageFaultManager();
+        auto pageFaultManager = memoryManager->getPageFaultManager();
         if (pageFaultManager) {
             /* If Read Only and Device Preferred Hints have been cleared, then cpu_migration of Shared memory can be re-enabled*/
             if (flags.cpu_migration_blocked) {
@@ -789,6 +790,10 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendMemAdvise(ze_device_hand
             /* Given MemAdvise hints, use different gpu Domain Handler for the Page Fault Handling */
             pageFaultManager->setGpuDomainHandler(L0::handleGpuDomainTransferForHwWithHints);
         }
+
+        auto alloc = allocData->gpuAllocations.getGraphicsAllocation(deviceImp->getRootDeviceIndex());
+        memoryManager->setMemAdvise(alloc, flags);
+
         deviceImp->memAdviseSharedAllocations[allocData] = flags;
         return ZE_RESULT_SUCCESS;
     }

level_zero/core/source/device/device_imp.h

@@ -9,6 +9,7 @@
 
 #include "shared/source/helpers/topology_map.h"
 #include "shared/source/memory_manager/allocations_list.h"
+#include "shared/source/memory_manager/memadvise_flags.h"
 #include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
 #include "shared/source/utilities/spinlock.h"
 
@@ -27,21 +28,6 @@
 namespace L0 {
 struct SysmanDevice;
 
-typedef union {
-    uint8_t memadvise_flags; /* all memadvise_flags */
-    struct
-    {
-        uint8_t read_only : 1,             /* ZE_MEMORY_ADVICE_SET_READ_MOSTLY or ZE_MEMORY_ADVICE_CLEAR_READ_MOSTLY */
-            device_preferred_location : 1, /* ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION  or ZE_MEMORY_ADVICE_CLEAR_PREFERRED_LOCATION  */
-            non_atomic : 1,                /* ZE_MEMORY_ADVICE_SET_NON_ATOMIC_MOSTLY  or ZE_MEMORY_ADVICE_CLEAR_NON_ATOMIC_MOSTLY  */
-            cached_memory : 1,             /* ZE_MEMORY_ADVICE_BIAS_CACHED or ZE_MEMORY_ADVICE_BIAS_UNCACHED */
-            cpu_migration_blocked : 1,     /* ZE_MEMORY_ADVICE_SET_READ_MOSTLY and ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION */
-            reserved2 : 1,
-            reserved1 : 1,
-            reserved0 : 1;
-    };
-} MemAdviseFlags;
-
 struct DeviceImp : public Device {
     uint32_t getRootDeviceIndex() override;
     ze_result_t canAccessPeer(ze_device_handle_t hPeerDevice, ze_bool_t *value) override;
@@ -135,7 +121,7 @@ struct DeviceImp : public Device {
 
     NEO::SVMAllocsManager::MapBasedAllocationTracker peerAllocations;
     NEO::SpinLock peerAllocationsMutex;
-    std::map<NEO::SvmAllocationData *, MemAdviseFlags> memAdviseSharedAllocations;
+    std::map<NEO::SvmAllocationData *, NEO::MemAdviseFlags> memAdviseSharedAllocations;
     NEO::AllocationsList allocationsForReuse;
     void createSysmanHandle(bool isSubDevice);
 

level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_1.cpp

@@ -177,7 +177,7 @@ TEST_F(CommandListCreate, givenValidPtrThenAppendMemAdviseSetAndClearReadMostlyT
     EXPECT_NE(nullptr, ptr);
 
     ze_result_t returnValue;
-    L0::MemAdviseFlags flags;
+    NEO::MemAdviseFlags flags;
     std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
     ASSERT_NE(nullptr, commandList);
 
@@ -209,7 +209,7 @@ TEST_F(CommandListCreate, givenValidPtrThenAppendMemAdviseSetAndClearPreferredLo
     EXPECT_NE(nullptr, ptr);
 
     ze_result_t returnValue;
-    L0::MemAdviseFlags flags;
+    NEO::MemAdviseFlags flags;
     std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
     ASSERT_NE(nullptr, commandList);
 
@@ -241,7 +241,7 @@ TEST_F(CommandListCreate, givenValidPtrThenAppendMemAdviseSetAndClearNonAtomicTh
     EXPECT_NE(nullptr, ptr);
 
     ze_result_t returnValue;
-    L0::MemAdviseFlags flags;
+    NEO::MemAdviseFlags flags;
     std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
     ASSERT_NE(nullptr, commandList);
 
@@ -273,7 +273,7 @@ TEST_F(CommandListCreate, givenValidPtrThenAppendMemAdviseSetAndClearCachingThen
     EXPECT_NE(nullptr, ptr);
 
     ze_result_t returnValue;
-    L0::MemAdviseFlags flags;
+    NEO::MemAdviseFlags flags;
     std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
     ASSERT_NE(nullptr, commandList);
 
@@ -283,10 +283,13 @@ TEST_F(CommandListCreate, givenValidPtrThenAppendMemAdviseSetAndClearCachingThen
     L0::DeviceImp *deviceImp = static_cast<L0::DeviceImp *>((L0::Device::fromHandle(device)));
     flags = deviceImp->memAdviseSharedAllocations[allocData];
     EXPECT_EQ(1, flags.cached_memory);
+    auto memoryManager = static_cast<MockMemoryManager *>(device->getDriverHandle()->getMemoryManager());
+    EXPECT_EQ(1, memoryManager->memAdviseFlags.cached_memory);
     res = commandList->appendMemAdvise(device, ptr, size, ZE_MEMORY_ADVICE_BIAS_UNCACHED);
     EXPECT_EQ(ZE_RESULT_SUCCESS, res);
     flags = deviceImp->memAdviseSharedAllocations[allocData];
     EXPECT_EQ(0, flags.cached_memory);
+    EXPECT_EQ(0, memoryManager->memAdviseFlags.cached_memory);
 
     res = context->freeMem(ptr);
     ASSERT_EQ(res, ZE_RESULT_SUCCESS);
@@ -307,7 +310,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerThenAppend
     EXPECT_NE(nullptr, ptr);
 
     ze_result_t returnValue;
-    L0::MemAdviseFlags flags;
+    NEO::MemAdviseFlags flags;
     std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
     ASSERT_NE(nullptr, commandList);
 
@@ -355,7 +358,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerThenGpuDom
     EXPECT_NE(nullptr, ptr);
 
     ze_result_t returnValue;
-    L0::MemAdviseFlags flags;
+    NEO::MemAdviseFlags flags;
     std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
     ASSERT_NE(nullptr, commandList);
 
@@ -396,7 +399,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
     EXPECT_NE(nullptr, ptr);
 
     ze_result_t returnValue;
-    L0::MemAdviseFlags flags;
+    NEO::MemAdviseFlags flags;
     std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
     ASSERT_NE(nullptr, commandList);
 
@@ -442,7 +445,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
     EXPECT_NE(nullptr, ptr);
 
     ze_result_t returnValue;
-    L0::MemAdviseFlags flags;
+    NEO::MemAdviseFlags flags;
     std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
     ASSERT_NE(nullptr, commandList);
 
@@ -511,7 +514,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
     EXPECT_NE(nullptr, ptr);
 
     ze_result_t returnValue;
-    L0::MemAdviseFlags flags;
+    NEO::MemAdviseFlags flags;
     std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
     ASSERT_NE(nullptr, commandList);
 
@@ -566,7 +569,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
     EXPECT_NE(nullptr, ptr);
 
     ze_result_t returnValue;
-    L0::MemAdviseFlags flags;
+    NEO::MemAdviseFlags flags;
     std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
     ASSERT_NE(nullptr, commandList);
 
@@ -607,7 +610,7 @@ TEST_F(CommandListMemAdvisePageFault, givenValidPtrAndPageFaultHandlerAndGpuDoma
     EXPECT_NE(nullptr, ptr);
 
     ze_result_t returnValue;
-    L0::MemAdviseFlags flags;
+    NEO::MemAdviseFlags flags;
     std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
     ASSERT_NE(nullptr, commandList);
 
@@ -653,7 +656,7 @@ TEST_F(CommandListMemAdvisePageFault, givenInvalidPtrAndPageFaultHandlerAndGpuDo
     EXPECT_NE(nullptr, ptr);
 
     ze_result_t returnValue;
-    L0::MemAdviseFlags flags;
+    NEO::MemAdviseFlags flags;
     std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue));
     ASSERT_NE(nullptr, commandList);
 

opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp

@@ -4352,6 +4352,42 @@ TEST(DrmAllocationTest, givenDrmAllocationWhenCacheRegionIsSetSuccessfullyThenSe
     }
 }
 
+TEST(DrmAllocationTest, givenDrmAllocationWhenBufferObjectIsCreatedThenApplyDefaultCachePolicy) {
+    auto executionEnvironment = std::make_unique<ExecutionEnvironment>();
+    executionEnvironment->prepareRootDeviceEnvironments(1);
+
+    DrmMock drm(*executionEnvironment->rootDeviceEnvironments[0]);
+
+    MockBufferObject bo(&drm, 0, 0, 1);
+    MockDrmAllocation allocation(GraphicsAllocation::AllocationType::BUFFER, MemoryPool::LocalMemory);
+    allocation.bufferObjects[0] = &bo;
+
+    for (auto bo : allocation.bufferObjects) {
+        if (bo != nullptr) {
+            EXPECT_EQ(CachePolicy::WriteBack, bo->peekCachePolicy());
+        }
+    }
+}
+
+TEST(DrmAllocationTest, givenDrmAllocationWhenSetCachePolicyIsCalledThenUpdatePolicyInBufferObject) {
+    auto executionEnvironment = std::make_unique<ExecutionEnvironment>();
+    executionEnvironment->prepareRootDeviceEnvironments(1);
+
+    DrmMock drm(*executionEnvironment->rootDeviceEnvironments[0]);
+
+    MockBufferObject bo(&drm, 0, 0, 1);
+    MockDrmAllocation allocation(GraphicsAllocation::AllocationType::BUFFER, MemoryPool::LocalMemory);
+    allocation.bufferObjects[0] = &bo;
+
+    allocation.setCachePolicy(CachePolicy::Uncached);
+
+    for (auto bo : allocation.bufferObjects) {
+        if (bo != nullptr) {
+            EXPECT_EQ(CachePolicy::Uncached, bo->peekCachePolicy());
+        }
+    }
+}
+
 TEST(DrmAllocationTest, givenBoWhenMarkingForCaptureThenBosAreMarked) {
     auto executionEnvironment = std::make_unique<ExecutionEnvironment>();
     executionEnvironment->prepareRootDeviceEnvironments(1);
@@ -4678,4 +4714,20 @@ TEST_F(DrmMemoryManagerTest, givenDrmMemoryManagerWhenGetLocalMemoryIsCalledThen
     EXPECT_EQ(0 * GB, memoryManager->getLocalMemorySize(rootDeviceIndex, 0xF));
 }
 
+TEST_F(DrmMemoryManagerTest, givenDrmMemoryManagerWhenSetMemAdviseIsCalledThenUpdateCachePolicyInBufferObject) {
+    TestedDrmMemoryManager memoryManager(false, false, false, *executionEnvironment);
+    BufferObject bo(mock, 1, 1024, 0);
+
+    DrmAllocation drmAllocation(0, GraphicsAllocation::AllocationType::UNIFIED_SHARED_MEMORY, &bo, nullptr, 0u, 0u, MemoryPool::LocalMemory);
+    EXPECT_EQ(&bo, drmAllocation.getBO());
+
+    for (auto isCached : {false, true}) {
+        MemAdviseFlags flags{};
+        flags.cached_memory = isCached;
+
+        memoryManager.setMemAdvise(&drmAllocation, flags);
+        EXPECT_EQ(isCached ? CachePolicy::WriteBack : CachePolicy::Uncached, bo.peekCachePolicy());
+    }
+}
+
 } // namespace NEO

shared/source/memory_manager/memadvise_flags.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2021 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+namespace NEO {
+
+typedef union {
+    uint8_t memadvise_flags; /* all memadvise_flags */
+    struct
+    {
+        uint8_t read_only : 1,             /* ZE_MEMORY_ADVICE_SET_READ_MOSTLY or ZE_MEMORY_ADVICE_CLEAR_READ_MOSTLY */
+            device_preferred_location : 1, /* ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION  or ZE_MEMORY_ADVICE_CLEAR_PREFERRED_LOCATION  */
+            non_atomic : 1,                /* ZE_MEMORY_ADVICE_SET_NON_ATOMIC_MOSTLY  or ZE_MEMORY_ADVICE_CLEAR_NON_ATOMIC_MOSTLY  */
+            cached_memory : 1,             /* ZE_MEMORY_ADVICE_BIAS_CACHED or ZE_MEMORY_ADVICE_BIAS_UNCACHED */
+            cpu_migration_blocked : 1,     /* ZE_MEMORY_ADVICE_SET_READ_MOSTLY and ZE_MEMORY_ADVICE_SET_PREFERRED_LOCATION */
+            reserved2 : 1,
+            reserved1 : 1,
+            reserved0 : 1;
+    };
+} MemAdviseFlags;
+
+} // namespace NEO

shared/source/memory_manager/memory_manager.h

@@ -19,6 +19,7 @@
 #include "shared/source/memory_manager/graphics_allocation.h"
 #include "shared/source/memory_manager/host_ptr_defines.h"
 #include "shared/source/memory_manager/local_memory_usage.h"
+#include "shared/source/memory_manager/memadvise_flags.h"
 #include "shared/source/memory_manager/multi_graphics_allocation.h"
 #include "shared/source/os_interface/os_interface.h"
 #include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
@@ -208,6 +209,8 @@ class MemoryManager {
     virtual void registerSysMemAlloc(GraphicsAllocation *allocation){};
     virtual void registerLocalMemAlloc(GraphicsAllocation *allocation, uint32_t rootDeviceIndex){};
 
+    virtual void setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags){};
+
     bool isExternalAllocation(GraphicsAllocation::AllocationType allocationType);
     LocalMemoryUsageBankSelector *getLocalMemoryUsageBankSelector(GraphicsAllocation::AllocationType allocationType, uint32_t rootDeviceIndex);
 

shared/source/os_interface/linux/cache_info.h

@@ -12,6 +12,13 @@
 
 namespace NEO {
 
+enum class CachePolicy : uint32_t {
+    Uncached = 0,
+    WriteCombined = 1,
+    WriteThrough = 2,
+    WriteBack = 3,
+};
+
 enum class CacheRegion : uint16_t {
     Default = 0,
     Region1,

shared/source/os_interface/linux/drm_allocation.cpp

@@ -29,6 +29,14 @@ uint64_t DrmAllocation::peekInternalHandle(MemoryManager *memoryManager) {
     return static_cast<uint64_t>((static_cast<DrmMemoryManager *>(memoryManager))->obtainFdFromHandle(getBO()->peekHandle(), this->rootDeviceIndex));
 }
 
+void DrmAllocation::setCachePolicy(CachePolicy memType) {
+    for (auto bo : bufferObjects) {
+        if (bo != nullptr) {
+            bo->setCachePolicy(memType);
+        }
+    }
+}
+
 bool DrmAllocation::setCacheAdvice(Drm *drm, size_t regionSize, CacheRegion regionIndex) {
     if (!drm->getCacheInfo()->getCacheRegion(regionSize, regionIndex)) {
         return false;

shared/source/os_interface/linux/drm_allocation.h

@@ -13,6 +13,7 @@ namespace NEO {
 class BufferObject;
 class OsContext;
 class Drm;
+enum class CachePolicy : uint32_t;
 enum class CacheRegion : uint16_t;
 
 struct OsHandleLinux : OsHandle {
@@ -66,6 +67,7 @@ class DrmAllocation : public GraphicsAllocation {
 
     bool setCacheRegion(Drm *drm, CacheRegion regionIndex);
     bool setCacheAdvice(Drm *drm, size_t regionSize, CacheRegion regionIndex);
+    void setCachePolicy(CachePolicy memType);
 
     void *getMmapPtr() { return this->mmapPtr; }
     void setMmapPtr(void *ptr) { this->mmapPtr = ptr; }

shared/source/os_interface/linux/drm_buffer_object.h

@@ -90,6 +90,9 @@ class BufferObject {
     void setCacheRegion(CacheRegion regionIndex) { cacheRegion = regionIndex; }
     CacheRegion peekCacheRegion() const { return cacheRegion; }
 
+    void setCachePolicy(CachePolicy memType) { cachePolicy = memType; }
+    CachePolicy peekCachePolicy() const { return cachePolicy; }
+
   protected:
     Drm *drm = nullptr;
     bool perContextVmsUsed = false;
@@ -115,6 +118,7 @@ class BufferObject {
     uint64_t unmapSize = 0;
 
     CacheRegion cacheRegion = CacheRegion::Default;
+    CachePolicy cachePolicy = CachePolicy::WriteBack;
 
     std::vector<std::array<bool, EngineLimits::maxHandleCount>> bindInfo;
     StackVec<uint32_t, 2> bindExtHandles;

shared/source/os_interface/linux/drm_memory_manager.cpp

@@ -201,6 +201,13 @@ bool DrmMemoryManager::isKmdMigrationAvailable(uint32_t rootDeviceIndex) {
     return useKmdMigration;
 }
 
+void DrmMemoryManager::setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags) {
+    DrmAllocation *drmAllocation = static_cast<DrmAllocation *>(gfxAllocation);
+
+    CachePolicy memType = flags.cached_memory ? CachePolicy::WriteBack : CachePolicy::Uncached;
+    drmAllocation->setCachePolicy(memType);
+}
+
 NEO::BufferObject *DrmMemoryManager::allocUserptr(uintptr_t address, size_t size, uint64_t flags, uint32_t rootDeviceIndex) {
     drm_i915_gem_userptr userptr = {};
     userptr.user_ptr = address;

shared/source/os_interface/linux/drm_memory_manager.h

@@ -65,6 +65,8 @@ class DrmMemoryManager : public MemoryManager {
 
     bool isKmdMigrationAvailable(uint32_t rootDeviceIndex) override;
 
+    void setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags) override;
+
     std::unique_lock<std::mutex> acquireAllocLock();
     std::vector<GraphicsAllocation *> &getSysMemAllocs();
     std::vector<GraphicsAllocation *> &getLocalMemAllocs(uint32_t rootDeviceIndex);

shared/test/common/mocks/mock_memory_manager.h

@@ -141,6 +141,11 @@ class MockMemoryManager : public MemoryManagerCreate<OsAgnosticMemoryManager> {
     }
     void forceLimitedRangeAllocator(uint32_t rootDeviceIndex, uint64_t range) { getGfxPartition(rootDeviceIndex)->init(range, 0, 0, gfxPartitions.size()); }
 
+    void setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags) override {
+        memAdviseFlags = flags;
+        MemoryManager::setMemAdvise(gfxAllocation, flags);
+    }
+
     uint32_t freeGraphicsMemoryCalled = 0u;
     uint32_t unlockResourceCalled = 0u;
     uint32_t lockResourceCalled = 0u;
@@ -178,6 +183,7 @@ class MockMemoryManager : public MemoryManagerCreate<OsAgnosticMemoryManager> {
     std::unique_ptr<MockExecutionEnvironment> mockExecutionEnvironment;
     DeviceBitfield recentlyPassedDeviceBitfield{};
     std::unique_ptr<MultiGraphicsAllocation> waitAllocations = nullptr;
+    MemAdviseFlags memAdviseFlags{};
 };
 
 class GMockMemoryManager : public MockMemoryManager {