Add mapping based colouring

Signed-off-by: Lukasz Jobczyk <lukasz.jobczyk@intel.com>

opencl/test/unit_test/memory_manager/storage_info_tests.cpp

@@ -275,6 +275,7 @@ TEST_F(MultiDeviceStorageInfoTest, whenCreatingStorageInfoForSVMGPUThenAllMemory
 TEST_F(MultiDeviceStorageInfoTest, givenMultiStorageGranularityWhenCreatingStorageInfoThenProperGranularityIsSet) {
     DebugManagerStateRestore restorer;
     DebugManager.flags.MultiStorageGranularity.set(128);
+    DebugManager.flags.MultiStoragePolicy.set(1);
 
     AllocationProperties properties{mockRootDeviceIndex, false, 10 * MemoryConstants::pageSize64k, GraphicsAllocation::AllocationType::SVM_GPU, true, allTilesMask};
     auto storageInfo = memoryManager->createStorageInfoFromProperties(properties);

opencl/test/unit_test/test_files/igdrcl.config

@@ -191,6 +191,7 @@ OverrideLeastOccupiedBank = -1
 UseAsyncDrmExec = -1
 EnableMultiStorageResources = -1
 MultiStorageGranularity = -1
+MultiStoragePolicy = -1;
 PrintExecutionBuffer = 0
 PrintBOsForSubmit = 0
 EnableCrossDeviceAccess = -1

shared/source/debug_settings/debug_variables_base.inl

@@ -94,7 +94,8 @@ DECLARE_DEBUG_VARIABLE(int32_t, PauseOnEnqueue, -1, "-1: default, -2: always, x:
 DECLARE_DEBUG_VARIABLE(int32_t, PauseOnBlitCopy, -1, "-1: default, -2: always, x: pause on blit enqueue number x and ask for user confirmation before and after execution, counted from 0. Note that single blit enqueue may have multiple copy instructions")
 DECLARE_DEBUG_VARIABLE(int32_t, PauseOnGpuMode, -1, "-1: default (before and after), 0: before only, 1: after only")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableMultiStorageResources, -1, "-1: default, 0: Disable, 1: Enable")
-DECLARE_DEBUG_VARIABLE(int32_t, MultiStorageGranularity, -1, "Forces chunk size based resource colouring with given granularity: -1 - disabled (default, subdevice count based colouring), >0 - enabled chunk size based colouring with granularity in kb (should be multiple of 64kb page size, because of alignment restrictions)")
+DECLARE_DEBUG_VARIABLE(int32_t, MultiStoragePolicy, -1, "-1: default, 0: subdevice count based, 1: chunk size based, 2: mapping based")
+DECLARE_DEBUG_VARIABLE(int32_t, MultiStorageGranularity, -1, "Forces granularity for chunk size and mapping colouring policies: -1: default 64 kB, >0 - enabled chunk size based colouring with granularity in kb (should be multiple of 64kb page size, because of alignment restrictions)")
 DECLARE_DEBUG_VARIABLE(int32_t, LimitBlitterMaxWidth, -1, "-1: default, >=0: Max width")
 DECLARE_DEBUG_VARIABLE(int32_t, LimitBlitterMaxHeight, -1, "-1: default, >=0: Max height")
 DECLARE_DEBUG_VARIABLE(int32_t, PostBlitCommand, -1, "-1: default, 0: MI_ARB_CHECK, 1: MI_FLUSH, 2: Nothing")

shared/source/helpers/common_types.h

@@ -27,8 +27,9 @@ enum class DebugPauseState : uint32_t {
 };
 
 enum class ColouringPolicy : uint32_t {
+    DeviceCountBased,
     ChunkSizeBased,
-    DeviceCountBased
+    MappingBased
 };
 
 class TagTypeBase {

shared/source/memory_manager/definitions/storage_info.cpp

@@ -115,8 +115,11 @@ StorageInfo MemoryManager::createStorageInfoFromProperties(const AllocationPrope
         auto colouringPolicy = properties.colouringPolicy;
         auto granularity = properties.colouringGranularity;
 
+        if (DebugManager.flags.MultiStoragePolicy.get() != -1) {
+            colouringPolicy = static_cast<ColouringPolicy>(DebugManager.flags.MultiStoragePolicy.get());
+        }
+
         if (DebugManager.flags.MultiStorageGranularity.get() != -1) {
-            colouringPolicy = ColouringPolicy::ChunkSizeBased;
             granularity = DebugManager.flags.MultiStorageGranularity.get() * MemoryConstants::kiloByte;
         }
 

shared/source/os_interface/linux/drm_buffer_object.h

@@ -97,6 +97,29 @@ class BufferObject {
     void setCachePolicy(CachePolicy memType) { cachePolicy = memType; }
     CachePolicy peekCachePolicy() const { return cachePolicy; }
 
+    void setColourWithBind() {
+        this->colourWithBind = true;
+    }
+    void setColourChunk(size_t size) {
+        this->colourChunk = size;
+    }
+    void addColouringAddress(uint64_t address) {
+        this->bindAddresses.push_back(address);
+    }
+    void reserveAddressVector(size_t size) {
+        this->bindAddresses.reserve(size);
+    }
+
+    bool getColourWithBind() {
+        return this->colourWithBind;
+    }
+    size_t getColourChunk() {
+        return this->colourChunk;
+    }
+    std::vector<uint64_t> &getColourAddresses() {
+        return this->bindAddresses;
+    }
+
   protected:
     Drm *drm = nullptr;
     bool perContextVmsUsed = false;
@@ -125,6 +148,10 @@ class BufferObject {
     std::vector<std::array<bool, EngineLimits::maxHandleCount>> bindInfo;
     StackVec<uint32_t, 2> bindExtHandles;
 
+    bool colourWithBind = false;
+    size_t colourChunk = 0;
+    std::vector<uint64_t> bindAddresses;
+
   private:
     uint64_t gpuAddress = 0llu;
 };

shared/source/os_interface/linux/drm_memory_manager.cpp

@@ -1386,6 +1386,32 @@ bool DrmMemoryManager::createDrmAllocation(Drm *drm, DrmAllocation *allocation,
             boAddress += boSize;
         }
     }
+
+    if (storageInfo.colouringPolicy == ColouringPolicy::MappingBased) {
+        auto size = alignUp(allocation->getUnderlyingBufferSize(), storageInfo.colouringGranularity);
+        auto chunks = static_cast<uint32_t>(size / storageInfo.colouringGranularity);
+        auto granularity = storageInfo.colouringGranularity;
+
+        for (uint32_t boHandle = 0; boHandle < handles; boHandle++) {
+            bos[boHandle]->setColourWithBind();
+            bos[boHandle]->setColourChunk(granularity);
+            bos[boHandle]->reserveAddressVector(alignUp(chunks, handles) / handles);
+        }
+
+        auto boHandle = 0u;
+        auto colourAddress = gpuAddress;
+        for (auto chunk = 0u; chunk < chunks; chunk++) {
+            if (boHandle == handles) {
+                boHandle = 0u;
+            }
+
+            bos[boHandle]->addColouringAddress(colourAddress);
+            colourAddress += granularity;
+
+            boHandle++;
+        }
+    }
+
     return true;
 }