Add support for IPC handles with implicit scaling

When using implicit scaling, device allocations may have
more than one internal allocation created internally. In that case,
a separate dma-buf handle per internal allocation needs to be
exported.

So introduced two driver experimental extensions to export and
import more than one IPC handle:

- zexMemGetIpcHandles
- zexMemOpenIpcHandles

Related-To: LOCI-2919

Signed-off-by: Jaime Arteaga <jaime.a.arteaga.molina@intel.com>

shared/source/memory_manager/graphics_allocation.h

@@ -176,6 +176,17 @@ class GraphicsAllocation : public IDNode<GraphicsAllocation> {
     virtual std::string getAllocationInfoString() const;
     virtual uint64_t peekInternalHandle(MemoryManager *memoryManager) { return 0llu; }
 
+    virtual uint64_t peekInternalHandle(MemoryManager *memoryManager, uint32_t handleId) {
+        return 0u;
+    }
+
+    virtual uint32_t getNumHandles() {
+        return 0u;
+    }
+
+    virtual void setNumHandles(uint32_t numHandles) {
+    }
+
     static bool isCpuAccessRequired(AllocationType allocationType) {
         return allocationType == AllocationType::COMMAND_BUFFER ||
                allocationType == AllocationType::CONSTANT_SURFACE ||

shared/source/memory_manager/memory_manager.h

@@ -92,6 +92,7 @@ class MemoryManager {
 
     virtual bool verifyHandle(osHandle handle, uint32_t rootDeviceIndex, bool) { return true; }
     virtual bool isNTHandle(osHandle handle, uint32_t rootDeviceIndex) { return false; }
+    virtual GraphicsAllocation *createGraphicsAllocationFromMultipleSharedHandles(std::vector<osHandle> handles, AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) = 0;
     virtual GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) = 0;
     virtual void closeSharedHandle(GraphicsAllocation *graphicsAllocation){};
     virtual GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex, AllocationType allocType) = 0;

shared/source/memory_manager/os_agnostic_memory_manager.cpp

@@ -223,6 +223,10 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocate32BitGraphicsMemoryImpl(con
     return memoryAllocation;
 }
 
+GraphicsAllocation *OsAgnosticMemoryManager::createGraphicsAllocationFromMultipleSharedHandles(std::vector<osHandle> handles, AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) {
+    return nullptr;
+}
+
 GraphicsAllocation *OsAgnosticMemoryManager::createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) {
     auto graphicsAllocation = createMemoryAllocation(properties.allocationType, nullptr, reinterpret_cast<void *>(1), 1,
                                                      4096u, static_cast<uint64_t>(handle), MemoryPool::SystemCpuInaccessible, properties.rootDeviceIndex,

shared/source/memory_manager/os_agnostic_memory_manager.h

@@ -69,6 +69,7 @@ class OsAgnosticMemoryManager : public MemoryManager {
     OsAgnosticMemoryManager(bool aubUsage, ExecutionEnvironment &executionEnvironment);
     void initialize(bool aubUsage);
     ~OsAgnosticMemoryManager() override;
+    GraphicsAllocation *createGraphicsAllocationFromMultipleSharedHandles(std::vector<osHandle> handles, AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override;
     GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override;
     GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex, AllocationType allocType) override { return nullptr; }
 

shared/source/os_interface/linux/drm_allocation.cpp

@@ -40,6 +40,10 @@ uint64_t DrmAllocation::peekInternalHandle(MemoryManager *memoryManager) {
     return static_cast<uint64_t>((static_cast<DrmMemoryManager *>(memoryManager))->obtainFdFromHandle(getBO()->peekHandle(), this->rootDeviceIndex));
 }
 
+uint64_t DrmAllocation::peekInternalHandle(MemoryManager *memoryManager, uint32_t handleId) {
+    return static_cast<uint64_t>((static_cast<DrmMemoryManager *>(memoryManager))->obtainFdFromHandle(getBufferObjectToModify(handleId)->peekHandle(), this->rootDeviceIndex));
+}
+
 void DrmAllocation::setCachePolicy(CachePolicy memType) {
     for (auto bo : bufferObjects) {
         if (bo != nullptr) {

shared/source/os_interface/linux/drm_allocation.h

@@ -78,8 +78,18 @@ class DrmAllocation : public GraphicsAllocation {
         this->bufferObjects.resize(size);
     }
 
+    uint32_t getNumHandles() override {
+        return this->numHandles;
+    }
+
+    void setNumHandles(uint32_t numHandles) override {
+        this->numHandles = numHandles;
+    }
+
     uint64_t peekInternalHandle(MemoryManager *memoryManager) override;
 
+    uint64_t peekInternalHandle(MemoryManager *memoryManager, uint32_t handleId) override;
+
     bool setCacheRegion(Drm *drm, CacheRegion regionIndex);
     bool setCacheAdvice(Drm *drm, size_t regionSize, CacheRegion regionIndex);
     void setCachePolicy(CachePolicy memType);
@@ -107,6 +117,7 @@ class DrmAllocation : public GraphicsAllocation {
     StackVec<uint32_t, 1> registeredBoBindHandles;
     MemAdviseFlags enabledMemAdviseFlags{};
     StackVec<MemoryToUnmap, 1> memoryToUnmap;
+    uint32_t numHandles = 0u;
 
     void *mmapPtr = nullptr;
     size_t mmapSize = 0u;

shared/source/os_interface/linux/drm_memory_manager.cpp

@@ -20,6 +20,7 @@
 #include "shared/source/helpers/ptr_math.h"
 #include "shared/source/helpers/string.h"
 #include "shared/source/helpers/surface_format_info.h"
+#include "shared/source/memory_manager/allocation_properties.h"
 #include "shared/source/memory_manager/host_ptr_manager.h"
 #include "shared/source/memory_manager/memory_banks.h"
 #include "shared/source/memory_manager/memory_pool.h"
@@ -654,6 +655,101 @@ BufferObject *DrmMemoryManager::findAndReferenceSharedBufferObject(int boHandle,
     return bo;
 }
 
+GraphicsAllocation *DrmMemoryManager::createGraphicsAllocationFromMultipleSharedHandles(std::vector<osHandle> handles, AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) {
+    BufferObjects bos;
+    std::vector<size_t> sizes;
+    size_t totalSize = 0;
+
+    std::unique_lock<std::mutex> lock(mtx);
+
+    uint32_t i = 0;
+
+    if (handles.size() != 1) {
+        properties.multiStorageResource = true;
+    }
+
+    auto &drm = this->getDrm(properties.rootDeviceIndex);
+
+    bool areBosSharedObjects = true;
+
+    for (auto handle : handles) {
+        drm_prime_handle openFd = {0, 0, 0};
+        openFd.fd = handle;
+
+        auto ret = this->getDrm(properties.rootDeviceIndex).ioctl(DRM_IOCTL_PRIME_FD_TO_HANDLE, &openFd);
+
+        if (ret != 0) {
+            [[maybe_unused]] int err = errno;
+            PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(PRIME_FD_TO_HANDLE) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
+
+            return nullptr;
+        }
+
+        auto boHandle = openFd.handle;
+        auto bo = findAndReferenceSharedBufferObject(boHandle, properties.rootDeviceIndex);
+
+        if (bo == nullptr) {
+            areBosSharedObjects = false;
+
+            size_t size = lseekFunction(handle, 0, SEEK_END);
+            totalSize += size;
+
+            auto patIndex = drm.getPatIndex(nullptr, properties.allocationType, CacheRegion::Default, CachePolicy::WriteBack, false);
+
+            bo = new (std::nothrow) BufferObject(&drm, patIndex, boHandle, size, maxOsContextCount);
+            bo->setRootDeviceIndex(properties.rootDeviceIndex);
+            i++;
+        }
+        bos.push_back(bo);
+        sizes.push_back(bo->peekSize());
+    }
+
+    auto heapIndex = HeapIndex::HEAP_STANDARD2MB;
+    auto gpuRange = acquireGpuRange(totalSize, properties.rootDeviceIndex, heapIndex);
+
+    lock.unlock();
+
+    AllocationData allocationData;
+    properties.size = totalSize;
+    getAllocationData(allocationData, properties, nullptr, createStorageInfoFromProperties(properties));
+
+    auto drmAllocation = new DrmAllocation(properties.rootDeviceIndex,
+                                           handles.size(),
+                                           properties.allocationType,
+                                           bos,
+                                           nullptr,
+                                           gpuRange,
+                                           totalSize,
+                                           MemoryPool::LocalMemory);
+    drmAllocation->storageInfo = allocationData.storageInfo;
+
+    auto gmmHelper = executionEnvironment.rootDeviceEnvironments[properties.rootDeviceIndex]->getGmmHelper();
+    for (i = 0u; i < handles.size(); i++) {
+        auto bo = bos[i];
+        StorageInfo limitedStorageInfo = allocationData.storageInfo;
+        limitedStorageInfo.memoryBanks &= (1u << (i % handles.size()));
+        auto gmm = new Gmm(gmmHelper,
+                           nullptr,
+                           bo->peekSize(),
+                           0u,
+                           CacheSettingsHelper::getGmmUsageType(drmAllocation->getAllocationType(), false, *gmmHelper->getHardwareInfo()),
+                           false,
+                           allocationData.storageInfo,
+                           true);
+        drmAllocation->setGmm(gmm, i);
+
+        if (areBosSharedObjects == false) {
+            bo->setAddress(gpuRange);
+            gpuRange += bo->peekSize();
+            bo->setUnmapSize(sizes[i]);
+            pushSharedBufferObject(bo);
+        }
+        drmAllocation->getBufferObjectToModify(i) = bo;
+    }
+
+    return drmAllocation;
+}
+
 GraphicsAllocation *DrmMemoryManager::createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) {
     if (isHostIpcAllocation) {
         return createUSMHostAllocationFromSharedHandle(handle, properties, false);
@@ -1468,6 +1564,7 @@ bool DrmMemoryManager::createDrmAllocation(Drm *drm, DrmAllocation *allocation,
         allocation->resizeBufferObjects(handles);
         bos.resize(handles);
     }
+    allocation->setNumHandles(handles);
 
     for (auto handleId = 0u; handleId < handles; handleId++, currentBank++) {
         if (currentBank == banksCnt) {

shared/source/os_interface/linux/drm_memory_manager.h

@@ -36,6 +36,7 @@ class DrmMemoryManager : public MemoryManager {
     void freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation, bool isImportedAllocation) override;
     void handleFenceCompletion(GraphicsAllocation *allocation) override;
     GraphicsAllocation *createGraphicsAllocationFromExistingStorage(AllocationProperties &properties, void *ptr, MultiGraphicsAllocation &multiGraphicsAllocation) override;
+    GraphicsAllocation *createGraphicsAllocationFromMultipleSharedHandles(std::vector<osHandle> handles, AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override;
     GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override;
     void closeSharedHandle(GraphicsAllocation *gfxAllocation) override;
     GraphicsAllocation *createPaddedAllocation(GraphicsAllocation *inputGraphicsAllocation, size_t sizeWithPadding) override;
@@ -88,7 +89,7 @@ class DrmMemoryManager : public MemoryManager {
     }
 
   protected:
-    BufferObject *findAndReferenceSharedBufferObject(int boHandle, uint32_t rootDeviceIndex);
+    MOCKABLE_VIRTUAL BufferObject *findAndReferenceSharedBufferObject(int boHandle, uint32_t rootDeviceIndex);
     void eraseSharedBufferObject(BufferObject *bo);
     void pushSharedBufferObject(BufferObject *bo);
     BufferObject *allocUserptr(uintptr_t address, size_t size, uint64_t flags, uint32_t rootDeviceIndex);

shared/source/os_interface/windows/wddm_memory_manager.cpp

@@ -451,6 +451,10 @@ GraphicsAllocation *WddmMemoryManager::createAllocationFromHandle(osHandle handl
     return allocation.release();
 }
 
+GraphicsAllocation *WddmMemoryManager::createGraphicsAllocationFromMultipleSharedHandles(std::vector<osHandle> handles, AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) {
+    return nullptr;
+}
+
 GraphicsAllocation *WddmMemoryManager::createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) {
     return createAllocationFromHandle(handle, requireSpecificBitness, false, properties.allocationType, properties.rootDeviceIndex);
 }

shared/source/os_interface/windows/wddm_memory_manager.h

@@ -44,6 +44,7 @@ class WddmMemoryManager : public MemoryManager {
     void freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation, bool isImportedAllocation) override;
     void handleFenceCompletion(GraphicsAllocation *allocation) override;
 
+    GraphicsAllocation *createGraphicsAllocationFromMultipleSharedHandles(std::vector<osHandle> handles, AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override;
     GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override;
     GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex, AllocationType allocType) override;
 

shared/test/common/fixtures/memory_allocator_multi_device_fixture.h

@@ -62,3 +62,41 @@ class MemoryAllocatorMultiDeviceFixture : public MemoryManagementFixture, public
     DebugManagerStateRestore restorer;
     bool isOsAgnosticMemoryManager;
 };
+
+template <uint32_t numRootDevices, uint32_t numSubDevices>
+class MemoryAllocatorMultiDeviceAndMultiTileFixture : public MemoryManagementFixture, public MemoryAllocatorMultiDeviceSystemSpecificFixture, public ::testing::TestWithParam<bool> {
+  public:
+    void SetUp() override {
+        MemoryManagementFixture::SetUp();
+
+        isOsAgnosticMemoryManager = GetParam();
+        DebugManager.flags.CreateMultipleRootDevices.set(numRootDevices);
+        DebugManager.flags.CreateMultipleRootDevices.set(numSubDevices);
+        VariableBackup<UltHwConfig> backup(&ultHwConfig);
+        ultHwConfig.useMockedPrepareDeviceEnvironmentsFunc = false;
+        ultHwConfig.forceOsAgnosticMemoryManager = isOsAgnosticMemoryManager;
+
+        executionEnvironment = new MockExecutionEnvironment(defaultHwInfo.get(), true, numRootDevices);
+        devices = DeviceFactory::createDevices(*executionEnvironment);
+        memoryManager = executionEnvironment->memoryManager.get();
+
+        if (!isOsAgnosticMemoryManager) {
+            MemoryAllocatorMultiDeviceSystemSpecificFixture::SetUp(*executionEnvironment);
+        }
+    }
+
+    void TearDown() override {
+        if (!isOsAgnosticMemoryManager) {
+            MemoryAllocatorMultiDeviceSystemSpecificFixture::TearDown(*executionEnvironment);
+        }
+    }
+
+    uint32_t getNumRootDevices() { return numRootDevices; }
+
+  protected:
+    std::vector<std::unique_ptr<Device>> devices;
+    ExecutionEnvironment *executionEnvironment = nullptr;
+    MemoryManager *memoryManager = nullptr;
+    DebugManagerStateRestore restorer;
+    bool isOsAgnosticMemoryManager;
+};

shared/test/common/mocks/linux/mock_drm_memory_manager.cpp

@@ -53,6 +53,7 @@ TestedDrmMemoryManager::TestedDrmMemoryManager(bool enableLocalMemory,
     lseekCalledCount = 0;
     closeInputFd = 0;
     closeCalledCount = 0;
+    this->executionEnvironment = &executionEnvironment;
 }
 
 void TestedDrmMemoryManager::injectPinBB(BufferObject *newPinBB, uint32_t rootDeviceIndex) {

shared/test/common/mocks/linux/mock_drm_memory_manager.h

@@ -8,6 +8,7 @@
 #pragma once
 #include "shared/source/os_interface/linux/drm_memory_manager.h"
 #include "shared/test/common/mocks/mock_memory_manager.h"
+#include "shared/test/common/os_interface/linux/device_command_stream_fixture.h"
 
 #include <atomic>
 
@@ -112,6 +113,15 @@ class TestedDrmMemoryManager : public MemoryManagerCreate<DrmMemoryManager> {
     void forceLimitedRangeAllocator(uint64_t range);
     void overrideGfxPartition(GfxPartition *newGfxPartition);
 
+    BufferObject *findAndReferenceSharedBufferObject(int boHandle, uint32_t rootDeviceIndex) override {
+        if (failOnfindAndReferenceSharedBufferObject) {
+            DrmMockCustom drmMock(*executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]);
+            auto patIndex = drmMock.getPatIndex(nullptr, AllocationType::BUFFER, CacheRegion::Default, CachePolicy::WriteBack, false);
+            return new (std::nothrow) BufferObject(&drmMock, patIndex, boHandle, 4096u, 2u);
+        }
+        return MemoryManagerCreate<DrmMemoryManager>::findAndReferenceSharedBufferObject(boHandle, rootDeviceIndex);
+    }
+
     DrmAllocation *allocate32BitGraphicsMemory(uint32_t rootDeviceIndex, size_t size, const void *ptr, AllocationType allocationType);
     ~TestedDrmMemoryManager() override;
     size_t peekSharedBosSize() {
@@ -153,6 +163,10 @@ class TestedDrmMemoryManager : public MemoryManagerCreate<DrmMemoryManager> {
     uint32_t alignedFreeWrapperCalled = 0u;
     uint32_t callsToCloseSharedHandle = 0;
 
+    bool failOnfindAndReferenceSharedBufferObject = false;
+
+    ExecutionEnvironment *executionEnvironment = nullptr;
+
   protected:
     std::mutex unreferenceMtx;
     std::mutex releaseGpuRangeMtx;

shared/test/common/mocks/mock_memory_manager.h

@@ -302,6 +302,10 @@ class FailMemoryManager : public MockMemoryManager {
         return nullptr;
     }
 
+    GraphicsAllocation *createGraphicsAllocationFromMultipleSharedHandles(std::vector<osHandle> handles, AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override {
+        return nullptr;
+    }
+
     GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override {
         return nullptr;
     }

shared/test/common/os_interface/linux/device_command_stream_fixture.cpp

@@ -109,6 +109,9 @@ int DrmMockCustom::ioctl(unsigned long request, void *arg) {
         primeToHandleParams->handle = outputHandle;
         inputFd = primeToHandleParams->fd;
         ioctl_cnt.primeFdToHandle++;
+        if (failOnPrimeFdToHandle == true) {
+            return -1;
+        }
     } break;
     case DRM_IOCTL_PRIME_HANDLE_TO_FD: {
         auto *handleToPrimeParams = (drm_prime_handle *)arg;

shared/test/common/os_interface/linux/device_command_stream_fixture.h

@@ -212,6 +212,7 @@ class DrmMockCustom : public Drm {
     __u64 mmapOffsetExpected = 0;
     __u64 mmapOffsetFlags = 0;
     bool failOnMmapOffset = false;
+    bool failOnPrimeFdToHandle = false;
 
     int errnoValue = 0;
 

shared/test/unit_test/device/neo_device_tests.cpp

@@ -282,6 +282,7 @@ TEST_F(DeviceGetCapsTest, givenFlagEnabled64kbPagesWhenCallConstructorMemoryMana
         MockMemoryManager(ExecutionEnvironment &executionEnvironment) : MemoryManager(executionEnvironment) {}
         void addAllocationToHostPtrManager(GraphicsAllocation *memory) override{};
         void removeAllocationFromHostPtrManager(GraphicsAllocation *memory) override{};
+        GraphicsAllocation *createGraphicsAllocationFromMultipleSharedHandles(std::vector<osHandle> handles, AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override { return nullptr; }
         GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override { return nullptr; };
         GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex, AllocationType allocType) override { return nullptr; };
         AllocationStatus populateOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override { return AllocationStatus::Success; };

shared/test/unit_test/memory_manager/graphics_allocation_tests.cpp

@@ -216,6 +216,17 @@ TEST(GraphicsAllocationTest, givenDefaultGraphicsAllocationWhenInternalHandleIsB
     EXPECT_EQ(0llu, graphicsAllocation.peekInternalHandle(nullptr));
 }
 
+TEST(GraphicsAllocationTest, givenDefaultGraphicsAllocationWhenGettingNumHandlesThenZeroIsReturned) {
+    MockGraphicsAllocation graphicsAllocation;
+    EXPECT_EQ(0u, graphicsAllocation.getNumHandles());
+}
+
+TEST(GraphicsAllocationTest, givenDefaultGraphicsAllocationWhenGettingNumHandlesAfterSettingNonZeroNumberThenZeroIsReturned) {
+    MockGraphicsAllocation graphicsAllocation;
+    graphicsAllocation.setNumHandles(64u);
+    EXPECT_EQ(0u, graphicsAllocation.getNumHandles());
+}
+
 TEST(GraphicsAllocationTest, givenGraphicsAllocationWhenQueryingUsedPageSizeThenCorrectSizeForMemoryPoolUsedIsReturned) {
 
     MemoryPool::Type page4kPools[] = {MemoryPool::MemoryNull,