Add manager of imported host pointers

Related-To: NEO-5126

Signed-off-by: Zbigniew Zdanowicz <zbigniew.zdanowicz@intel.com>

level_zero/core/source/driver/driver_handle.h

@@ -70,6 +70,13 @@ struct DriverHandle : _ze_driver_handle_t {
     virtual NEO::SVMAllocsManager *getSvmAllocsManager() = 0;
     virtual ze_result_t sysmanEventsListen(uint32_t timeout, uint32_t count, zes_device_handle_t *phDevices,
                                            uint32_t *pNumDeviceEvents, zes_event_type_flags_t *pEvents) = 0;
+    virtual ze_result_t importExternalPointer(void *ptr, size_t size) = 0;
+    virtual ze_result_t releaseImportedPointer(void *ptr) = 0;
+    virtual ze_result_t getHostPointerBaseAddress(void *ptr, void **baseAddress) = 0;
+
+    virtual NEO::GraphicsAllocation *findHostPointerAllocation(void *ptr, size_t size, uint32_t rootDeviceIndex) = 0;
+    virtual NEO::GraphicsAllocation *getDriverSystemMemoryAllocation(void *ptr, size_t size, uint32_t rootDeviceIndex) = 0;
+
     static DriverHandle *fromHandle(ze_driver_handle_t handle) { return static_cast<DriverHandle *>(handle); }
     inline ze_driver_handle_t toHandle() { return this; }
 

level_zero/core/source/driver/driver_handle_imp.cpp

@@ -17,6 +17,7 @@
 #include "level_zero/core/source/debugger/debugger_l0.h"
 #include "level_zero/core/source/device/device_imp.h"
 #include "level_zero/core/source/driver/driver_imp.h"
+#include "level_zero/core/source/driver/host_pointer_manager.h"
 
 #include "driver_version_l0.h"
 
@@ -275,6 +276,10 @@ ze_result_t DriverHandleImp::initialize(std::vector<std::unique_ptr<NEO::Device>
 
     uuidTimestamp = static_cast<uint64_t>(std::chrono::system_clock::now().time_since_epoch().count());
 
+    if (NEO::DebugManager.flags.EnableHostPointerImport.get() == 1) {
+        createHostPointerManager();
+    }
+
     return ZE_RESULT_SUCCESS;
 }
 
@@ -294,7 +299,7 @@ DriverHandle *DriverHandle::create(std::vector<std::unique_ptr<NEO::Device>> dev
 
     GlobalDriver = driverHandle;
 
-    driverHandle->memoryManager->setForceNonSvmForExternalHostPtr(true);
+    driverHandle->getMemoryManager()->setForceNonSvmForExternalHostPtr(true);
 
     return driverHandle;
 }
@@ -392,4 +397,67 @@ ze_result_t DriverHandleImp::openEventPoolIpcHandle(ze_ipc_event_pool_handle_t h
     return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
 }
 
+void DriverHandleImp::createHostPointerManager() {
+    hostPointerManager = std::make_unique<HostPointerManager>(getMemoryManager());
+}
+
+ze_result_t DriverHandleImp::importExternalPointer(void *ptr, size_t size) {
+    if (hostPointerManager.get() != nullptr) {
+        auto ret = hostPointerManager->createHostPointerMultiAllocation(this->devices,
+                                                                        ptr,
+                                                                        size);
+        return ret;
+    }
+
+    return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
+}
+
+ze_result_t DriverHandleImp::releaseImportedPointer(void *ptr) {
+    if (hostPointerManager.get() != nullptr) {
+        bool ret = hostPointerManager->freeHostPointerAllocation(ptr);
+        return ret ? ZE_RESULT_SUCCESS : ZE_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+    return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
+}
+
+ze_result_t DriverHandleImp::getHostPointerBaseAddress(void *ptr, void **baseAddress) {
+    if (hostPointerManager.get() != nullptr) {
+        auto hostPointerData = hostPointerManager->getHostPointerAllocation(ptr);
+        if (hostPointerData != nullptr) {
+            if (baseAddress != nullptr) {
+                *baseAddress = hostPointerData->basePtr;
+            }
+            return ZE_RESULT_SUCCESS;
+        }
+        return ZE_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+    return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
+}
+
+NEO::GraphicsAllocation *DriverHandleImp::findHostPointerAllocation(void *ptr, size_t size, uint32_t rootDeviceIndex) {
+    if (hostPointerManager.get() != nullptr) {
+        HostPointerData *hostData = hostPointerManager->getHostPointerAllocation(ptr);
+        if (hostData != nullptr) {
+            size_t foundEndSize = reinterpret_cast<size_t>(hostData->basePtr) + hostData->size;
+            size_t inputEndSize = reinterpret_cast<size_t>(ptr) + size;
+            if (foundEndSize >= inputEndSize) {
+                return hostData->hostPtrAllocations.getGraphicsAllocation(rootDeviceIndex);
+            }
+            return nullptr;
+        }
+        return nullptr;
+    }
+
+    return nullptr;
+}
+
+NEO::GraphicsAllocation *DriverHandleImp::getDriverSystemMemoryAllocation(void *ptr, size_t size, uint32_t rootDeviceIndex) {
+    NEO::SvmAllocationData *allocData = nullptr;
+    bool allocFound = findAllocationDataForRange(ptr, size, &allocData);
+    if (allocFound) {
+        return allocData->gpuAllocations.getGraphicsAllocation(rootDeviceIndex);
+    }
+    return findHostPointerAllocation(ptr, size, rootDeviceIndex);
+}
+
 } // namespace L0

level_zero/core/source/driver/driver_handle_imp.h

@@ -14,6 +14,7 @@
 #include "level_zero/extensions/public/ze_exp_ext.h"
 
 namespace L0 {
+class HostPointerManager;
 
 struct DriverHandleImp : public DriverHandle {
     ~DriverHandleImp() override;
@@ -68,25 +69,35 @@ struct DriverHandleImp : public DriverHandle {
     ze_result_t sysmanEventsListen(uint32_t timeout, uint32_t count, zes_device_handle_t *phDevices,
                                    uint32_t *pNumDeviceEvents, zes_event_type_flags_t *pEvents) override;
 
+    ze_result_t importExternalPointer(void *ptr, size_t size) override;
+    ze_result_t releaseImportedPointer(void *ptr) override;
+    ze_result_t getHostPointerBaseAddress(void *ptr, void **baseAddress) override;
+
+    virtual NEO::GraphicsAllocation *findHostPointerAllocation(void *ptr, size_t size, uint32_t rootDeviceIndex) override;
+    virtual NEO::GraphicsAllocation *getDriverSystemMemoryAllocation(void *ptr, size_t size, uint32_t rootDeviceIndex) override;
     uint32_t parseAffinityMask(std::vector<std::unique_ptr<NEO::Device>> &neoDevices);
+    void createHostPointerManager();
 
-    uint32_t numDevices = 0;
-    std::vector<Device *> devices;
-    NEO::MemoryManager *memoryManager = nullptr;
-    NEO::SVMAllocsManager *svmAllocsManager = nullptr;
-    uint64_t uuidTimestamp = 0u;
+    std::unique_ptr<HostPointerManager> hostPointerManager;
+    // Experimental functions
+    std::unordered_map<std::string, void *> extensionFunctionsLookupMap;
 
-    // Environment Variables
     std::string affinityMaskString = "";
-    bool enableProgramDebugging = false;
-    bool enableSysman = false;
-
+    std::vector<Device *> devices;
     // Spec extensions
     const std::vector<std::pair<std::string, uint32_t>> extensionsSupported = {
         {ZE_MODULE_PROGRAM_EXP_NAME, ZE_MODULE_PROGRAM_EXP_VERSION_CURRENT}};
 
-    // Experimental functions
-    std::unordered_map<std::string, void *> extensionFunctionsLookupMap;
+    uint64_t uuidTimestamp = 0u;
+
+    NEO::MemoryManager *memoryManager = nullptr;
+    NEO::SVMAllocsManager *svmAllocsManager = nullptr;
+
+    uint32_t numDevices = 0;
+
+    // Environment Variables
+    bool enableProgramDebugging = false;
+    bool enableSysman = false;
 };
 
 extern struct DriverHandleImp *GlobalDriver;

level_zero/core/source/driver/host_pointer_manager.cpp

@@ -0,0 +1,143 @@
+/*
+ * Copyright (C) 2020 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/core/source/driver/host_pointer_manager.h"
+
+#include "shared/source/helpers/aligned_memory.h"
+#include "shared/source/helpers/constants.h"
+#include "shared/source/memory_manager/allocation_properties.h"
+#include "shared/source/memory_manager/graphics_allocation.h"
+#include "shared/source/memory_manager/memory_manager.h"
+#include "shared/source/memory_manager/multi_graphics_allocation.h"
+
+#include "level_zero/core/source/device/device_imp.h"
+
+namespace L0 {
+
+void HostPointerManager::MapBasedAllocationTracker::insert(HostPointerData allocationsData) {
+    allocations.insert(std::make_pair(reinterpret_cast<void *>(allocationsData.basePtr), allocationsData));
+}
+
+void HostPointerManager::MapBasedAllocationTracker::remove(const void *ptr) {
+    HostPointerContainer::iterator iter;
+    iter = allocations.find(ptr);
+    allocations.erase(iter);
+}
+
+HostPointerData *HostPointerManager::MapBasedAllocationTracker::get(const void *ptr) {
+    HostPointerContainer::iterator iter, end;
+    HostPointerData *hostPtrData;
+    if ((ptr == nullptr) || (allocations.size() == 0)) {
+        return nullptr;
+    }
+    end = allocations.end();
+    iter = allocations.lower_bound(ptr);
+    if (((iter != end) && (iter->first != ptr)) ||
+        (iter == end)) {
+        if (iter == allocations.begin()) {
+            iter = end;
+        } else {
+            iter--;
+        }
+    }
+    if (iter != end) {
+        hostPtrData = &iter->second;
+        char *charPtr = reinterpret_cast<char *>(hostPtrData->basePtr);
+        if (ptr < (charPtr + hostPtrData->size)) {
+            return hostPtrData;
+        }
+    }
+    return nullptr;
+}
+
+HostPointerManager::HostPointerManager(NEO::MemoryManager *memoryManager) : memoryManager(memoryManager) {
+}
+
+HostPointerManager::~HostPointerManager() {
+}
+
+ze_result_t HostPointerManager::createHostPointerMultiAllocation(std::vector<Device *> &devices, void *ptr, size_t size) {
+    if (size == 0) {
+        return ZE_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    void *basePtr = alignDown(ptr, MemoryConstants::pageSize);
+    size_t endAddress = reinterpret_cast<size_t>(ptr) + size;
+    endAddress = alignUp(endAddress, MemoryConstants::pageSize);
+    size_t totalSize = endAddress - reinterpret_cast<size_t>(basePtr);
+
+    std::unique_lock<NEO::SpinLock> lock(this->mtx);
+    auto baseAllocation = hostPointerAllocations.get(basePtr);
+    auto endingAllocation = hostPointerAllocations.get(reinterpret_cast<void *>(endAddress - 1));
+    if (baseAllocation != nullptr && baseAllocation == endingAllocation) {
+        return ZE_RESULT_SUCCESS;
+    }
+    if (baseAllocation != nullptr) {
+        if (endingAllocation != nullptr) {
+            return ZE_RESULT_ERROR_OVERLAPPING_REGIONS;
+        }
+        return ZE_RESULT_ERROR_HANDLE_OBJECT_IN_USE;
+    }
+    if (endingAllocation != nullptr) {
+        UNRECOVERABLE_IF(endingAllocation->basePtr == basePtr);
+        return ZE_RESULT_ERROR_INVALID_SIZE;
+    }
+
+    HostPointerData hostData(static_cast<uint32_t>(devices.size() - 1));
+    hostData.basePtr = basePtr;
+    hostData.size = totalSize;
+    for (auto device : devices) {
+        NEO::GraphicsAllocation *gfxAlloc = createHostPointerAllocation(device->getRootDeviceIndex(),
+                                                                        basePtr,
+                                                                        totalSize,
+                                                                        device->getNEODevice()->getDeviceBitfield());
+        if (gfxAlloc == nullptr) {
+            auto allocations = hostData.hostPtrAllocations.getGraphicsAllocations();
+            for (auto &allocation : allocations) {
+                memoryManager->freeGraphicsMemory(allocation);
+            }
+            return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY;
+        }
+        hostData.hostPtrAllocations.addAllocation(gfxAlloc);
+    }
+    hostPointerAllocations.insert(hostData);
+    return ZE_RESULT_SUCCESS;
+}
+
+NEO::GraphicsAllocation *HostPointerManager::createHostPointerAllocation(uint32_t rootDeviceIndex,
+                                                                         void *ptr,
+                                                                         size_t size,
+                                                                         const NEO::DeviceBitfield &deviceBitfield) {
+    NEO::AllocationProperties properties = {rootDeviceIndex, false, size,
+                                            NEO::GraphicsAllocation::AllocationType::EXTERNAL_HOST_PTR,
+                                            false, deviceBitfield};
+    properties.flags.flushL3RequiredForRead = properties.flags.flushL3RequiredForWrite = true;
+    auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(properties,
+                                                                          ptr);
+    return allocation;
+}
+
+HostPointerData *HostPointerManager::getHostPointerAllocation(const void *ptr) {
+    std::unique_lock<NEO::SpinLock> lock(mtx);
+    return hostPointerAllocations.get(ptr);
+}
+
+bool HostPointerManager::freeHostPointerAllocation(void *ptr) {
+    std::unique_lock<NEO::SpinLock> lock(mtx);
+    HostPointerData *hostPtrData = hostPointerAllocations.get(ptr);
+    if (hostPtrData == nullptr) {
+        return false;
+    }
+    auto graphicsAllocations = hostPtrData->hostPtrAllocations.getGraphicsAllocations();
+    for (auto gpuAllocation : graphicsAllocations) {
+        memoryManager->freeGraphicsMemory(gpuAllocation);
+    }
+    hostPointerAllocations.remove(hostPtrData->basePtr);
+    return true;
+}
+
+} // namespace L0

level_zero/core/source/driver/host_pointer_manager.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2020 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+#include "shared/source/helpers/common_types.h"
+#include "shared/source/memory_manager/multi_graphics_allocation.h"
+#include "shared/source/utilities/spinlock.h"
+
+#include <level_zero/ze_api.h>
+
+#include <cstdint>
+#include <map>
+#include <mutex>
+#include <vector>
+
+namespace NEO {
+class GraphicsAllocation;
+class MemoryManager;
+} // namespace NEO
+
+namespace L0 {
+struct Device;
+
+struct HostPointerData {
+    HostPointerData(uint32_t maxRootDeviceIndex)
+        : hostPtrAllocations(maxRootDeviceIndex),
+          maxRootDeviceIndex(maxRootDeviceIndex) {
+    }
+    HostPointerData(const HostPointerData &hostPtrData)
+        : HostPointerData(hostPtrData.maxRootDeviceIndex) {
+        basePtr = hostPtrData.basePtr;
+        size = hostPtrData.size;
+        for (auto allocation : hostPtrData.hostPtrAllocations.getGraphicsAllocations()) {
+            if (allocation) {
+                this->hostPtrAllocations.addAllocation(allocation);
+            }
+        }
+    }
+    NEO::MultiGraphicsAllocation hostPtrAllocations;
+    void *basePtr = nullptr;
+    size_t size = 0u;
+
+  protected:
+    const uint32_t maxRootDeviceIndex;
+};
+
+class HostPointerManager {
+  public:
+    class MapBasedAllocationTracker {
+        friend class HostPointerManager;
+
+      public:
+        using HostPointerContainer = std::map<const void *, HostPointerData>;
+        void insert(HostPointerData allocationsData);
+        void remove(const void *ptr);
+        HostPointerData *get(const void *ptr);
+        size_t getNumAllocs() const { return allocations.size(); };
+
+      protected:
+        HostPointerContainer allocations;
+    };
+
+    HostPointerManager(NEO::MemoryManager *memoryManager);
+    virtual ~HostPointerManager();
+    ze_result_t createHostPointerMultiAllocation(std::vector<Device *> &devices, void *ptr, size_t size);
+    HostPointerData *getHostPointerAllocation(const void *ptr);
+    bool freeHostPointerAllocation(void *ptr);
+
+  protected:
+    NEO::GraphicsAllocation *createHostPointerAllocation(uint32_t rootDeviceIndex,
+                                                         void *ptr,
+                                                         size_t size,
+                                                         const NEO::DeviceBitfield &deviceBitfield);
+
+    MapBasedAllocationTracker hostPointerAllocations;
+    NEO::MemoryManager *memoryManager;
+    NEO::SpinLock mtx;
+};
+} // namespace L0