Initial draft to implement Fabric Vertex Apis using Driver Handle Information

Implement the below APIs
- zeFabricVertexGetExp
- zeFabricVertexGetSubVerticesExp
- zeFabricVertexGetPropertiesExp
- zeFabricVertexGetDeviceExp

Related-To: LOCI-3332

Signed-off-by: Joshua Santosh Ranjan <joshua.santosh.ranjan@intel.com>

level_zero/core/source/CMakeLists.txt

@@ -50,6 +50,8 @@ set(L0_RUNTIME_SOURCES
     ${CMAKE_CURRENT_SOURCE_DIR}/event/event.h
     ${CMAKE_CURRENT_SOURCE_DIR}/fence/fence.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/fence/fence.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/fabric/fabric.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/fabric/fabric.h
     ${CMAKE_CURRENT_SOURCE_DIR}/helpers/allocation_extensions.h
     ${CMAKE_CURRENT_SOURCE_DIR}/helpers/allocation_extensions.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/helpers/api_specific_config_l0.cpp

level_zero/core/source/device/device_imp.cpp

@@ -38,6 +38,7 @@
 #include "level_zero/core/source/context/context_imp.h"
 #include "level_zero/core/source/driver/driver_handle_imp.h"
 #include "level_zero/core/source/event/event.h"
+#include "level_zero/core/source/fabric/fabric.h"
 #include "level_zero/core/source/hw_helpers/l0_hw_helper.h"
 #include "level_zero/core/source/image/image.h"
 #include "level_zero/core/source/module/module.h"
@@ -998,6 +999,7 @@ Device *Device::create(DriverHandle *driverHandle, NEO::Device *neoDevice, bool
 
     UNRECOVERABLE_IF(device == nullptr);
 
+    device->isSubdevice = isSubDevice;
     device->setDriverHandle(driverHandle);
     neoDevice->setSpecializedDevice(device);
 
@@ -1066,7 +1068,6 @@ Device *Device::create(DriverHandle *driverHandle, NEO::Device *neoDevice, bool
         if (subDevice == nullptr) {
             return nullptr;
         }
-        static_cast<DeviceImp *>(subDevice)->isSubdevice = true;
         static_cast<DeviceImp *>(subDevice)->setDebugSurface(debugSurface);
         device->subDevices.push_back(static_cast<Device *>(subDevice));
     }
@@ -1113,6 +1114,8 @@ Device *Device::create(DriverHandle *driverHandle, NEO::Device *neoDevice, bool
 
     device->populateSubDeviceCopyEngineGroups();
 
+    device->fabricVertex = std::unique_ptr<FabricVertex>(FabricVertex::createFromDevice(device));
+
     return device;
 }
 

level_zero/core/source/device/device_imp.h

@@ -24,6 +24,7 @@ class DriverInfo;
 
 namespace L0 {
 struct SysmanDevice;
+struct FabricVertex;
 class CacheReservation;
 
 struct DeviceImp : public Device {
@@ -132,6 +133,7 @@ struct DeviceImp : public Device {
 
     using CmdListCreateFunPtrT = L0::CommandList *(*)(uint32_t, Device *, NEO::EngineGroupType, ze_command_list_flags_t, ze_result_t &);
     CmdListCreateFunPtrT getCmdListCreateFunc(const ze_command_list_desc_t *desc);
+    std::unique_ptr<FabricVertex> fabricVertex;
 
   protected:
     void adjustCommandQueueDesc(uint32_t &ordinal, uint32_t &index);

level_zero/core/source/driver/driver_handle.h

@@ -65,6 +65,7 @@ struct DriverHandle : _ze_driver_handle_t {
                                                                      size_t size,
                                                                      uint32_t rootDeviceIndex,
                                                                      uintptr_t *gpuAddress) = 0;
+    virtual ze_result_t fabricVertexGetExp(uint32_t *pCount, ze_fabric_vertex_handle_t *phDevices) = 0;
 
     static DriverHandle *fromHandle(ze_driver_handle_t handle) { return static_cast<DriverHandle *>(handle); }
     inline ze_driver_handle_t toHandle() { return this; }

level_zero/core/source/driver/driver_handle_imp.cpp

@@ -19,6 +19,7 @@
 #include "level_zero/core/source/device/device_imp.h"
 #include "level_zero/core/source/driver/driver_imp.h"
 #include "level_zero/core/source/driver/host_pointer_manager.h"
+#include "level_zero/core/source/fabric/fabric.h"
 
 #include "driver_version_l0.h"
 
@@ -616,4 +617,28 @@ ze_result_t DriverHandleImp::checkMemoryAccessFromDevice(Device *device, const v
     return ZE_RESULT_ERROR_INVALID_ARGUMENT;
 }
 
+ze_result_t DriverHandleImp::fabricVertexGetExp(uint32_t *pCount, ze_fabric_vertex_handle_t *phVertices) {
+
+    uint32_t deviceCount = 0;
+    getDevice(&deviceCount, nullptr);
+
+    if (*pCount == 0) {
+        *pCount = deviceCount;
+        return ZE_RESULT_SUCCESS;
+    }
+
+    std::vector<ze_device_handle_t> deviceHandles;
+    deviceHandles.resize(deviceCount);
+    getDevice(&deviceCount, deviceHandles.data());
+
+    *pCount = std::min(deviceCount, *pCount);
+
+    for (uint32_t index = 0; index < *pCount; index++) {
+        auto deviceImp = static_cast<DeviceImp *>(deviceHandles[index]);
+        phVertices[index] = deviceImp->fabricVertex->toHandle();
+    }
+
+    return ZE_RESULT_SUCCESS;
+}
+
 } // namespace L0

level_zero/core/source/driver/driver_handle_imp.h

@@ -69,6 +69,7 @@ struct DriverHandleImp : public DriverHandle {
                                                NEO::SvmAllocationData *allocData,
                                                void *basePtr,
                                                uintptr_t *peerGpuAddress);
+    ze_result_t fabricVertexGetExp(uint32_t *pCount, ze_fabric_vertex_handle_t *phDevices) override;
     void createHostPointerManager();
     void sortNeoDevices(std::vector<std::unique_ptr<NEO::Device>> &neoDevices);
 

level_zero/core/source/fabric/fabric.cpp

@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/core/source/fabric/fabric.h"
+
+#include "shared/source/helpers/string.h"
+
+#include "level_zero/core/source/device/device_imp.h"
+
+namespace L0 {
+
+FabricVertex *FabricVertex::createFromDevice(Device *device) {
+
+    auto fabricVertex = new FabricVertex();
+    UNRECOVERABLE_IF(fabricVertex == nullptr);
+    ze_device_properties_t deviceProperties = {};
+    ze_pci_ext_properties_t pciProperties = {};
+
+    deviceProperties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
+    pciProperties.stype = ZE_STRUCTURE_TYPE_PCI_EXT_PROPERTIES;
+
+    device->getProperties(&deviceProperties);
+
+    fabricVertex->properties.stype = ZE_STRUCTURE_TYPE_FABRIC_VERTEX_EXP_PROPERTIES;
+    memcpy_s(fabricVertex->properties.uuid.id, ZE_MAX_UUID_SIZE, deviceProperties.uuid.id, ZE_MAX_DEVICE_UUID_SIZE);
+
+    if (deviceProperties.flags & ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE) {
+        fabricVertex->properties.type = ZE_FABRIC_VERTEX_EXP_TYPE_SUBEVICE;
+    } else {
+        fabricVertex->properties.type = ZE_FABRIC_VERTEX_EXP_TYPE_DEVICE;
+    }
+    fabricVertex->properties.remote = false;
+    fabricVertex->device = device;
+    if (device->getPciProperties(&pciProperties) == ZE_RESULT_SUCCESS) {
+        fabricVertex->properties.address.domain = pciProperties.address.domain;
+        fabricVertex->properties.address.bus = pciProperties.address.bus;
+        fabricVertex->properties.address.device = pciProperties.address.device;
+        fabricVertex->properties.address.function = pciProperties.address.function;
+    }
+
+    return fabricVertex;
+}
+
+ze_result_t FabricVertex::getSubVertices(uint32_t *pCount, ze_fabric_vertex_handle_t *phSubvertices) {
+
+    auto deviceImp = static_cast<DeviceImp *>(device);
+    if (*pCount == 0) {
+        *pCount = deviceImp->numSubDevices;
+        return ZE_RESULT_SUCCESS;
+    }
+
+    *pCount = std::min(deviceImp->numSubDevices, *pCount);
+    for (uint32_t index = 0; index < *pCount; index++) {
+        auto subDeviceImp = static_cast<DeviceImp *>(deviceImp->subDevices[index]);
+        phSubvertices[index] = subDeviceImp->fabricVertex->toHandle();
+    }
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t FabricVertex::getProperties(ze_fabric_vertex_exp_properties_t *pVertexProperties) {
+    *pVertexProperties = properties;
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t FabricVertex::getDevice(ze_device_handle_t *phDevice) {
+
+    *phDevice = device->toHandle();
+    return ZE_RESULT_SUCCESS;
+}
+
+} // namespace L0

level_zero/core/source/fabric/fabric.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include <level_zero/ze_api.h>
+
+struct _ze_fabric_vertex_handle_t {};
+
+namespace L0 {
+struct Device;
+
+struct FabricVertex : _ze_fabric_vertex_handle_t {
+
+  public:
+    static FabricVertex *createFromDevice(Device *device);
+    virtual ~FabricVertex() = default;
+    ze_result_t getSubVertices(uint32_t *pCount, ze_fabric_vertex_handle_t *phSubvertices);
+    ze_result_t getProperties(ze_fabric_vertex_exp_properties_t *pVertexProperties);
+    ze_result_t getDevice(ze_device_handle_t *phDevice);
+    static FabricVertex *fromHandle(ze_fabric_vertex_handle_t handle) { return static_cast<FabricVertex *>(handle); }
+    inline ze_fabric_vertex_handle_t toHandle() { return this; }
+
+  private:
+    struct Device *device = nullptr;
+    ze_fabric_vertex_exp_properties_t properties = {};
+};
+
+} // namespace L0