Update Fabric Vertex Support to include iaf querying

This patch adds
1. enumeration of connected iaf devices
2. move fabric vertex to driver, to support deviceless
vertices case

Related-To: LOCI-3376

Signed-off-by: Joshua Santosh Ranjan <joshua.santosh.ranjan@intel.com>

level_zero/core/source/CMakeLists.txt

@@ -53,8 +53,6 @@ set(L0_RUNTIME_SOURCES
     ${CMAKE_CURRENT_SOURCE_DIR}/event/event.h
     ${CMAKE_CURRENT_SOURCE_DIR}/fence/fence.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/fence/fence.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/fabric/fabric.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/fabric/fabric.h
     ${CMAKE_CURRENT_SOURCE_DIR}/helpers/allocation_extensions.h
     ${CMAKE_CURRENT_SOURCE_DIR}/helpers/allocation_extensions.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/helpers/api_specific_config_l0.cpp

level_zero/core/source/device/device_imp.cpp

@@ -1099,8 +1099,6 @@ Device *Device::create(DriverHandle *driverHandle, NEO::Device *neoDevice, bool
 
     device->populateSubDeviceCopyEngineGroups();
 
-    device->fabricVertex = std::unique_ptr<FabricVertex>(FabricVertex::createFromDevice(device));
-
     return device;
 }
 

level_zero/core/source/device/device_imp.h

@@ -132,10 +132,11 @@ struct DeviceImp : public Device {
     void createSysmanHandle(bool isSubDevice);
     void populateSubDeviceCopyEngineGroups();
     bool isQueueGroupOrdinalValid(uint32_t ordinal);
+    void setFabricVertex(FabricVertex *inFabricVertex) { fabricVertex = inFabricVertex; }
 
     using CmdListCreateFunPtrT = L0::CommandList *(*)(uint32_t, Device *, NEO::EngineGroupType, ze_command_list_flags_t, ze_result_t &);
     CmdListCreateFunPtrT getCmdListCreateFunc(const ze_command_list_desc_t *desc);
-    std::unique_ptr<FabricVertex> fabricVertex;
+    FabricVertex *fabricVertex = nullptr;
 
     ze_result_t queryDeviceLuid(ze_device_luid_ext_properties_t *deviceLuidProperties);
     ze_result_t setDeviceLuid(ze_device_luid_ext_properties_t *deviceLuidProperties);

level_zero/core/source/driver/driver_handle_imp.cpp

@@ -159,6 +159,12 @@ DriverHandleImp::~DriverHandleImp() {
     for (auto &device : this->devices) {
         delete device;
     }
+
+    for (auto &fabricVertex : this->fabricVertices) {
+        delete fabricVertex;
+    }
+    this->fabricVertices.clear();
+
     if (this->svmAllocsManager) {
         delete this->svmAllocsManager;
         this->svmAllocsManager = nullptr;
@@ -232,6 +238,14 @@ ze_result_t DriverHandleImp::initialize(std::vector<std::unique_ptr<NEO::Device>
         createHostPointerManager();
     }
 
+    for (auto &device : this->devices) {
+
+        auto deviceImpl = static_cast<DeviceImp *>(device);
+        auto fabricVertex = FabricVertex::createFromDevice(device);
+        deviceImpl->setFabricVertex(fabricVertex);
+        this->fabricVertices.push_back(fabricVertex);
+    }
+
     return ZE_RESULT_SUCCESS;
 }
 
@@ -621,23 +635,16 @@ ze_result_t DriverHandleImp::checkMemoryAccessFromDevice(Device *device, const v
 
 ze_result_t DriverHandleImp::fabricVertexGetExp(uint32_t *pCount, ze_fabric_vertex_handle_t *phVertices) {
 
-    uint32_t deviceCount = 0;
-    getDevice(&deviceCount, nullptr);
-
+    uint32_t fabricVertexCount = static_cast<uint32_t>(this->fabricVertices.size());
     if (*pCount == 0) {
-        *pCount = deviceCount;
+        *pCount = fabricVertexCount;
         return ZE_RESULT_SUCCESS;
     }
 
-    std::vector<ze_device_handle_t> deviceHandles;
-    deviceHandles.resize(deviceCount);
-    getDevice(&deviceCount, deviceHandles.data());
-
-    *pCount = std::min(deviceCount, *pCount);
+    *pCount = std::min(fabricVertexCount, *pCount);
 
     for (uint32_t index = 0; index < *pCount; index++) {
-        auto deviceImp = static_cast<DeviceImp *>(deviceHandles[index]);
-        phVertices[index] = deviceImp->fabricVertex->toHandle();
+        phVertices[index] = this->fabricVertices[index]->toHandle();
     }
 
     return ZE_RESULT_SUCCESS;

level_zero/core/source/driver/driver_handle_imp.h

@@ -18,6 +18,7 @@
 
 namespace L0 {
 class HostPointerManager;
+struct FabricVertex;
 
 struct DriverHandleImp : public DriverHandle {
     ~DriverHandleImp() override;
@@ -86,6 +87,7 @@ struct DriverHandleImp : public DriverHandle {
     std::map<void *, NEO::GraphicsAllocation *> sharedMakeResidentAllocations;
 
     std::vector<Device *> devices;
+    std::vector<FabricVertex *> fabricVertices;
     // Spec extensions
     const std::vector<std::pair<std::string, uint32_t>> extensionsSupported = {
         {ZE_FLOAT_ATOMICS_EXT_NAME, ZE_FLOAT_ATOMICS_EXT_VERSION_CURRENT},

level_zero/core/source/fabric/CMakeLists.txt

@@ -0,0 +1,20 @@
+#
+# Copyright (C) 2022 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+set(L0_SRCS_FABRIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
+    ${CMAKE_CURRENT_SOURCE_DIR}/fabric.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/fabric_device_interface.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/fabric.cpp
+)
+
+add_subdirectories()
+set(L0_RUNTIME_SOURCES
+    ${L0_RUNTIME_SOURCES}
+    ${L0_SRCS_FABRIC}
+    ${L0_SRCS_FABRIC_OS}
+    PARENT_SCOPE
+)

level_zero/core/source/fabric/fabric.cpp

@@ -10,13 +10,39 @@
 #include "shared/source/helpers/string.h"
 
 #include "level_zero/core/source/device/device_imp.h"
+#include "level_zero/core/source/fabric/fabric_device_interface.h"
 
 namespace L0 {
 
+FabricVertex::~FabricVertex() {
+
+    for (auto subVertex : subVertices) {
+        delete subVertex;
+    }
+    subVertices.clear();
+}
+
 FabricVertex *FabricVertex::createFromDevice(Device *device) {
 
+    // Fabric Vertices are not created for engine instanced devices
+    if (device->getNEODevice()->isEngineInstanced()) {
+        return nullptr;
+    }
+
     auto fabricVertex = new FabricVertex();
     UNRECOVERABLE_IF(fabricVertex == nullptr);
+
+    auto deviceImpl = static_cast<DeviceImp *>(device);
+    for (auto &subDevice : deviceImpl->subDevices) {
+        auto subVertex = FabricVertex::createFromDevice(subDevice);
+        if (subVertex == nullptr) {
+            continue;
+        }
+        auto subDeviceImpl = static_cast<DeviceImp *>(subDevice);
+        subDeviceImpl->setFabricVertex(subVertex);
+        fabricVertex->subVertices.push_back(subVertex);
+    }
+
     ze_device_properties_t deviceProperties = {};
     ze_pci_ext_properties_t pciProperties = {};
 
@@ -42,31 +68,36 @@ FabricVertex *FabricVertex::createFromDevice(Device *device) {
         fabricVertex->properties.address.function = pciProperties.address.function;
     }
 
+    fabricVertex->pFabricDeviceInterface = FabricDeviceInterface::createFabricDeviceInterface(*fabricVertex);
+    UNRECOVERABLE_IF(fabricVertex->pFabricDeviceInterface == nullptr);
+
+    fabricVertex->pFabricDeviceInterface->enumerate();
+
     return fabricVertex;
 }
 
 ze_result_t FabricVertex::getSubVertices(uint32_t *pCount, ze_fabric_vertex_handle_t *phSubvertices) {
 
-    auto deviceImp = static_cast<DeviceImp *>(device);
+    uint32_t subVertexCount = static_cast<uint32_t>(subVertices.size());
     if (*pCount == 0) {
-        *pCount = deviceImp->numSubDevices;
+        *pCount = subVertexCount;
         return ZE_RESULT_SUCCESS;
     }
 
-    *pCount = std::min(deviceImp->numSubDevices, *pCount);
+    *pCount = std::min(subVertexCount, *pCount);
     for (uint32_t index = 0; index < *pCount; index++) {
-        auto subDeviceImp = static_cast<DeviceImp *>(deviceImp->subDevices[index]);
-        phSubvertices[index] = subDeviceImp->fabricVertex->toHandle();
+        phSubvertices[index] = subVertices[index]->toHandle();
     }
+
     return ZE_RESULT_SUCCESS;
 }
 
-ze_result_t FabricVertex::getProperties(ze_fabric_vertex_exp_properties_t *pVertexProperties) {
+ze_result_t FabricVertex::getProperties(ze_fabric_vertex_exp_properties_t *pVertexProperties) const {
     *pVertexProperties = properties;
     return ZE_RESULT_SUCCESS;
 }
 
-ze_result_t FabricVertex::getDevice(ze_device_handle_t *phDevice) {
+ze_result_t FabricVertex::getDevice(ze_device_handle_t *phDevice) const {
 
     *phDevice = device->toHandle();
     return ZE_RESULT_SUCCESS;

level_zero/core/source/fabric/fabric.h

@@ -9,25 +9,33 @@
 
 #include <level_zero/ze_api.h>
 
+#include <memory>
+#include <vector>
+
 struct _ze_fabric_vertex_handle_t {};
 
 namespace L0 {
 struct Device;
+class FabricDeviceInterface;
 
 struct FabricVertex : _ze_fabric_vertex_handle_t {
 
   public:
     static FabricVertex *createFromDevice(Device *device);
-    virtual ~FabricVertex() = default;
+    virtual ~FabricVertex();
     ze_result_t getSubVertices(uint32_t *pCount, ze_fabric_vertex_handle_t *phSubvertices);
-    ze_result_t getProperties(ze_fabric_vertex_exp_properties_t *pVertexProperties);
-    ze_result_t getDevice(ze_device_handle_t *phDevice);
+    ze_result_t getProperties(ze_fabric_vertex_exp_properties_t *pVertexProperties) const;
+    ze_result_t getDevice(ze_device_handle_t *phDevice) const;
     static FabricVertex *fromHandle(ze_fabric_vertex_handle_t handle) { return static_cast<FabricVertex *>(handle); }
     inline ze_fabric_vertex_handle_t toHandle() { return this; }
 
+    std::unique_ptr<FabricDeviceInterface> pFabricDeviceInterface;
+
   private:
     struct Device *device = nullptr;
     ze_fabric_vertex_exp_properties_t properties = {};
+
+    std::vector<FabricVertex *> subVertices = {};
 };
 
 } // namespace L0

level_zero/core/source/fabric/fabric_device_interface.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "level_zero/core/source/fabric/fabric.h"
+#include <level_zero/ze_api.h>
+
+#include <memory>
+
+namespace L0 {
+
+class FabricDeviceInterface {
+
+  public:
+    virtual ~FabricDeviceInterface(){};
+    virtual ze_result_t enumerate() = 0;
+    static std::unique_ptr<FabricDeviceInterface> createFabricDeviceInterface(const FabricVertex &fabricVertex);
+};
+
+} // namespace L0

level_zero/core/source/fabric/linux/CMakeLists.txt

@@ -0,0 +1,25 @@
+#
+# Copyright (C) 2022 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+if(UNIX)
+
+  if(LIBGENL_FOUND AND NEO_ENABLE_i915_PRELIM_DETECTION)
+    set(L0_SRCS_FABRIC_OS
+        ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
+        ${CMAKE_CURRENT_SOURCE_DIR}/fabric_device_iaf.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/fabric_device_iaf.cpp
+        PARENT_SCOPE
+    )
+  else()
+    set(L0_SRCS_FABRIC_OS
+        ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
+        ${CMAKE_CURRENT_SOURCE_DIR}/fabric_device_iaf_stub.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/fabric_device_iaf_stub.cpp
+        PARENT_SCOPE
+    )
+  endif()
+
+endif()

level_zero/core/source/fabric/linux/fabric_device_iaf.cpp

@@ -0,0 +1,171 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/core/source/fabric/linux/fabric_device_iaf.h"
+
+#include "shared/source/debug_settings/debug_settings_manager.h"
+#include "shared/source/os_interface/linux/drm_neo.h"
+#include "shared/source/os_interface/linux/pci_path.h"
+#include "shared/source/os_interface/os_interface.h"
+
+#include "level_zero/core/source/device/device.h"
+#include "level_zero/core/source/device/device_imp.h"
+#include "level_zero/core/source/fabric/fabric.h"
+
+namespace L0 {
+
+FabricDeviceIaf::FabricDeviceIaf(Device *device) : device(device) {
+
+    DeviceImp *deviceImp = static_cast<DeviceImp *>(device);
+
+    if (deviceImp->numSubDevices == 0) {
+        //Add one sub-device
+        subDeviceIafs.push_back(std::make_unique<FabricSubDeviceIaf>(device));
+    } else {
+        for (const auto &subDevice : deviceImp->subDevices) {
+            subDeviceIafs.push_back(std::make_unique<FabricSubDeviceIaf>(subDevice));
+        }
+    }
+}
+
+ze_result_t FabricDeviceIaf::enumerate() {
+
+    ze_result_t result = ZE_RESULT_SUCCESS;
+    for (auto &subDeviceIaf : subDeviceIafs) {
+        result = subDeviceIaf->enumerate();
+        if (result != ZE_RESULT_SUCCESS) {
+            break;
+        }
+    }
+
+    return result;
+}
+
+FabricSubDeviceIaf::FabricSubDeviceIaf(Device *device) : device(device) {
+    pIafNlApi = std::make_unique<IafNlApi>();
+    UNRECOVERABLE_IF(nullptr == pIafNlApi);
+}
+
+ze_result_t FabricSubDeviceIaf::enumerate() {
+
+    auto &osInterface = device->getNEODevice()->getRootDeviceEnvironment().osInterface;
+
+    if (osInterface == nullptr) {
+        return ZE_RESULT_ERROR_UNINITIALIZED;
+    }
+
+    if (osInterface->getDriverModel()->getDriverModelType() != NEO::DriverModelType::DRM) {
+        return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
+    }
+
+    auto pDrm = osInterface->getDriverModel()->as<NEO::Drm>();
+    std::optional<std::string> rootPciPath = NEO::getPciLinkPath(pDrm->getFileDescriptor());
+    if (!rootPciPath.has_value()) {
+        return ZE_RESULT_ERROR_UNKNOWN;
+    }
+
+    std::vector<IafPort> iafPorts = {};
+    ze_result_t result = pIafNlApi->getPorts("/sys/class/drm/" + rootPciPath.value() + "/device/", iafPorts);
+    if (ZE_RESULT_SUCCESS != result) {
+        return result;
+    }
+
+    DeviceImp *deviceImp = static_cast<DeviceImp *>(device);
+    uint32_t physicalSubDeviceId = deviceImp->getPhysicalSubDeviceId();
+
+    // Remove ports which donot belong to this device
+    for (auto iter = iafPorts.begin(); iter != iafPorts.end();) {
+        IafPort &port = *iter;
+        if (port.portId.attachId != physicalSubDeviceId) {
+            iter = iafPorts.erase(iter);
+            continue;
+        }
+        ++iter;
+    }
+
+    // Get Connections
+    for (auto iafPort : iafPorts) {
+        FabricPortConnection connection = {};
+        ze_result_t result = getConnection(iafPort, connection);
+        if (result != ZE_RESULT_SUCCESS) {
+            NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr,
+                                  "failure observed for IafPort{0x%x, 0x%x, 0x%x}: 0x%x\n",
+                                  iafPort.portId.fabricId, iafPort.portId.attachId, iafPort.portId.portNumber,
+                                  result);
+            continue;
+        }
+        connections.push_back(connection);
+    }
+
+    // Get guid
+    if (iafPorts.size() > 0) {
+        std::vector<uint8_t> ports = {};
+        if (ZE_RESULT_SUCCESS != pIafNlApi->subdevicePropertiesGet(iafPorts[0].portId.fabricId, physicalSubDeviceId, guid, ports)) {
+            connections.clear();
+            NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr,
+                                  "failure during fabric port guid reading {0x%x, 0x%x, 0x%x}: result: 0x%x subdeviceId:%d\n",
+                                  iafPorts[0].portId.fabricId, iafPorts[0].portId.attachId, iafPorts[0].portId.portNumber,
+                                  result, physicalSubDeviceId);
+            return ZE_RESULT_ERROR_UNKNOWN;
+        }
+    }
+
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t FabricSubDeviceIaf::getConnection(IafPort &port, FabricPortConnection &connection) {
+
+    IafPortState iafPortState = {};
+    ze_result_t result = pIafNlApi->fPortStatusQuery(port.portId, iafPortState);
+    if (ZE_RESULT_SUCCESS != result || iafPortState.healthStatus != IAF_FPORT_HEALTH_HEALTHY) {
+        return ZE_RESULT_ERROR_UNKNOWN;
+    }
+
+    // Consider only healthy ports
+    uint64_t neighbourGuid;
+    uint8_t neighbourPortNumber;
+    IafPortSpeed maxRxSpeed = {}, maxTxSpeed = {};
+    IafPortSpeed rxSpeed = {}, txSpeed = {};
+
+    result = pIafNlApi->fportProperties(port.portId, neighbourGuid, neighbourPortNumber, maxRxSpeed, maxTxSpeed, rxSpeed, txSpeed);
+    if (ZE_RESULT_SUCCESS != result) {
+        return result;
+    }
+
+    connection.currentid = port.portId;
+    connection.neighbourPortNumber = neighbourPortNumber;
+    connection.neighbourGuid = neighbourGuid;
+    const double bandwidthInBitsPerSecond = static_cast<double>(maxRxSpeed.bitRate * maxRxSpeed.width);
+    // 8ull - bits to bytes; 1e9 - seconds to nano-seconds
+    connection.bandwidthInBytesPerNanoSecond = static_cast<uint32_t>(bandwidthInBitsPerSecond / (8ull * 1e9));
+    connection.isDuplex = true;
+    return ZE_RESULT_SUCCESS;
+}
+
+void FabricSubDeviceIaf::setIafNlApi(IafNlApi *iafNlApi) {
+    pIafNlApi.reset(iafNlApi);
+}
+
+std::unique_ptr<FabricDeviceInterface> FabricDeviceInterface::createFabricDeviceInterface(const FabricVertex &fabricVertex) {
+
+    ze_fabric_vertex_exp_properties_t vertexProperties = {};
+    fabricVertex.getProperties(&vertexProperties);
+
+    ze_device_handle_t hDevice = nullptr;
+    fabricVertex.getDevice(&hDevice);
+    DEBUG_BREAK_IF(hDevice == nullptr);
+
+    DeviceImp *deviceImp = static_cast<DeviceImp *>(hDevice);
+    Device *device = static_cast<Device *>(hDevice);
+    if (deviceImp->isSubdevice) {
+        return std::unique_ptr<FabricDeviceInterface>(new (std::nothrow) FabricSubDeviceIaf(device));
+    }
+
+    return std::unique_ptr<FabricDeviceInterface>(new (std::nothrow) FabricDeviceIaf(device));
+}
+
+} // namespace L0

level_zero/core/source/fabric/linux/fabric_device_iaf.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "level_zero/core/source/fabric/fabric_device_interface.h"
+#include "level_zero/tools/source/sysman/linux/nl_api/iaf_nl_api.h"
+#include <level_zero/ze_api.h>
+
+#include <vector>
+
+namespace L0 {
+
+class FabricSubDeviceIaf;
+
+struct FabricPortConnection {
+    IafPortId currentid;
+    uint8_t neighbourPortNumber;
+    uint64_t neighbourGuid;
+    uint32_t bandwidthInBytesPerNanoSecond;
+    bool isDuplex;
+};
+
+class FabricDeviceIaf : public FabricDeviceInterface {
+
+  public:
+    FabricDeviceIaf(Device *device);
+    ~FabricDeviceIaf() override = default;
+    ze_result_t enumerate() override;
+    std::vector<std::unique_ptr<FabricSubDeviceIaf>> subDeviceIafs = {};
+
+  protected:
+    Device *device = nullptr;
+};
+
+class FabricSubDeviceIaf : public FabricDeviceInterface {
+
+  public:
+    FabricSubDeviceIaf(Device *device);
+    ~FabricSubDeviceIaf() override = default;
+    ze_result_t enumerate() override;
+    void setIafNlApi(IafNlApi *iafNlApi);
+    std::vector<FabricPortConnection> connections = {};
+
+  protected:
+    ze_result_t getConnection(IafPort &port, FabricPortConnection &connection);
+
+    std::unique_ptr<IafNlApi> pIafNlApi = nullptr;
+    Device *device = nullptr;
+    uint64_t guid = 0ull;
+};
+
+} // namespace L0

level_zero/core/source/fabric/linux/fabric_device_iaf_stub.cpp

@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/core/source/fabric/linux/fabric_device_iaf_stub.h"
+
+#include "level_zero/core/source/fabric/fabric.h"
+
+namespace L0 {
+
+std::unique_ptr<FabricDeviceInterface> FabricDeviceInterface::createFabricDeviceInterface(const FabricVertex &fabricVertex) {
+
+    return std::unique_ptr<FabricDeviceInterface>(new (std::nothrow) FabricDeviceIafStub());
+}
+
+} // namespace L0

level_zero/core/source/fabric/linux/fabric_device_iaf_stub.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "level_zero/core/source/fabric/fabric_device_interface.h"
+#include <level_zero/ze_api.h>
+
+namespace L0 {
+
+class FabricDeviceIafStub : public FabricDeviceInterface {
+
+  public:
+    FabricDeviceIafStub() = default;
+    ~FabricDeviceIafStub() override = default;
+    ze_result_t enumerate() override {
+        return ZE_RESULT_SUCCESS;
+    }
+};
+
+} // namespace L0

level_zero/core/source/fabric/windows/CMakeLists.txt

@@ -0,0 +1,14 @@
+#
+# Copyright (C) 2022 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+if(WIN32)
+  set(L0_SRCS_FABRIC_OS
+      ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
+      ${CMAKE_CURRENT_SOURCE_DIR}/fabric_device_iaf.h
+      ${CMAKE_CURRENT_SOURCE_DIR}/fabric_device_iaf.cpp
+      PARENT_SCOPE
+  )
+endif()

level_zero/core/source/fabric/windows/fabric_device_iaf.cpp

@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/core/source/fabric/windows/fabric_device_iaf.h"
+
+#include "level_zero/core/source/fabric/fabric.h"
+
+namespace L0 {
+
+std::unique_ptr<FabricDeviceInterface> FabricDeviceInterface::createFabricDeviceInterface(const FabricVertex &fabricVertex) {
+
+    return std::unique_ptr<FabricDeviceInterface>(new (std::nothrow) FabricDeviceIaf());
+}
+
+} // namespace L0

level_zero/core/source/fabric/windows/fabric_device_iaf.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "level_zero/core/source/fabric/fabric_device_interface.h"
+#include <level_zero/ze_api.h>
+
+namespace L0 {
+
+class FabricDeviceIaf : public FabricDeviceInterface {
+
+  public:
+    FabricDeviceIaf() = default;
+    ~FabricDeviceIaf() override = default;
+    ze_result_t enumerate() override {
+        return ZE_RESULT_SUCCESS;
+    }
+};
+
+} // namespace L0

level_zero/core/test/unit_tests/fixtures/device_fixture.h

@@ -99,7 +99,6 @@ struct MultiDeviceFixture {
 
     DebugManagerStateRestore restorer;
     std::unique_ptr<Mock<L0::DriverHandleImp>> driverHandle;
-    std::vector<NEO::Device *> devices;
     uint32_t numRootDevices = 4u;
     uint32_t numSubDevices = 2u;
     L0::ContextImp *context = nullptr;

level_zero/core/test/unit_tests/sources/fabric/CMakeLists.txt

@@ -8,3 +8,4 @@ target_sources(${TARGET_NAME} PRIVATE
                ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
                ${CMAKE_CURRENT_SOURCE_DIR}/test_fabric.cpp
 )
+add_subdirectories()

level_zero/core/test/unit_tests/sources/fabric/linux/CMakeLists.txt

@@ -0,0 +1,14 @@
+#
+# Copyright (C) 2022 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+if(UNIX)
+  if(LIBGENL_FOUND)
+    target_sources(${TARGET_NAME} PRIVATE
+                   ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
+                   ${CMAKE_CURRENT_SOURCE_DIR}/test_fabric_iaf.cpp
+    )
+  endif()
+endif()

level_zero/core/test/unit_tests/sources/fabric/linux/test_fabric_iaf.cpp

@@ -0,0 +1,231 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/test/common/helpers/debug_manager_state_restore.h"
+#include "shared/test/common/libult/linux/drm_mock.h"
+
+#include "level_zero/core/source/fabric/linux/fabric_device_iaf.h"
+#include "level_zero/core/test/unit_tests/fixtures/device_fixture.h"
+
+#include "gtest/gtest.h"
+
+namespace L0 {
+namespace ult {
+
+class MockIafNlApi : public IafNlApi {
+  public:
+    ze_result_t fPortStatusQueryStatus = ZE_RESULT_SUCCESS;
+    ze_result_t fportPropertiesStatus = ZE_RESULT_SUCCESS;
+    ze_result_t subDevicePropertiesStatus = ZE_RESULT_SUCCESS;
+    ze_result_t getPortsStatus = ZE_RESULT_SUCCESS;
+    iaf_fport_health fPortStatusQueryHealthStatus = IAF_FPORT_HEALTH_HEALTHY;
+    bool portEnumerationEnable = true;
+
+    ze_result_t handleResponse(const uint16_t cmdOp, struct genl_info *info, void *pOutput) override { return ZE_RESULT_SUCCESS; };
+    ze_result_t fPortStatusQuery(const IafPortId portId, IafPortState &state) override {
+        state.healthStatus = fPortStatusQueryHealthStatus;
+        return fPortStatusQueryStatus;
+    };
+
+    ze_result_t getThroughput(const IafPortId portId, IafPortThroughPut &throughput) override { return ZE_RESULT_SUCCESS; };
+    ze_result_t portStateQuery(const IafPortId portId, bool &enabled) override { return ZE_RESULT_SUCCESS; };
+    ze_result_t portBeaconStateQuery(const IafPortId portId, bool &enabled) override { return ZE_RESULT_SUCCESS; };
+    ze_result_t portBeaconEnable(const IafPortId portId) override { return ZE_RESULT_SUCCESS; };
+    ze_result_t portBeaconDisable(const IafPortId portId) override { return ZE_RESULT_SUCCESS; };
+    ze_result_t portEnable(const IafPortId portId) override { return ZE_RESULT_SUCCESS; };
+    ze_result_t portDisable(const IafPortId portId) override { return ZE_RESULT_SUCCESS; };
+    ze_result_t portUsageEnable(const IafPortId portId) override { return ZE_RESULT_SUCCESS; };
+    ze_result_t portUsageDisable(const IafPortId portId) override { return ZE_RESULT_SUCCESS; };
+    ze_result_t remRequest() override { return ZE_RESULT_SUCCESS; };
+    ze_result_t routingGenQuery(uint32_t &start, uint32_t &end) override { return ZE_RESULT_SUCCESS; };
+    ze_result_t deviceEnum(std::vector<uint32_t> &fabricIds) override { return ZE_RESULT_SUCCESS; };
+    ze_result_t fabricDeviceProperties(const uint32_t fabricId, uint32_t &numSubdevices) override { return ZE_RESULT_SUCCESS; };
+    ze_result_t subdevicePropertiesGet(const uint32_t fabricId, const uint32_t attachId, uint64_t &guid, std::vector<uint8_t> &ports) override {
+        return subDevicePropertiesStatus;
+    };
+    ze_result_t fportProperties(const IafPortId portId, uint64_t &neighborGuid, uint8_t &neighborPortNumber,
+                                IafPortSpeed &maxRxSpeed, IafPortSpeed &maxTxSpeed,
+                                IafPortSpeed &rxSpeed, IafPortSpeed &txSpeed) override {
+        neighborPortNumber = 8;
+        neighborGuid = 0xFEEDBEAD;
+        maxRxSpeed.width = 4;
+        maxRxSpeed.bitRate = 53125000000;
+        maxTxSpeed = maxRxSpeed;
+        rxSpeed = maxRxSpeed;
+        txSpeed = maxRxSpeed;
+
+        return fportPropertiesStatus;
+    };
+    ze_result_t getPorts(const std::string &devicePciPath, std::vector<IafPort> &ports) override {
+
+        if (portEnumerationEnable) {
+            IafPort defaultPort;
+            defaultPort.onSubdevice = true;
+            defaultPort.portId.fabricId = testPortId.fabricId;
+            defaultPort.portId.attachId = testPortId.attachId;
+            defaultPort.portId.portNumber = testPortId.portNumber;
+            defaultPort.model = "XeLink";
+            defaultPort.maxRxSpeed.width = 4;
+            defaultPort.maxRxSpeed.bitRate = 53125000000;
+            defaultPort.maxTxSpeed = defaultPort.maxRxSpeed;
+
+            ports.push_back(defaultPort);
+            defaultPort.portId.fabricId = testPortId.fabricId + 1;
+            defaultPort.portId.attachId = testPortId.attachId + 1;
+            defaultPort.portId.portNumber = testPortId.portNumber + 1;
+            ports.push_back(defaultPort);
+            defaultPort.portId.fabricId = testPortId.fabricId + 2;
+            defaultPort.portId.attachId = testPortId.attachId;
+            defaultPort.portId.portNumber = testPortId.portNumber + 2;
+            ports.push_back(defaultPort);
+        }
+
+        return getPortsStatus;
+    };
+
+    IafPortId testPortId = {};
+};
+
+struct TestFabricIaf : public ::testing::Test {
+    void SetUp() override {
+        DebugManager.flags.CreateMultipleSubDevices.set(1);
+        executionEnvironment = new NEO::ExecutionEnvironment();
+        executionEnvironment->prepareRootDeviceEnvironments(1);
+        executionEnvironment->rootDeviceEnvironments[0]->setHwInfo(defaultHwInfo.get());
+        osInterface = new OSInterface();
+        drmMock = new DrmMockResources(*executionEnvironment->rootDeviceEnvironments[0]);
+        executionEnvironment->rootDeviceEnvironments[0]->osInterface.reset(osInterface);
+        executionEnvironment->rootDeviceEnvironments[0]->osInterface->setDriverModel(std::unique_ptr<Drm>(drmMock));
+        neoDevice = NEO::MockDevice::create<NEO::MockDevice>(executionEnvironment, 0u);
+    }
+    void TearDown() override {}
+
+    DebugManagerStateRestore restorer;
+
+    NEO::ExecutionEnvironment *executionEnvironment = nullptr;
+    OSInterface *osInterface = nullptr;
+    NEO::MockDevice *neoDevice = nullptr;
+    DrmMockResources *drmMock = nullptr;
+    L0::DriverHandleImp *driverHandle = nullptr;
+};
+
+TEST_F(TestFabricIaf, GivenIafFabricAvailableWhenFabricVerticesAreCreatedThenEnumerationIsSuccessful) {
+
+    NEO::DeviceVector devices;
+    devices.push_back(std::unique_ptr<NEO::Device>(neoDevice));
+    std::unique_ptr<Mock<L0::DriverHandleImp>> driverHandle = std::make_unique<Mock<L0::DriverHandleImp>>();
+    driverHandle->initialize(std::move(devices));
+
+    auto deviceImp = static_cast<DeviceImp *>(driverHandle->devices[0]);
+
+    FabricSubDeviceIaf *subDeviceFabric = new FabricSubDeviceIaf(deviceImp);
+    subDeviceFabric->setIafNlApi(new MockIafNlApi());
+    EXPECT_EQ(ZE_RESULT_SUCCESS, subDeviceFabric->enumerate());
+    delete subDeviceFabric;
+
+    FabricDeviceIaf *deviceFabric = new FabricDeviceIaf(deviceImp);
+    deviceFabric->subDeviceIafs[0]->setIafNlApi(new MockIafNlApi());
+    EXPECT_EQ(ZE_RESULT_SUCCESS, deviceFabric->enumerate());
+    delete deviceFabric;
+}
+
+TEST_F(TestFabricIaf, GivenIafFabricAvailableWhenPortStatusQueryIsUnsuccessfulThenNoConnectionsAreEnumerated) {
+
+    NEO::DeviceVector devices;
+    devices.push_back(std::unique_ptr<NEO::Device>(neoDevice));
+    std::unique_ptr<Mock<L0::DriverHandleImp>> driverHandle = std::make_unique<Mock<L0::DriverHandleImp>>();
+    driverHandle->initialize(std::move(devices));
+
+    auto deviceImp = static_cast<DeviceImp *>(driverHandle->devices[0]);
+
+    FabricSubDeviceIaf *subDeviceFabric = new FabricSubDeviceIaf(deviceImp);
+    MockIafNlApi *mockNlApi = new MockIafNlApi();
+    mockNlApi->fPortStatusQueryStatus = ZE_RESULT_ERROR_UNKNOWN;
+    subDeviceFabric->setIafNlApi(mockNlApi);
+    subDeviceFabric->enumerate();
+    EXPECT_EQ(subDeviceFabric->connections.size(), 0u);
+    delete subDeviceFabric;
+
+    subDeviceFabric = new FabricSubDeviceIaf(deviceImp);
+    mockNlApi = new MockIafNlApi();
+    mockNlApi->fPortStatusQueryStatus = ZE_RESULT_SUCCESS;
+    mockNlApi->fPortStatusQueryHealthStatus = IAF_FPORT_HEALTH_DEGRADED;
+    subDeviceFabric->setIafNlApi(mockNlApi);
+    subDeviceFabric->enumerate();
+    EXPECT_EQ(subDeviceFabric->connections.size(), 0u);
+    delete subDeviceFabric;
+}
+
+TEST_F(TestFabricIaf, GivenIafFabricAvailableWhenPortPropertiesQueryIsUnsuccessfulThenNoConnectionsAreEnumerated) {
+
+    NEO::DeviceVector devices;
+    devices.push_back(std::unique_ptr<NEO::Device>(neoDevice));
+    std::unique_ptr<Mock<L0::DriverHandleImp>> driverHandle = std::make_unique<Mock<L0::DriverHandleImp>>();
+    driverHandle->initialize(std::move(devices));
+
+    auto deviceImp = static_cast<DeviceImp *>(driverHandle->devices[0]);
+
+    FabricSubDeviceIaf *subDeviceFabric = new FabricSubDeviceIaf(deviceImp);
+    MockIafNlApi *mockNlApi = new MockIafNlApi();
+    mockNlApi->fportPropertiesStatus = ZE_RESULT_ERROR_UNKNOWN;
+    subDeviceFabric->setIafNlApi(mockNlApi);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, subDeviceFabric->enumerate());
+    EXPECT_EQ(subDeviceFabric->connections.size(), 0u);
+    delete subDeviceFabric;
+}
+
+TEST_F(TestFabricIaf, GivenIafFabricAvailableWhenSubDevicePropertiesGetIsUnsuccessfulThenReturnError) {
+
+    NEO::DeviceVector devices;
+    devices.push_back(std::unique_ptr<NEO::Device>(neoDevice));
+    std::unique_ptr<Mock<L0::DriverHandleImp>> driverHandle = std::make_unique<Mock<L0::DriverHandleImp>>();
+    driverHandle->initialize(std::move(devices));
+
+    auto deviceImp = static_cast<DeviceImp *>(driverHandle->devices[0]);
+
+    FabricSubDeviceIaf *subDeviceFabric = new FabricSubDeviceIaf(deviceImp);
+    MockIafNlApi *mockNlApi = new MockIafNlApi();
+    mockNlApi->subDevicePropertiesStatus = ZE_RESULT_ERROR_UNKNOWN;
+    subDeviceFabric->setIafNlApi(mockNlApi);
+    EXPECT_EQ(ZE_RESULT_ERROR_UNKNOWN, subDeviceFabric->enumerate());
+    delete subDeviceFabric;
+}
+
+TEST_F(TestFabricIaf, GivenIafFabricAvailableWhenNoPortsCanBeEnumeratedThenReturnSuccess) {
+
+    NEO::DeviceVector devices;
+    devices.push_back(std::unique_ptr<NEO::Device>(neoDevice));
+    std::unique_ptr<Mock<L0::DriverHandleImp>> driverHandle = std::make_unique<Mock<L0::DriverHandleImp>>();
+    driverHandle->initialize(std::move(devices));
+
+    auto deviceImp = static_cast<DeviceImp *>(driverHandle->devices[0]);
+
+    FabricSubDeviceIaf *subDeviceFabric = new FabricSubDeviceIaf(deviceImp);
+    MockIafNlApi *mockNlApi = new MockIafNlApi();
+    mockNlApi->portEnumerationEnable = false;
+    subDeviceFabric->setIafNlApi(mockNlApi);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, subDeviceFabric->enumerate());
+    delete subDeviceFabric;
+}
+
+TEST_F(TestFabricIaf, GivenIafFabricAvailableWhenGetPortsReturnsErrorThenReturnError) {
+    NEO::DeviceVector devices;
+    devices.push_back(std::unique_ptr<NEO::Device>(neoDevice));
+    std::unique_ptr<Mock<L0::DriverHandleImp>> driverHandle = std::make_unique<Mock<L0::DriverHandleImp>>();
+    driverHandle->initialize(std::move(devices));
+    auto deviceImp = static_cast<DeviceImp *>(driverHandle->devices[0]);
+
+    FabricSubDeviceIaf *subDeviceFabric = new FabricSubDeviceIaf(deviceImp);
+    MockIafNlApi *mockNlApi = new MockIafNlApi();
+    mockNlApi->getPortsStatus = ZE_RESULT_ERROR_UNKNOWN;
+    subDeviceFabric->setIafNlApi(mockNlApi);
+    EXPECT_EQ(ZE_RESULT_ERROR_UNKNOWN, subDeviceFabric->enumerate());
+    delete subDeviceFabric;
+}
+
+} // namespace ult
+} // namespace L0

level_zero/core/test/unit_tests/sources/fabric/test_fabric.cpp

@@ -54,20 +54,28 @@ TEST_F(FabricVertexFixture, givenDevicesAreCreatedWhenzeFabricVertexGetSubVertic
 
 TEST_F(FabricVertexFixture, givenFabricVerticesAreCreatedWhenzeFabricVertexGetPropertiesExpIsCalledThenValidPropertiesAreReturned) {
 
+    // Delete existing fabric vertices
+    for (auto fabricVertex : driverHandle->fabricVertices) {
+        delete fabricVertex;
+    }
+    driverHandle->fabricVertices.clear();
+
     // Setup pci address for all devices
     for (auto l0Device : driverHandle->devices) {
         auto deviceImp = static_cast<L0::DeviceImp *>(l0Device);
+
         for (auto l0SubDevice : deviceImp->subDevices) {
+
             auto subDeviceImp = static_cast<L0::DeviceImp *>(l0SubDevice);
             NEO::DriverInfoMock *driverInfo = new DriverInfoMock();
             driverInfo->setPciBusInfo({0, 1, 2, 3});
             subDeviceImp->driverInfo.reset(driverInfo);
-            subDeviceImp->fabricVertex.reset(FabricVertex::createFromDevice(l0SubDevice));
         }
+
         NEO::DriverInfoMock *driverInfo = new DriverInfoMock();
         driverInfo->setPciBusInfo({0, 1, 2, 3});
         deviceImp->driverInfo.reset(driverInfo);
-        deviceImp->fabricVertex.reset(FabricVertex::createFromDevice(l0Device));
+        driverHandle->fabricVertices.push_back(FabricVertex::createFromDevice(l0Device));
     }
 
     uint32_t count = 0;
@@ -129,5 +137,27 @@ TEST_F(FabricVertexFixture, givenFabricVerticesAreCreatedWhenzeFabricVertexGetPr
     }
 }
 
+TEST(FabricEngineInstanceTest, GivenEngineInstancedDeviceWhenFabricVerticesAreCreatedThenSkipCreationForEngineInstanced) {
+
+    auto hwInfo = *NEO::defaultHwInfo.get();
+    auto executionEnvironment = MockDevice::prepareExecutionEnvironment(&hwInfo, 0u);
+    std::vector<std::unique_ptr<NEO::Device>> devices(1);
+    devices[0].reset(static_cast<NEO::Device *>(NEO::MockDevice::createWithExecutionEnvironment<NEO::MockDevice>(&hwInfo, executionEnvironment, 0u)));
+
+    auto mockDevice = static_cast<NEO::MockDevice *>(devices[0].get());
+    NEO::SubDevice *subDevice = static_cast<NEO::SubDevice *>(mockDevice->createEngineInstancedSubDevice(0, defaultHwInfo->capabilityTable.defaultEngineType));
+    mockDevice->subdevices.push_back(subDevice);
+
+    std::unique_ptr<Mock<L0::DriverHandleImp>> driverHandle;
+    driverHandle = std::make_unique<Mock<L0::DriverHandleImp>>();
+    ze_result_t res = driverHandle->initialize(std::move(devices));
+    EXPECT_EQ(ZE_RESULT_SUCCESS, res);
+
+    EXPECT_EQ(driverHandle->fabricVertices.size(), 1u);
+    uint32_t count = 0;
+    EXPECT_EQ(ZE_RESULT_SUCCESS, driverHandle->fabricVertices[0]->getSubVertices(&count, nullptr));
+    EXPECT_EQ(count, 0u);
+}
+
 } // namespace ult
 } // namespace L0