Initial draft to implement Fabric Vertex Apis using Driver Handle Information

Implement the below APIs
- zeFabricVertexGetExp
- zeFabricVertexGetSubVerticesExp
- zeFabricVertexGetPropertiesExp
- zeFabricVertexGetDeviceExp

Related-To: LOCI-3332

Signed-off-by: Joshua Santosh Ranjan <joshua.santosh.ranjan@intel.com>

level_zero/api/core/ze_core_loader.cpp

@@ -626,3 +626,23 @@ zeGetImageExpProcAddrTable(
     driver_ddiTable.core_ddiTable.ImageExp = *pDdiTable;
     return result;
 }
+
+ZE_APIEXPORT ze_result_t ZE_APICALL
+zeGetFabricVertexExpProcAddrTable(
+    ze_api_version_t version,
+    ze_fabric_vertex_exp_dditable_t *pDdiTable) {
+
+    if (nullptr == pDdiTable)
+        return ZE_RESULT_ERROR_INVALID_ARGUMENT;
+    if (ZE_MAJOR_VERSION(driver_ddiTable.version) != ZE_MAJOR_VERSION(version) ||
+        ZE_MINOR_VERSION(driver_ddiTable.version) > ZE_MINOR_VERSION(version))
+        return ZE_RESULT_ERROR_UNSUPPORTED_VERSION;
+
+    ze_result_t result = ZE_RESULT_SUCCESS;
+    pDdiTable->pfnGetExp = L0::zeFabricVertexGetExp;
+    pDdiTable->pfnGetSubVerticesExp = L0::zeFabricVertexGetSubVerticesExp;
+    pDdiTable->pfnGetPropertiesExp = L0::zeFabricVertexGetPropertiesExp;
+    pDdiTable->pfnGetDeviceExp = L0::zeFabricVertexGetDeviceExp;
+    driver_ddiTable.core_ddiTable.FabricVertexExp = *pDdiTable;
+    return result;
+}

level_zero/api/extensions/public/ze_exp_ext.cpp

@@ -9,7 +9,9 @@
 
 #include "level_zero/core/source/context/context.h"
 #include "level_zero/core/source/device/device.h"
+#include "level_zero/core/source/driver/driver_handle.h"
 #include "level_zero/core/source/event/event.h"
+#include "level_zero/core/source/fabric/fabric.h"
 #include "level_zero/core/source/image/image.h"
 #include "level_zero/core/source/kernel/kernel.h"
 
@@ -52,6 +54,36 @@ ze_result_t zeEventQueryTimestampsExp(
     return L0::Event::fromHandle(hEvent)->queryTimestampsExp(L0::Device::fromHandle(hDevice), pCount, pTimestamps);
 }
 
+ze_result_t zeFabricVertexGetExp(
+    ze_driver_handle_t hDriver,
+    uint32_t *pCount,
+    ze_fabric_vertex_handle_t *phVertices) {
+
+    return L0::DriverHandle::fromHandle(hDriver)->fabricVertexGetExp(pCount, phVertices);
+}
+
+ze_result_t zeFabricVertexGetSubVerticesExp(
+    ze_fabric_vertex_handle_t hVertex,
+    uint32_t *pCount,
+    ze_fabric_vertex_handle_t *phSubvertices) {
+
+    return L0::FabricVertex::fromHandle(hVertex)->getSubVertices(pCount, phSubvertices);
+}
+
+ze_result_t zeFabricVertexGetPropertiesExp(
+    ze_fabric_vertex_handle_t hVertex,
+    ze_fabric_vertex_exp_properties_t *pVertexProperties) {
+
+    return L0::FabricVertex::fromHandle(hVertex)->getProperties(pVertexProperties);
+}
+
+ze_result_t zeFabricVertexGetDeviceExp(
+    ze_fabric_vertex_handle_t hVertex,
+    ze_device_handle_t *phDevice) {
+
+    return L0::FabricVertex::fromHandle(hVertex)->getDevice(phDevice);
+}
+
 } // namespace L0
 
 extern "C" {
@@ -99,4 +131,34 @@ zeEventQueryTimestampsExp(
     return L0::zeEventQueryTimestampsExp(hEvent, hDevice, pCount, pTimestamps);
 }
 
+ZE_APIEXPORT ze_result_t ZE_APICALL
+zeFabricVertexGetExp(
+    ze_driver_handle_t hDriver,
+    uint32_t *pCount,
+    ze_fabric_vertex_handle_t *phVertices) {
+    return L0::zeFabricVertexGetExp(hDriver, pCount, phVertices);
+}
+
+ZE_APIEXPORT ze_result_t ZE_APICALL
+zeFabricVertexGetSubVerticesExp(
+    ze_fabric_vertex_handle_t hVertex,
+    uint32_t *pCount,
+    ze_fabric_vertex_handle_t *phSubvertices) {
+    return L0::zeFabricVertexGetSubVerticesExp(hVertex, pCount, phSubvertices);
+}
+
+ZE_APIEXPORT ze_result_t ZE_APICALL
+zeFabricVertexGetPropertiesExp(
+    ze_fabric_vertex_handle_t hVertex,
+    ze_fabric_vertex_exp_properties_t *pVertexProperties) {
+    return L0::zeFabricVertexGetPropertiesExp(hVertex, pVertexProperties);
+}
+
+ZE_APIEXPORT ze_result_t ZE_APICALL
+zeFabricVertexGetDeviceExp(
+    ze_fabric_vertex_handle_t hVertex,
+    ze_device_handle_t *phDevice) {
+    return L0::zeFabricVertexGetDeviceExp(hVertex, phDevice);
+}
+
 } // extern "C"

level_zero/api/extensions/public/ze_exp_ext.h

@@ -49,4 +49,22 @@ ze_result_t zetMetricGroupCalculateMultipleMetricValuesExp(
     uint32_t *pMetricCounts,
     zet_typed_value_t *pMetricValues);
 
+ze_result_t zeFabricVertexGetExp(
+    ze_driver_handle_t hDriver,
+    uint32_t *pCount,
+    ze_fabric_vertex_handle_t *phVertices);
+
+ze_result_t zeFabricVertexGetSubVerticesExp(
+    ze_fabric_vertex_handle_t hVertex,
+    uint32_t *pCount,
+    ze_fabric_vertex_handle_t *phSubvertices);
+
+ze_result_t zeFabricVertexGetPropertiesExp(
+    ze_fabric_vertex_handle_t hVertex,
+    ze_fabric_vertex_exp_properties_t *pVertexProperties);
+
+ze_result_t zeFabricVertexGetDeviceExp(
+    ze_fabric_vertex_handle_t hVertex,
+    ze_device_handle_t *phDevice);
+
 } // namespace L0

level_zero/core/source/CMakeLists.txt

@@ -50,6 +50,8 @@ set(L0_RUNTIME_SOURCES
     ${CMAKE_CURRENT_SOURCE_DIR}/event/event.h
     ${CMAKE_CURRENT_SOURCE_DIR}/fence/fence.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/fence/fence.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/fabric/fabric.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/fabric/fabric.h
     ${CMAKE_CURRENT_SOURCE_DIR}/helpers/allocation_extensions.h
     ${CMAKE_CURRENT_SOURCE_DIR}/helpers/allocation_extensions.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/helpers/api_specific_config_l0.cpp

level_zero/core/source/device/device_imp.cpp

@@ -38,6 +38,7 @@
 #include "level_zero/core/source/context/context_imp.h"
 #include "level_zero/core/source/driver/driver_handle_imp.h"
 #include "level_zero/core/source/event/event.h"
+#include "level_zero/core/source/fabric/fabric.h"
 #include "level_zero/core/source/hw_helpers/l0_hw_helper.h"
 #include "level_zero/core/source/image/image.h"
 #include "level_zero/core/source/module/module.h"
@@ -998,6 +999,7 @@ Device *Device::create(DriverHandle *driverHandle, NEO::Device *neoDevice, bool
 
     UNRECOVERABLE_IF(device == nullptr);
 
+    device->isSubdevice = isSubDevice;
     device->setDriverHandle(driverHandle);
     neoDevice->setSpecializedDevice(device);
 
@@ -1066,7 +1068,6 @@ Device *Device::create(DriverHandle *driverHandle, NEO::Device *neoDevice, bool
         if (subDevice == nullptr) {
             return nullptr;
         }
-        static_cast<DeviceImp *>(subDevice)->isSubdevice = true;
         static_cast<DeviceImp *>(subDevice)->setDebugSurface(debugSurface);
         device->subDevices.push_back(static_cast<Device *>(subDevice));
     }
@@ -1113,6 +1114,8 @@ Device *Device::create(DriverHandle *driverHandle, NEO::Device *neoDevice, bool
 
     device->populateSubDeviceCopyEngineGroups();
 
+    device->fabricVertex = std::unique_ptr<FabricVertex>(FabricVertex::createFromDevice(device));
+
     return device;
 }
 

level_zero/core/source/device/device_imp.h

@@ -24,6 +24,7 @@ class DriverInfo;
 
 namespace L0 {
 struct SysmanDevice;
+struct FabricVertex;
 class CacheReservation;
 
 struct DeviceImp : public Device {
@@ -132,6 +133,7 @@ struct DeviceImp : public Device {
 
     using CmdListCreateFunPtrT = L0::CommandList *(*)(uint32_t, Device *, NEO::EngineGroupType, ze_command_list_flags_t, ze_result_t &);
     CmdListCreateFunPtrT getCmdListCreateFunc(const ze_command_list_desc_t *desc);
+    std::unique_ptr<FabricVertex> fabricVertex;
 
   protected:
     void adjustCommandQueueDesc(uint32_t &ordinal, uint32_t &index);

level_zero/core/source/driver/driver_handle.h

@@ -65,6 +65,7 @@ struct DriverHandle : _ze_driver_handle_t {
                                                                      size_t size,
                                                                      uint32_t rootDeviceIndex,
                                                                      uintptr_t *gpuAddress) = 0;
+    virtual ze_result_t fabricVertexGetExp(uint32_t *pCount, ze_fabric_vertex_handle_t *phDevices) = 0;
 
     static DriverHandle *fromHandle(ze_driver_handle_t handle) { return static_cast<DriverHandle *>(handle); }
     inline ze_driver_handle_t toHandle() { return this; }

level_zero/core/source/driver/driver_handle_imp.cpp

@@ -19,6 +19,7 @@
 #include "level_zero/core/source/device/device_imp.h"
 #include "level_zero/core/source/driver/driver_imp.h"
 #include "level_zero/core/source/driver/host_pointer_manager.h"
+#include "level_zero/core/source/fabric/fabric.h"
 
 #include "driver_version_l0.h"
 
@@ -616,4 +617,28 @@ ze_result_t DriverHandleImp::checkMemoryAccessFromDevice(Device *device, const v
     return ZE_RESULT_ERROR_INVALID_ARGUMENT;
 }
 
+ze_result_t DriverHandleImp::fabricVertexGetExp(uint32_t *pCount, ze_fabric_vertex_handle_t *phVertices) {
+
+    uint32_t deviceCount = 0;
+    getDevice(&deviceCount, nullptr);
+
+    if (*pCount == 0) {
+        *pCount = deviceCount;
+        return ZE_RESULT_SUCCESS;
+    }
+
+    std::vector<ze_device_handle_t> deviceHandles;
+    deviceHandles.resize(deviceCount);
+    getDevice(&deviceCount, deviceHandles.data());
+
+    *pCount = std::min(deviceCount, *pCount);
+
+    for (uint32_t index = 0; index < *pCount; index++) {
+        auto deviceImp = static_cast<DeviceImp *>(deviceHandles[index]);
+        phVertices[index] = deviceImp->fabricVertex->toHandle();
+    }
+
+    return ZE_RESULT_SUCCESS;
+}
+
 } // namespace L0

level_zero/core/source/driver/driver_handle_imp.h

@@ -69,6 +69,7 @@ struct DriverHandleImp : public DriverHandle {
                                                NEO::SvmAllocationData *allocData,
                                                void *basePtr,
                                                uintptr_t *peerGpuAddress);
+    ze_result_t fabricVertexGetExp(uint32_t *pCount, ze_fabric_vertex_handle_t *phDevices) override;
     void createHostPointerManager();
     void sortNeoDevices(std::vector<std::unique_ptr<NEO::Device>> &neoDevices);
 

level_zero/core/source/fabric/fabric.cpp

@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/core/source/fabric/fabric.h"
+
+#include "shared/source/helpers/string.h"
+
+#include "level_zero/core/source/device/device_imp.h"
+
+namespace L0 {
+
+FabricVertex *FabricVertex::createFromDevice(Device *device) {
+
+    auto fabricVertex = new FabricVertex();
+    UNRECOVERABLE_IF(fabricVertex == nullptr);
+    ze_device_properties_t deviceProperties = {};
+    ze_pci_ext_properties_t pciProperties = {};
+
+    deviceProperties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
+    pciProperties.stype = ZE_STRUCTURE_TYPE_PCI_EXT_PROPERTIES;
+
+    device->getProperties(&deviceProperties);
+
+    fabricVertex->properties.stype = ZE_STRUCTURE_TYPE_FABRIC_VERTEX_EXP_PROPERTIES;
+    memcpy_s(fabricVertex->properties.uuid.id, ZE_MAX_UUID_SIZE, deviceProperties.uuid.id, ZE_MAX_DEVICE_UUID_SIZE);
+
+    if (deviceProperties.flags & ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE) {
+        fabricVertex->properties.type = ZE_FABRIC_VERTEX_EXP_TYPE_SUBEVICE;
+    } else {
+        fabricVertex->properties.type = ZE_FABRIC_VERTEX_EXP_TYPE_DEVICE;
+    }
+    fabricVertex->properties.remote = false;
+    fabricVertex->device = device;
+    if (device->getPciProperties(&pciProperties) == ZE_RESULT_SUCCESS) {
+        fabricVertex->properties.address.domain = pciProperties.address.domain;
+        fabricVertex->properties.address.bus = pciProperties.address.bus;
+        fabricVertex->properties.address.device = pciProperties.address.device;
+        fabricVertex->properties.address.function = pciProperties.address.function;
+    }
+
+    return fabricVertex;
+}
+
+ze_result_t FabricVertex::getSubVertices(uint32_t *pCount, ze_fabric_vertex_handle_t *phSubvertices) {
+
+    auto deviceImp = static_cast<DeviceImp *>(device);
+    if (*pCount == 0) {
+        *pCount = deviceImp->numSubDevices;
+        return ZE_RESULT_SUCCESS;
+    }
+
+    *pCount = std::min(deviceImp->numSubDevices, *pCount);
+    for (uint32_t index = 0; index < *pCount; index++) {
+        auto subDeviceImp = static_cast<DeviceImp *>(deviceImp->subDevices[index]);
+        phSubvertices[index] = subDeviceImp->fabricVertex->toHandle();
+    }
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t FabricVertex::getProperties(ze_fabric_vertex_exp_properties_t *pVertexProperties) {
+    *pVertexProperties = properties;
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t FabricVertex::getDevice(ze_device_handle_t *phDevice) {
+
+    *phDevice = device->toHandle();
+    return ZE_RESULT_SUCCESS;
+}
+
+} // namespace L0

level_zero/core/source/fabric/fabric.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include <level_zero/ze_api.h>
+
+struct _ze_fabric_vertex_handle_t {};
+
+namespace L0 {
+struct Device;
+
+struct FabricVertex : _ze_fabric_vertex_handle_t {
+
+  public:
+    static FabricVertex *createFromDevice(Device *device);
+    virtual ~FabricVertex() = default;
+    ze_result_t getSubVertices(uint32_t *pCount, ze_fabric_vertex_handle_t *phSubvertices);
+    ze_result_t getProperties(ze_fabric_vertex_exp_properties_t *pVertexProperties);
+    ze_result_t getDevice(ze_device_handle_t *phDevice);
+    static FabricVertex *fromHandle(ze_fabric_vertex_handle_t handle) { return static_cast<FabricVertex *>(handle); }
+    inline ze_fabric_vertex_handle_t toHandle() { return this; }
+
+  private:
+    struct Device *device = nullptr;
+    ze_fabric_vertex_exp_properties_t properties = {};
+};
+
+} // namespace L0

level_zero/core/test/unit_tests/sources/fabric/CMakeLists.txt

@@ -0,0 +1,10 @@
+#
+# Copyright (C) 2022 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+target_sources(${TARGET_NAME} PRIVATE
+               ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
+               ${CMAKE_CURRENT_SOURCE_DIR}/test_fabric.cpp
+)

level_zero/core/test/unit_tests/sources/fabric/test_fabric.cpp

@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/os_interface/device_factory.h"
+#include "shared/test/common/mocks/mock_driver_info.h"
+#include "shared/test/common/mocks/mock_driver_model.h"
+#include "shared/test/common/test_macros/test.h"
+
+#include "level_zero/core/source/fabric/fabric.h"
+#include "level_zero/core/test/unit_tests/fixtures/device_fixture.h"
+
+namespace L0 {
+namespace ult {
+
+using FabricVertexFixture = Test<MultiDeviceFixture>;
+
+TEST_F(FabricVertexFixture, WhenDevicesAreCreatedThenVerifyFabricVerticesAreCreated) {
+
+    uint32_t count = 0;
+    EXPECT_EQ(ZE_RESULT_SUCCESS, L0::zeFabricVertexGetExp(driverHandle->toHandle(), &count, nullptr));
+    EXPECT_EQ(count, numRootDevices);
+    std::vector<ze_fabric_vertex_handle_t> vertexHandles;
+    vertexHandles.resize(count);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, L0::zeFabricVertexGetExp(driverHandle->toHandle(), &count, vertexHandles.data()));
+    for (const auto &handle : vertexHandles) {
+        EXPECT_NE(handle, nullptr);
+    }
+}
+
+TEST_F(FabricVertexFixture, givenDevicesAreCreatedWhenzeFabricVertexGetSubVerticesExpIsCalledThenValidSubVerticesAreReturned) {
+
+    uint32_t count = 0;
+    L0::zeFabricVertexGetExp(driverHandle->toHandle(), &count, nullptr);
+    EXPECT_EQ(count, numRootDevices);
+    std::vector<ze_fabric_vertex_handle_t> vertexHandles;
+    vertexHandles.resize(count);
+    L0::zeFabricVertexGetExp(driverHandle->toHandle(), &count, vertexHandles.data());
+    for (const auto &handle : vertexHandles) {
+        uint32_t subVertexCount = 0;
+        EXPECT_EQ(ZE_RESULT_SUCCESS, L0::zeFabricVertexGetSubVerticesExp(handle, &subVertexCount, nullptr));
+        EXPECT_EQ(subVertexCount, numSubDevices);
+        std::vector<ze_fabric_vertex_handle_t> subVertexHandles;
+        subVertexHandles.resize(subVertexCount);
+        EXPECT_EQ(ZE_RESULT_SUCCESS, L0::zeFabricVertexGetSubVerticesExp(handle, &subVertexCount, subVertexHandles.data()));
+        for (const auto &handle : subVertexHandles) {
+            EXPECT_NE(handle, nullptr);
+        }
+    }
+}
+
+TEST_F(FabricVertexFixture, givenFabricVerticesAreCreatedWhenzeFabricVertexGetPropertiesExpIsCalledThenValidPropertiesAreReturned) {
+
+    // Setup pci address for all devices
+    for (auto l0Device : driverHandle->devices) {
+        auto deviceImp = static_cast<L0::DeviceImp *>(l0Device);
+        for (auto l0SubDevice : deviceImp->subDevices) {
+            auto subDeviceImp = static_cast<L0::DeviceImp *>(l0SubDevice);
+            NEO::DriverInfoMock *driverInfo = new DriverInfoMock();
+            driverInfo->setPciBusInfo({0, 1, 2, 3});
+            subDeviceImp->driverInfo.reset(driverInfo);
+            subDeviceImp->fabricVertex.reset(FabricVertex::createFromDevice(l0SubDevice));
+        }
+        NEO::DriverInfoMock *driverInfo = new DriverInfoMock();
+        driverInfo->setPciBusInfo({0, 1, 2, 3});
+        deviceImp->driverInfo.reset(driverInfo);
+        deviceImp->fabricVertex.reset(FabricVertex::createFromDevice(l0Device));
+    }
+
+    uint32_t count = 0;
+    std::vector<ze_fabric_vertex_handle_t> vertexHandles;
+    vertexHandles.resize(numRootDevices);
+    std::vector<ze_fabric_vertex_handle_t> subVertexHandles;
+    subVertexHandles.resize(numSubDevices);
+    L0::zeFabricVertexGetExp(driverHandle->toHandle(), &count, nullptr);
+    L0::zeFabricVertexGetExp(driverHandle->toHandle(), &count, vertexHandles.data());
+
+    // Verify vertex properties are same as device properties
+    auto verifyProperties = [](const ze_fabric_vertex_exp_properties_t &vertexProperties,
+                               const ze_device_properties_t &deviceProperties,
+                               const ze_pci_ext_properties_t &pciProperties) {
+        EXPECT_TRUE(0 == std::memcmp(vertexProperties.uuid.id, deviceProperties.uuid.id, sizeof(vertexProperties.uuid.id)));
+        if (deviceProperties.flags & ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE) {
+            EXPECT_EQ(vertexProperties.type, ZE_FABRIC_VERTEX_EXP_TYPE_SUBEVICE);
+        } else {
+            EXPECT_EQ(vertexProperties.type, ZE_FABRIC_VERTEX_EXP_TYPE_DEVICE);
+        }
+        EXPECT_FALSE(vertexProperties.remote);
+        EXPECT_EQ(vertexProperties.address.domain, pciProperties.address.domain);
+        EXPECT_EQ(vertexProperties.address.bus, pciProperties.address.bus);
+        EXPECT_EQ(vertexProperties.address.device, pciProperties.address.device);
+        EXPECT_EQ(vertexProperties.address.function, pciProperties.address.function);
+    };
+
+    for (const auto &handle : vertexHandles) {
+        uint32_t subVertexCount = 0;
+        ze_fabric_vertex_exp_properties_t vertexProperties = {};
+        ze_device_properties_t deviceProperties = {};
+        ze_pci_ext_properties_t pciProperties = {};
+        ze_device_handle_t hDevice = {};
+
+        vertexProperties.stype = ZE_STRUCTURE_TYPE_FABRIC_VERTEX_EXP_PROPERTIES;
+        deviceProperties.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
+        pciProperties.stype = ZE_STRUCTURE_TYPE_PCI_EXT_PROPERTIES;
+
+        L0::zeFabricVertexGetSubVerticesExp(handle, &subVertexCount, nullptr);
+        L0::zeFabricVertexGetSubVerticesExp(handle, &subVertexCount, subVertexHandles.data());
+
+        // Verify properties of sub vertices
+        for (const auto &subVertexHandle : subVertexHandles) {
+            EXPECT_EQ(ZE_RESULT_SUCCESS, L0::zeFabricVertexGetPropertiesExp(subVertexHandle, &vertexProperties));
+            L0::zeFabricVertexGetDeviceExp(subVertexHandle, &hDevice);
+            EXPECT_NE(hDevice, nullptr);
+            EXPECT_EQ(ZE_RESULT_SUCCESS, zeDeviceGetProperties(hDevice, &deviceProperties));
+            EXPECT_EQ(ZE_RESULT_SUCCESS, zeDevicePciGetPropertiesExt(hDevice, &pciProperties));
+            verifyProperties(vertexProperties, deviceProperties, pciProperties);
+        }
+
+        // Verify properties of Vertices
+        EXPECT_EQ(ZE_RESULT_SUCCESS, L0::zeFabricVertexGetPropertiesExp(handle, &vertexProperties));
+        L0::zeFabricVertexGetDeviceExp(handle, &hDevice);
+        EXPECT_NE(hDevice, nullptr);
+        EXPECT_EQ(ZE_RESULT_SUCCESS, zeDeviceGetProperties(hDevice, &deviceProperties));
+        EXPECT_EQ(ZE_RESULT_SUCCESS, zeDevicePciGetPropertiesExt(hDevice, &pciProperties));
+        verifyProperties(vertexProperties, deviceProperties, pciProperties);
+    }
+}
+
+} // namespace ult
+} // namespace L0