diff --git a/level_zero/core/source/CMakeLists.txt b/level_zero/core/source/CMakeLists.txt index d08b7ffcac..18d00fa358 100644 --- a/level_zero/core/source/CMakeLists.txt +++ b/level_zero/core/source/CMakeLists.txt @@ -53,8 +53,6 @@ set(L0_RUNTIME_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/event/event.h ${CMAKE_CURRENT_SOURCE_DIR}/fence/fence.cpp ${CMAKE_CURRENT_SOURCE_DIR}/fence/fence.h - ${CMAKE_CURRENT_SOURCE_DIR}/fabric/fabric.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/fabric/fabric.h ${CMAKE_CURRENT_SOURCE_DIR}/helpers/allocation_extensions.h ${CMAKE_CURRENT_SOURCE_DIR}/helpers/allocation_extensions.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/api_specific_config_l0.cpp diff --git a/level_zero/core/source/device/device_imp.cpp b/level_zero/core/source/device/device_imp.cpp index 23c1f82a9a..007d203bab 100644 --- a/level_zero/core/source/device/device_imp.cpp +++ b/level_zero/core/source/device/device_imp.cpp @@ -1099,8 +1099,6 @@ Device *Device::create(DriverHandle *driverHandle, NEO::Device *neoDevice, bool device->populateSubDeviceCopyEngineGroups(); - device->fabricVertex = std::unique_ptr(FabricVertex::createFromDevice(device)); - return device; } diff --git a/level_zero/core/source/device/device_imp.h b/level_zero/core/source/device/device_imp.h index d150ce84bf..26725c54a8 100644 --- a/level_zero/core/source/device/device_imp.h +++ b/level_zero/core/source/device/device_imp.h @@ -132,10 +132,11 @@ struct DeviceImp : public Device { void createSysmanHandle(bool isSubDevice); void populateSubDeviceCopyEngineGroups(); bool isQueueGroupOrdinalValid(uint32_t ordinal); + void setFabricVertex(FabricVertex *inFabricVertex) { fabricVertex = inFabricVertex; } using CmdListCreateFunPtrT = L0::CommandList *(*)(uint32_t, Device *, NEO::EngineGroupType, ze_command_list_flags_t, ze_result_t &); CmdListCreateFunPtrT getCmdListCreateFunc(const ze_command_list_desc_t *desc); - std::unique_ptr fabricVertex; + FabricVertex *fabricVertex = nullptr; ze_result_t queryDeviceLuid(ze_device_luid_ext_properties_t *deviceLuidProperties); ze_result_t setDeviceLuid(ze_device_luid_ext_properties_t *deviceLuidProperties); diff --git a/level_zero/core/source/driver/driver_handle_imp.cpp b/level_zero/core/source/driver/driver_handle_imp.cpp index 5d465f9240..d8d2c86fa8 100644 --- a/level_zero/core/source/driver/driver_handle_imp.cpp +++ b/level_zero/core/source/driver/driver_handle_imp.cpp @@ -159,6 +159,12 @@ DriverHandleImp::~DriverHandleImp() { for (auto &device : this->devices) { delete device; } + + for (auto &fabricVertex : this->fabricVertices) { + delete fabricVertex; + } + this->fabricVertices.clear(); + if (this->svmAllocsManager) { delete this->svmAllocsManager; this->svmAllocsManager = nullptr; @@ -232,6 +238,14 @@ ze_result_t DriverHandleImp::initialize(std::vector createHostPointerManager(); } + for (auto &device : this->devices) { + + auto deviceImpl = static_cast(device); + auto fabricVertex = FabricVertex::createFromDevice(device); + deviceImpl->setFabricVertex(fabricVertex); + this->fabricVertices.push_back(fabricVertex); + } + return ZE_RESULT_SUCCESS; } @@ -621,23 +635,16 @@ ze_result_t DriverHandleImp::checkMemoryAccessFromDevice(Device *device, const v ze_result_t DriverHandleImp::fabricVertexGetExp(uint32_t *pCount, ze_fabric_vertex_handle_t *phVertices) { - uint32_t deviceCount = 0; - getDevice(&deviceCount, nullptr); - + uint32_t fabricVertexCount = static_cast(this->fabricVertices.size()); if (*pCount == 0) { - *pCount = deviceCount; + *pCount = fabricVertexCount; return ZE_RESULT_SUCCESS; } - std::vector deviceHandles; - deviceHandles.resize(deviceCount); - getDevice(&deviceCount, deviceHandles.data()); - - *pCount = std::min(deviceCount, *pCount); + *pCount = std::min(fabricVertexCount, *pCount); for (uint32_t index = 0; index < *pCount; index++) { - auto deviceImp = static_cast(deviceHandles[index]); - phVertices[index] = deviceImp->fabricVertex->toHandle(); + phVertices[index] = this->fabricVertices[index]->toHandle(); } return ZE_RESULT_SUCCESS; diff --git a/level_zero/core/source/driver/driver_handle_imp.h b/level_zero/core/source/driver/driver_handle_imp.h index 14751ccf82..1089d55ae9 100644 --- a/level_zero/core/source/driver/driver_handle_imp.h +++ b/level_zero/core/source/driver/driver_handle_imp.h @@ -18,6 +18,7 @@ namespace L0 { class HostPointerManager; +struct FabricVertex; struct DriverHandleImp : public DriverHandle { ~DriverHandleImp() override; @@ -86,6 +87,7 @@ struct DriverHandleImp : public DriverHandle { std::map sharedMakeResidentAllocations; std::vector devices; + std::vector fabricVertices; // Spec extensions const std::vector> extensionsSupported = { {ZE_FLOAT_ATOMICS_EXT_NAME, ZE_FLOAT_ATOMICS_EXT_VERSION_CURRENT}, diff --git a/level_zero/core/source/fabric/CMakeLists.txt b/level_zero/core/source/fabric/CMakeLists.txt new file mode 100644 index 0000000000..598ee6e512 --- /dev/null +++ b/level_zero/core/source/fabric/CMakeLists.txt @@ -0,0 +1,20 @@ +# +# Copyright (C) 2022 Intel Corporation +# +# SPDX-License-Identifier: MIT +# + +set(L0_SRCS_FABRIC + ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt + ${CMAKE_CURRENT_SOURCE_DIR}/fabric.h + ${CMAKE_CURRENT_SOURCE_DIR}/fabric_device_interface.h + ${CMAKE_CURRENT_SOURCE_DIR}/fabric.cpp +) + +add_subdirectories() +set(L0_RUNTIME_SOURCES + ${L0_RUNTIME_SOURCES} + ${L0_SRCS_FABRIC} + ${L0_SRCS_FABRIC_OS} + PARENT_SCOPE +) diff --git a/level_zero/core/source/fabric/fabric.cpp b/level_zero/core/source/fabric/fabric.cpp index c3753859ef..47a6202eb5 100644 --- a/level_zero/core/source/fabric/fabric.cpp +++ b/level_zero/core/source/fabric/fabric.cpp @@ -10,13 +10,39 @@ #include "shared/source/helpers/string.h" #include "level_zero/core/source/device/device_imp.h" +#include "level_zero/core/source/fabric/fabric_device_interface.h" namespace L0 { +FabricVertex::~FabricVertex() { + + for (auto subVertex : subVertices) { + delete subVertex; + } + subVertices.clear(); +} + FabricVertex *FabricVertex::createFromDevice(Device *device) { + // Fabric Vertices are not created for engine instanced devices + if (device->getNEODevice()->isEngineInstanced()) { + return nullptr; + } + auto fabricVertex = new FabricVertex(); UNRECOVERABLE_IF(fabricVertex == nullptr); + + auto deviceImpl = static_cast(device); + for (auto &subDevice : deviceImpl->subDevices) { + auto subVertex = FabricVertex::createFromDevice(subDevice); + if (subVertex == nullptr) { + continue; + } + auto subDeviceImpl = static_cast(subDevice); + subDeviceImpl->setFabricVertex(subVertex); + fabricVertex->subVertices.push_back(subVertex); + } + ze_device_properties_t deviceProperties = {}; ze_pci_ext_properties_t pciProperties = {}; @@ -42,31 +68,36 @@ FabricVertex *FabricVertex::createFromDevice(Device *device) { fabricVertex->properties.address.function = pciProperties.address.function; } + fabricVertex->pFabricDeviceInterface = FabricDeviceInterface::createFabricDeviceInterface(*fabricVertex); + UNRECOVERABLE_IF(fabricVertex->pFabricDeviceInterface == nullptr); + + fabricVertex->pFabricDeviceInterface->enumerate(); + return fabricVertex; } ze_result_t FabricVertex::getSubVertices(uint32_t *pCount, ze_fabric_vertex_handle_t *phSubvertices) { - auto deviceImp = static_cast(device); + uint32_t subVertexCount = static_cast(subVertices.size()); if (*pCount == 0) { - *pCount = deviceImp->numSubDevices; + *pCount = subVertexCount; return ZE_RESULT_SUCCESS; } - *pCount = std::min(deviceImp->numSubDevices, *pCount); + *pCount = std::min(subVertexCount, *pCount); for (uint32_t index = 0; index < *pCount; index++) { - auto subDeviceImp = static_cast(deviceImp->subDevices[index]); - phSubvertices[index] = subDeviceImp->fabricVertex->toHandle(); + phSubvertices[index] = subVertices[index]->toHandle(); } + return ZE_RESULT_SUCCESS; } -ze_result_t FabricVertex::getProperties(ze_fabric_vertex_exp_properties_t *pVertexProperties) { +ze_result_t FabricVertex::getProperties(ze_fabric_vertex_exp_properties_t *pVertexProperties) const { *pVertexProperties = properties; return ZE_RESULT_SUCCESS; } -ze_result_t FabricVertex::getDevice(ze_device_handle_t *phDevice) { +ze_result_t FabricVertex::getDevice(ze_device_handle_t *phDevice) const { *phDevice = device->toHandle(); return ZE_RESULT_SUCCESS; diff --git a/level_zero/core/source/fabric/fabric.h b/level_zero/core/source/fabric/fabric.h index 7b0a74a2d8..6931500d04 100644 --- a/level_zero/core/source/fabric/fabric.h +++ b/level_zero/core/source/fabric/fabric.h @@ -9,25 +9,33 @@ #include +#include +#include + struct _ze_fabric_vertex_handle_t {}; namespace L0 { struct Device; +class FabricDeviceInterface; struct FabricVertex : _ze_fabric_vertex_handle_t { public: static FabricVertex *createFromDevice(Device *device); - virtual ~FabricVertex() = default; + virtual ~FabricVertex(); ze_result_t getSubVertices(uint32_t *pCount, ze_fabric_vertex_handle_t *phSubvertices); - ze_result_t getProperties(ze_fabric_vertex_exp_properties_t *pVertexProperties); - ze_result_t getDevice(ze_device_handle_t *phDevice); + ze_result_t getProperties(ze_fabric_vertex_exp_properties_t *pVertexProperties) const; + ze_result_t getDevice(ze_device_handle_t *phDevice) const; static FabricVertex *fromHandle(ze_fabric_vertex_handle_t handle) { return static_cast(handle); } inline ze_fabric_vertex_handle_t toHandle() { return this; } + std::unique_ptr pFabricDeviceInterface; + private: struct Device *device = nullptr; ze_fabric_vertex_exp_properties_t properties = {}; + + std::vector subVertices = {}; }; } // namespace L0 diff --git a/level_zero/core/source/fabric/fabric_device_interface.h b/level_zero/core/source/fabric/fabric_device_interface.h new file mode 100644 index 0000000000..fa0a36a9b6 --- /dev/null +++ b/level_zero/core/source/fabric/fabric_device_interface.h @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "level_zero/core/source/fabric/fabric.h" +#include + +#include + +namespace L0 { + +class FabricDeviceInterface { + + public: + virtual ~FabricDeviceInterface(){}; + virtual ze_result_t enumerate() = 0; + static std::unique_ptr createFabricDeviceInterface(const FabricVertex &fabricVertex); +}; + +} // namespace L0 diff --git a/level_zero/core/source/fabric/linux/CMakeLists.txt b/level_zero/core/source/fabric/linux/CMakeLists.txt new file mode 100644 index 0000000000..fb3b2ced95 --- /dev/null +++ b/level_zero/core/source/fabric/linux/CMakeLists.txt @@ -0,0 +1,25 @@ +# +# Copyright (C) 2022 Intel Corporation +# +# SPDX-License-Identifier: MIT +# + +if(UNIX) + + if(LIBGENL_FOUND AND NEO_ENABLE_i915_PRELIM_DETECTION) + set(L0_SRCS_FABRIC_OS + ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt + ${CMAKE_CURRENT_SOURCE_DIR}/fabric_device_iaf.h + ${CMAKE_CURRENT_SOURCE_DIR}/fabric_device_iaf.cpp + PARENT_SCOPE + ) + else() + set(L0_SRCS_FABRIC_OS + ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt + ${CMAKE_CURRENT_SOURCE_DIR}/fabric_device_iaf_stub.h + ${CMAKE_CURRENT_SOURCE_DIR}/fabric_device_iaf_stub.cpp + PARENT_SCOPE + ) + endif() + +endif() diff --git a/level_zero/core/source/fabric/linux/fabric_device_iaf.cpp b/level_zero/core/source/fabric/linux/fabric_device_iaf.cpp new file mode 100644 index 0000000000..a13c151640 --- /dev/null +++ b/level_zero/core/source/fabric/linux/fabric_device_iaf.cpp @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "level_zero/core/source/fabric/linux/fabric_device_iaf.h" + +#include "shared/source/debug_settings/debug_settings_manager.h" +#include "shared/source/os_interface/linux/drm_neo.h" +#include "shared/source/os_interface/linux/pci_path.h" +#include "shared/source/os_interface/os_interface.h" + +#include "level_zero/core/source/device/device.h" +#include "level_zero/core/source/device/device_imp.h" +#include "level_zero/core/source/fabric/fabric.h" + +namespace L0 { + +FabricDeviceIaf::FabricDeviceIaf(Device *device) : device(device) { + + DeviceImp *deviceImp = static_cast(device); + + if (deviceImp->numSubDevices == 0) { + //Add one sub-device + subDeviceIafs.push_back(std::make_unique(device)); + } else { + for (const auto &subDevice : deviceImp->subDevices) { + subDeviceIafs.push_back(std::make_unique(subDevice)); + } + } +} + +ze_result_t FabricDeviceIaf::enumerate() { + + ze_result_t result = ZE_RESULT_SUCCESS; + for (auto &subDeviceIaf : subDeviceIafs) { + result = subDeviceIaf->enumerate(); + if (result != ZE_RESULT_SUCCESS) { + break; + } + } + + return result; +} + +FabricSubDeviceIaf::FabricSubDeviceIaf(Device *device) : device(device) { + pIafNlApi = std::make_unique(); + UNRECOVERABLE_IF(nullptr == pIafNlApi); +} + +ze_result_t FabricSubDeviceIaf::enumerate() { + + auto &osInterface = device->getNEODevice()->getRootDeviceEnvironment().osInterface; + + if (osInterface == nullptr) { + return ZE_RESULT_ERROR_UNINITIALIZED; + } + + if (osInterface->getDriverModel()->getDriverModelType() != NEO::DriverModelType::DRM) { + return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + auto pDrm = osInterface->getDriverModel()->as(); + std::optional rootPciPath = NEO::getPciLinkPath(pDrm->getFileDescriptor()); + if (!rootPciPath.has_value()) { + return ZE_RESULT_ERROR_UNKNOWN; + } + + std::vector iafPorts = {}; + ze_result_t result = pIafNlApi->getPorts("/sys/class/drm/" + rootPciPath.value() + "/device/", iafPorts); + if (ZE_RESULT_SUCCESS != result) { + return result; + } + + DeviceImp *deviceImp = static_cast(device); + uint32_t physicalSubDeviceId = deviceImp->getPhysicalSubDeviceId(); + + // Remove ports which donot belong to this device + for (auto iter = iafPorts.begin(); iter != iafPorts.end();) { + IafPort &port = *iter; + if (port.portId.attachId != physicalSubDeviceId) { + iter = iafPorts.erase(iter); + continue; + } + ++iter; + } + + // Get Connections + for (auto iafPort : iafPorts) { + FabricPortConnection connection = {}; + ze_result_t result = getConnection(iafPort, connection); + if (result != ZE_RESULT_SUCCESS) { + NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, + "failure observed for IafPort{0x%x, 0x%x, 0x%x}: 0x%x\n", + iafPort.portId.fabricId, iafPort.portId.attachId, iafPort.portId.portNumber, + result); + continue; + } + connections.push_back(connection); + } + + // Get guid + if (iafPorts.size() > 0) { + std::vector ports = {}; + if (ZE_RESULT_SUCCESS != pIafNlApi->subdevicePropertiesGet(iafPorts[0].portId.fabricId, physicalSubDeviceId, guid, ports)) { + connections.clear(); + NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, + "failure during fabric port guid reading {0x%x, 0x%x, 0x%x}: result: 0x%x subdeviceId:%d\n", + iafPorts[0].portId.fabricId, iafPorts[0].portId.attachId, iafPorts[0].portId.portNumber, + result, physicalSubDeviceId); + return ZE_RESULT_ERROR_UNKNOWN; + } + } + + return ZE_RESULT_SUCCESS; +} + +ze_result_t FabricSubDeviceIaf::getConnection(IafPort &port, FabricPortConnection &connection) { + + IafPortState iafPortState = {}; + ze_result_t result = pIafNlApi->fPortStatusQuery(port.portId, iafPortState); + if (ZE_RESULT_SUCCESS != result || iafPortState.healthStatus != IAF_FPORT_HEALTH_HEALTHY) { + return ZE_RESULT_ERROR_UNKNOWN; + } + + // Consider only healthy ports + uint64_t neighbourGuid; + uint8_t neighbourPortNumber; + IafPortSpeed maxRxSpeed = {}, maxTxSpeed = {}; + IafPortSpeed rxSpeed = {}, txSpeed = {}; + + result = pIafNlApi->fportProperties(port.portId, neighbourGuid, neighbourPortNumber, maxRxSpeed, maxTxSpeed, rxSpeed, txSpeed); + if (ZE_RESULT_SUCCESS != result) { + return result; + } + + connection.currentid = port.portId; + connection.neighbourPortNumber = neighbourPortNumber; + connection.neighbourGuid = neighbourGuid; + const double bandwidthInBitsPerSecond = static_cast(maxRxSpeed.bitRate * maxRxSpeed.width); + // 8ull - bits to bytes; 1e9 - seconds to nano-seconds + connection.bandwidthInBytesPerNanoSecond = static_cast(bandwidthInBitsPerSecond / (8ull * 1e9)); + connection.isDuplex = true; + return ZE_RESULT_SUCCESS; +} + +void FabricSubDeviceIaf::setIafNlApi(IafNlApi *iafNlApi) { + pIafNlApi.reset(iafNlApi); +} + +std::unique_ptr FabricDeviceInterface::createFabricDeviceInterface(const FabricVertex &fabricVertex) { + + ze_fabric_vertex_exp_properties_t vertexProperties = {}; + fabricVertex.getProperties(&vertexProperties); + + ze_device_handle_t hDevice = nullptr; + fabricVertex.getDevice(&hDevice); + DEBUG_BREAK_IF(hDevice == nullptr); + + DeviceImp *deviceImp = static_cast(hDevice); + Device *device = static_cast(hDevice); + if (deviceImp->isSubdevice) { + return std::unique_ptr(new (std::nothrow) FabricSubDeviceIaf(device)); + } + + return std::unique_ptr(new (std::nothrow) FabricDeviceIaf(device)); +} + +} // namespace L0 diff --git a/level_zero/core/source/fabric/linux/fabric_device_iaf.h b/level_zero/core/source/fabric/linux/fabric_device_iaf.h new file mode 100644 index 0000000000..1ed5a5ad64 --- /dev/null +++ b/level_zero/core/source/fabric/linux/fabric_device_iaf.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "level_zero/core/source/fabric/fabric_device_interface.h" +#include "level_zero/tools/source/sysman/linux/nl_api/iaf_nl_api.h" +#include + +#include + +namespace L0 { + +class FabricSubDeviceIaf; + +struct FabricPortConnection { + IafPortId currentid; + uint8_t neighbourPortNumber; + uint64_t neighbourGuid; + uint32_t bandwidthInBytesPerNanoSecond; + bool isDuplex; +}; + +class FabricDeviceIaf : public FabricDeviceInterface { + + public: + FabricDeviceIaf(Device *device); + ~FabricDeviceIaf() override = default; + ze_result_t enumerate() override; + std::vector> subDeviceIafs = {}; + + protected: + Device *device = nullptr; +}; + +class FabricSubDeviceIaf : public FabricDeviceInterface { + + public: + FabricSubDeviceIaf(Device *device); + ~FabricSubDeviceIaf() override = default; + ze_result_t enumerate() override; + void setIafNlApi(IafNlApi *iafNlApi); + std::vector connections = {}; + + protected: + ze_result_t getConnection(IafPort &port, FabricPortConnection &connection); + + std::unique_ptr pIafNlApi = nullptr; + Device *device = nullptr; + uint64_t guid = 0ull; +}; + +} // namespace L0 \ No newline at end of file diff --git a/level_zero/core/source/fabric/linux/fabric_device_iaf_stub.cpp b/level_zero/core/source/fabric/linux/fabric_device_iaf_stub.cpp new file mode 100644 index 0000000000..d47ff04d24 --- /dev/null +++ b/level_zero/core/source/fabric/linux/fabric_device_iaf_stub.cpp @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "level_zero/core/source/fabric/linux/fabric_device_iaf_stub.h" + +#include "level_zero/core/source/fabric/fabric.h" + +namespace L0 { + +std::unique_ptr FabricDeviceInterface::createFabricDeviceInterface(const FabricVertex &fabricVertex) { + + return std::unique_ptr(new (std::nothrow) FabricDeviceIafStub()); +} + +} // namespace L0 diff --git a/level_zero/core/source/fabric/linux/fabric_device_iaf_stub.h b/level_zero/core/source/fabric/linux/fabric_device_iaf_stub.h new file mode 100644 index 0000000000..54f7e52b95 --- /dev/null +++ b/level_zero/core/source/fabric/linux/fabric_device_iaf_stub.h @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "level_zero/core/source/fabric/fabric_device_interface.h" +#include + +namespace L0 { + +class FabricDeviceIafStub : public FabricDeviceInterface { + + public: + FabricDeviceIafStub() = default; + ~FabricDeviceIafStub() override = default; + ze_result_t enumerate() override { + return ZE_RESULT_SUCCESS; + } +}; + +} // namespace L0 \ No newline at end of file diff --git a/level_zero/core/source/fabric/windows/CMakeLists.txt b/level_zero/core/source/fabric/windows/CMakeLists.txt new file mode 100644 index 0000000000..4ff65a0be1 --- /dev/null +++ b/level_zero/core/source/fabric/windows/CMakeLists.txt @@ -0,0 +1,14 @@ +# +# Copyright (C) 2022 Intel Corporation +# +# SPDX-License-Identifier: MIT +# + +if(WIN32) + set(L0_SRCS_FABRIC_OS + ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt + ${CMAKE_CURRENT_SOURCE_DIR}/fabric_device_iaf.h + ${CMAKE_CURRENT_SOURCE_DIR}/fabric_device_iaf.cpp + PARENT_SCOPE + ) +endif() \ No newline at end of file diff --git a/level_zero/core/source/fabric/windows/fabric_device_iaf.cpp b/level_zero/core/source/fabric/windows/fabric_device_iaf.cpp new file mode 100644 index 0000000000..95faeb24d5 --- /dev/null +++ b/level_zero/core/source/fabric/windows/fabric_device_iaf.cpp @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "level_zero/core/source/fabric/windows/fabric_device_iaf.h" + +#include "level_zero/core/source/fabric/fabric.h" + +namespace L0 { + +std::unique_ptr FabricDeviceInterface::createFabricDeviceInterface(const FabricVertex &fabricVertex) { + + return std::unique_ptr(new (std::nothrow) FabricDeviceIaf()); +} + +} // namespace L0 diff --git a/level_zero/core/source/fabric/windows/fabric_device_iaf.h b/level_zero/core/source/fabric/windows/fabric_device_iaf.h new file mode 100644 index 0000000000..9c937c8ced --- /dev/null +++ b/level_zero/core/source/fabric/windows/fabric_device_iaf.h @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "level_zero/core/source/fabric/fabric_device_interface.h" +#include + +namespace L0 { + +class FabricDeviceIaf : public FabricDeviceInterface { + + public: + FabricDeviceIaf() = default; + ~FabricDeviceIaf() override = default; + ze_result_t enumerate() override { + return ZE_RESULT_SUCCESS; + } +}; + +} // namespace L0 \ No newline at end of file diff --git a/level_zero/core/test/unit_tests/fixtures/device_fixture.h b/level_zero/core/test/unit_tests/fixtures/device_fixture.h index bc32af4cf3..df65c6d47c 100644 --- a/level_zero/core/test/unit_tests/fixtures/device_fixture.h +++ b/level_zero/core/test/unit_tests/fixtures/device_fixture.h @@ -99,7 +99,6 @@ struct MultiDeviceFixture { DebugManagerStateRestore restorer; std::unique_ptr> driverHandle; - std::vector devices; uint32_t numRootDevices = 4u; uint32_t numSubDevices = 2u; L0::ContextImp *context = nullptr; diff --git a/level_zero/core/test/unit_tests/sources/fabric/CMakeLists.txt b/level_zero/core/test/unit_tests/sources/fabric/CMakeLists.txt index 744f556b47..2ebefb45f0 100644 --- a/level_zero/core/test/unit_tests/sources/fabric/CMakeLists.txt +++ b/level_zero/core/test/unit_tests/sources/fabric/CMakeLists.txt @@ -8,3 +8,4 @@ target_sources(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt ${CMAKE_CURRENT_SOURCE_DIR}/test_fabric.cpp ) +add_subdirectories() \ No newline at end of file diff --git a/level_zero/core/test/unit_tests/sources/fabric/linux/CMakeLists.txt b/level_zero/core/test/unit_tests/sources/fabric/linux/CMakeLists.txt new file mode 100644 index 0000000000..4370e39fcc --- /dev/null +++ b/level_zero/core/test/unit_tests/sources/fabric/linux/CMakeLists.txt @@ -0,0 +1,14 @@ +# +# Copyright (C) 2022 Intel Corporation +# +# SPDX-License-Identifier: MIT +# + +if(UNIX) + if(LIBGENL_FOUND) + target_sources(${TARGET_NAME} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt + ${CMAKE_CURRENT_SOURCE_DIR}/test_fabric_iaf.cpp + ) + endif() +endif() \ No newline at end of file diff --git a/level_zero/core/test/unit_tests/sources/fabric/linux/test_fabric_iaf.cpp b/level_zero/core/test/unit_tests/sources/fabric/linux/test_fabric_iaf.cpp new file mode 100644 index 0000000000..52323d5d6e --- /dev/null +++ b/level_zero/core/test/unit_tests/sources/fabric/linux/test_fabric_iaf.cpp @@ -0,0 +1,231 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/test/common/helpers/debug_manager_state_restore.h" +#include "shared/test/common/libult/linux/drm_mock.h" + +#include "level_zero/core/source/fabric/linux/fabric_device_iaf.h" +#include "level_zero/core/test/unit_tests/fixtures/device_fixture.h" + +#include "gtest/gtest.h" + +namespace L0 { +namespace ult { + +class MockIafNlApi : public IafNlApi { + public: + ze_result_t fPortStatusQueryStatus = ZE_RESULT_SUCCESS; + ze_result_t fportPropertiesStatus = ZE_RESULT_SUCCESS; + ze_result_t subDevicePropertiesStatus = ZE_RESULT_SUCCESS; + ze_result_t getPortsStatus = ZE_RESULT_SUCCESS; + iaf_fport_health fPortStatusQueryHealthStatus = IAF_FPORT_HEALTH_HEALTHY; + bool portEnumerationEnable = true; + + ze_result_t handleResponse(const uint16_t cmdOp, struct genl_info *info, void *pOutput) override { return ZE_RESULT_SUCCESS; }; + ze_result_t fPortStatusQuery(const IafPortId portId, IafPortState &state) override { + state.healthStatus = fPortStatusQueryHealthStatus; + return fPortStatusQueryStatus; + }; + + ze_result_t getThroughput(const IafPortId portId, IafPortThroughPut &throughput) override { return ZE_RESULT_SUCCESS; }; + ze_result_t portStateQuery(const IafPortId portId, bool &enabled) override { return ZE_RESULT_SUCCESS; }; + ze_result_t portBeaconStateQuery(const IafPortId portId, bool &enabled) override { return ZE_RESULT_SUCCESS; }; + ze_result_t portBeaconEnable(const IafPortId portId) override { return ZE_RESULT_SUCCESS; }; + ze_result_t portBeaconDisable(const IafPortId portId) override { return ZE_RESULT_SUCCESS; }; + ze_result_t portEnable(const IafPortId portId) override { return ZE_RESULT_SUCCESS; }; + ze_result_t portDisable(const IafPortId portId) override { return ZE_RESULT_SUCCESS; }; + ze_result_t portUsageEnable(const IafPortId portId) override { return ZE_RESULT_SUCCESS; }; + ze_result_t portUsageDisable(const IafPortId portId) override { return ZE_RESULT_SUCCESS; }; + ze_result_t remRequest() override { return ZE_RESULT_SUCCESS; }; + ze_result_t routingGenQuery(uint32_t &start, uint32_t &end) override { return ZE_RESULT_SUCCESS; }; + ze_result_t deviceEnum(std::vector &fabricIds) override { return ZE_RESULT_SUCCESS; }; + ze_result_t fabricDeviceProperties(const uint32_t fabricId, uint32_t &numSubdevices) override { return ZE_RESULT_SUCCESS; }; + ze_result_t subdevicePropertiesGet(const uint32_t fabricId, const uint32_t attachId, uint64_t &guid, std::vector &ports) override { + return subDevicePropertiesStatus; + }; + ze_result_t fportProperties(const IafPortId portId, uint64_t &neighborGuid, uint8_t &neighborPortNumber, + IafPortSpeed &maxRxSpeed, IafPortSpeed &maxTxSpeed, + IafPortSpeed &rxSpeed, IafPortSpeed &txSpeed) override { + neighborPortNumber = 8; + neighborGuid = 0xFEEDBEAD; + maxRxSpeed.width = 4; + maxRxSpeed.bitRate = 53125000000; + maxTxSpeed = maxRxSpeed; + rxSpeed = maxRxSpeed; + txSpeed = maxRxSpeed; + + return fportPropertiesStatus; + }; + ze_result_t getPorts(const std::string &devicePciPath, std::vector &ports) override { + + if (portEnumerationEnable) { + IafPort defaultPort; + defaultPort.onSubdevice = true; + defaultPort.portId.fabricId = testPortId.fabricId; + defaultPort.portId.attachId = testPortId.attachId; + defaultPort.portId.portNumber = testPortId.portNumber; + defaultPort.model = "XeLink"; + defaultPort.maxRxSpeed.width = 4; + defaultPort.maxRxSpeed.bitRate = 53125000000; + defaultPort.maxTxSpeed = defaultPort.maxRxSpeed; + + ports.push_back(defaultPort); + defaultPort.portId.fabricId = testPortId.fabricId + 1; + defaultPort.portId.attachId = testPortId.attachId + 1; + defaultPort.portId.portNumber = testPortId.portNumber + 1; + ports.push_back(defaultPort); + defaultPort.portId.fabricId = testPortId.fabricId + 2; + defaultPort.portId.attachId = testPortId.attachId; + defaultPort.portId.portNumber = testPortId.portNumber + 2; + ports.push_back(defaultPort); + } + + return getPortsStatus; + }; + + IafPortId testPortId = {}; +}; + +struct TestFabricIaf : public ::testing::Test { + void SetUp() override { + DebugManager.flags.CreateMultipleSubDevices.set(1); + executionEnvironment = new NEO::ExecutionEnvironment(); + executionEnvironment->prepareRootDeviceEnvironments(1); + executionEnvironment->rootDeviceEnvironments[0]->setHwInfo(defaultHwInfo.get()); + osInterface = new OSInterface(); + drmMock = new DrmMockResources(*executionEnvironment->rootDeviceEnvironments[0]); + executionEnvironment->rootDeviceEnvironments[0]->osInterface.reset(osInterface); + executionEnvironment->rootDeviceEnvironments[0]->osInterface->setDriverModel(std::unique_ptr(drmMock)); + neoDevice = NEO::MockDevice::create(executionEnvironment, 0u); + } + void TearDown() override {} + + DebugManagerStateRestore restorer; + + NEO::ExecutionEnvironment *executionEnvironment = nullptr; + OSInterface *osInterface = nullptr; + NEO::MockDevice *neoDevice = nullptr; + DrmMockResources *drmMock = nullptr; + L0::DriverHandleImp *driverHandle = nullptr; +}; + +TEST_F(TestFabricIaf, GivenIafFabricAvailableWhenFabricVerticesAreCreatedThenEnumerationIsSuccessful) { + + NEO::DeviceVector devices; + devices.push_back(std::unique_ptr(neoDevice)); + std::unique_ptr> driverHandle = std::make_unique>(); + driverHandle->initialize(std::move(devices)); + + auto deviceImp = static_cast(driverHandle->devices[0]); + + FabricSubDeviceIaf *subDeviceFabric = new FabricSubDeviceIaf(deviceImp); + subDeviceFabric->setIafNlApi(new MockIafNlApi()); + EXPECT_EQ(ZE_RESULT_SUCCESS, subDeviceFabric->enumerate()); + delete subDeviceFabric; + + FabricDeviceIaf *deviceFabric = new FabricDeviceIaf(deviceImp); + deviceFabric->subDeviceIafs[0]->setIafNlApi(new MockIafNlApi()); + EXPECT_EQ(ZE_RESULT_SUCCESS, deviceFabric->enumerate()); + delete deviceFabric; +} + +TEST_F(TestFabricIaf, GivenIafFabricAvailableWhenPortStatusQueryIsUnsuccessfulThenNoConnectionsAreEnumerated) { + + NEO::DeviceVector devices; + devices.push_back(std::unique_ptr(neoDevice)); + std::unique_ptr> driverHandle = std::make_unique>(); + driverHandle->initialize(std::move(devices)); + + auto deviceImp = static_cast(driverHandle->devices[0]); + + FabricSubDeviceIaf *subDeviceFabric = new FabricSubDeviceIaf(deviceImp); + MockIafNlApi *mockNlApi = new MockIafNlApi(); + mockNlApi->fPortStatusQueryStatus = ZE_RESULT_ERROR_UNKNOWN; + subDeviceFabric->setIafNlApi(mockNlApi); + subDeviceFabric->enumerate(); + EXPECT_EQ(subDeviceFabric->connections.size(), 0u); + delete subDeviceFabric; + + subDeviceFabric = new FabricSubDeviceIaf(deviceImp); + mockNlApi = new MockIafNlApi(); + mockNlApi->fPortStatusQueryStatus = ZE_RESULT_SUCCESS; + mockNlApi->fPortStatusQueryHealthStatus = IAF_FPORT_HEALTH_DEGRADED; + subDeviceFabric->setIafNlApi(mockNlApi); + subDeviceFabric->enumerate(); + EXPECT_EQ(subDeviceFabric->connections.size(), 0u); + delete subDeviceFabric; +} + +TEST_F(TestFabricIaf, GivenIafFabricAvailableWhenPortPropertiesQueryIsUnsuccessfulThenNoConnectionsAreEnumerated) { + + NEO::DeviceVector devices; + devices.push_back(std::unique_ptr(neoDevice)); + std::unique_ptr> driverHandle = std::make_unique>(); + driverHandle->initialize(std::move(devices)); + + auto deviceImp = static_cast(driverHandle->devices[0]); + + FabricSubDeviceIaf *subDeviceFabric = new FabricSubDeviceIaf(deviceImp); + MockIafNlApi *mockNlApi = new MockIafNlApi(); + mockNlApi->fportPropertiesStatus = ZE_RESULT_ERROR_UNKNOWN; + subDeviceFabric->setIafNlApi(mockNlApi); + EXPECT_EQ(ZE_RESULT_SUCCESS, subDeviceFabric->enumerate()); + EXPECT_EQ(subDeviceFabric->connections.size(), 0u); + delete subDeviceFabric; +} + +TEST_F(TestFabricIaf, GivenIafFabricAvailableWhenSubDevicePropertiesGetIsUnsuccessfulThenReturnError) { + + NEO::DeviceVector devices; + devices.push_back(std::unique_ptr(neoDevice)); + std::unique_ptr> driverHandle = std::make_unique>(); + driverHandle->initialize(std::move(devices)); + + auto deviceImp = static_cast(driverHandle->devices[0]); + + FabricSubDeviceIaf *subDeviceFabric = new FabricSubDeviceIaf(deviceImp); + MockIafNlApi *mockNlApi = new MockIafNlApi(); + mockNlApi->subDevicePropertiesStatus = ZE_RESULT_ERROR_UNKNOWN; + subDeviceFabric->setIafNlApi(mockNlApi); + EXPECT_EQ(ZE_RESULT_ERROR_UNKNOWN, subDeviceFabric->enumerate()); + delete subDeviceFabric; +} + +TEST_F(TestFabricIaf, GivenIafFabricAvailableWhenNoPortsCanBeEnumeratedThenReturnSuccess) { + + NEO::DeviceVector devices; + devices.push_back(std::unique_ptr(neoDevice)); + std::unique_ptr> driverHandle = std::make_unique>(); + driverHandle->initialize(std::move(devices)); + + auto deviceImp = static_cast(driverHandle->devices[0]); + + FabricSubDeviceIaf *subDeviceFabric = new FabricSubDeviceIaf(deviceImp); + MockIafNlApi *mockNlApi = new MockIafNlApi(); + mockNlApi->portEnumerationEnable = false; + subDeviceFabric->setIafNlApi(mockNlApi); + EXPECT_EQ(ZE_RESULT_SUCCESS, subDeviceFabric->enumerate()); + delete subDeviceFabric; +} + +TEST_F(TestFabricIaf, GivenIafFabricAvailableWhenGetPortsReturnsErrorThenReturnError) { + NEO::DeviceVector devices; + devices.push_back(std::unique_ptr(neoDevice)); + std::unique_ptr> driverHandle = std::make_unique>(); + driverHandle->initialize(std::move(devices)); + auto deviceImp = static_cast(driverHandle->devices[0]); + + FabricSubDeviceIaf *subDeviceFabric = new FabricSubDeviceIaf(deviceImp); + MockIafNlApi *mockNlApi = new MockIafNlApi(); + mockNlApi->getPortsStatus = ZE_RESULT_ERROR_UNKNOWN; + subDeviceFabric->setIafNlApi(mockNlApi); + EXPECT_EQ(ZE_RESULT_ERROR_UNKNOWN, subDeviceFabric->enumerate()); + delete subDeviceFabric; +} + +} // namespace ult +} // namespace L0 \ No newline at end of file diff --git a/level_zero/core/test/unit_tests/sources/fabric/test_fabric.cpp b/level_zero/core/test/unit_tests/sources/fabric/test_fabric.cpp index 486754656f..b929d6811b 100644 --- a/level_zero/core/test/unit_tests/sources/fabric/test_fabric.cpp +++ b/level_zero/core/test/unit_tests/sources/fabric/test_fabric.cpp @@ -54,20 +54,28 @@ TEST_F(FabricVertexFixture, givenDevicesAreCreatedWhenzeFabricVertexGetSubVertic TEST_F(FabricVertexFixture, givenFabricVerticesAreCreatedWhenzeFabricVertexGetPropertiesExpIsCalledThenValidPropertiesAreReturned) { + // Delete existing fabric vertices + for (auto fabricVertex : driverHandle->fabricVertices) { + delete fabricVertex; + } + driverHandle->fabricVertices.clear(); + // Setup pci address for all devices for (auto l0Device : driverHandle->devices) { auto deviceImp = static_cast(l0Device); + for (auto l0SubDevice : deviceImp->subDevices) { + auto subDeviceImp = static_cast(l0SubDevice); NEO::DriverInfoMock *driverInfo = new DriverInfoMock(); driverInfo->setPciBusInfo({0, 1, 2, 3}); subDeviceImp->driverInfo.reset(driverInfo); - subDeviceImp->fabricVertex.reset(FabricVertex::createFromDevice(l0SubDevice)); } + NEO::DriverInfoMock *driverInfo = new DriverInfoMock(); driverInfo->setPciBusInfo({0, 1, 2, 3}); deviceImp->driverInfo.reset(driverInfo); - deviceImp->fabricVertex.reset(FabricVertex::createFromDevice(l0Device)); + driverHandle->fabricVertices.push_back(FabricVertex::createFromDevice(l0Device)); } uint32_t count = 0; @@ -129,5 +137,27 @@ TEST_F(FabricVertexFixture, givenFabricVerticesAreCreatedWhenzeFabricVertexGetPr } } +TEST(FabricEngineInstanceTest, GivenEngineInstancedDeviceWhenFabricVerticesAreCreatedThenSkipCreationForEngineInstanced) { + + auto hwInfo = *NEO::defaultHwInfo.get(); + auto executionEnvironment = MockDevice::prepareExecutionEnvironment(&hwInfo, 0u); + std::vector> devices(1); + devices[0].reset(static_cast(NEO::MockDevice::createWithExecutionEnvironment(&hwInfo, executionEnvironment, 0u))); + + auto mockDevice = static_cast(devices[0].get()); + NEO::SubDevice *subDevice = static_cast(mockDevice->createEngineInstancedSubDevice(0, defaultHwInfo->capabilityTable.defaultEngineType)); + mockDevice->subdevices.push_back(subDevice); + + std::unique_ptr> driverHandle; + driverHandle = std::make_unique>(); + ze_result_t res = driverHandle->initialize(std::move(devices)); + EXPECT_EQ(ZE_RESULT_SUCCESS, res); + + EXPECT_EQ(driverHandle->fabricVertices.size(), 1u); + uint32_t count = 0; + EXPECT_EQ(ZE_RESULT_SUCCESS, driverHandle->fabricVertices[0]->getSubVertices(&count, nullptr)); + EXPECT_EQ(count, 0u); +} + } // namespace ult } // namespace L0 \ No newline at end of file