diff --git a/level_zero/CMakeLists.txt b/level_zero/CMakeLists.txt index 772c773cbf..fe8c6a146e 100644 --- a/level_zero/CMakeLists.txt +++ b/level_zero/CMakeLists.txt @@ -75,6 +75,18 @@ if(BUILD_WITH_L0) endif() endif() + if(UNIX) + # Udev library interface + find_path(LIBUDEV_INCLUDE_DIR libudev.h) + if(LIBUDEV_INCLUDE_DIR) + message(STATUS "libudev headers directory: ${LIBUDEV_INCLUDE_DIR}") + include_directories(SYSTEM ${LIBUDEV_INCLUDE_DIR}) + set(LIBUDEV_FOUND TRUE) + else() + message(STATUS "libudev headers not available. Building without") + endif() + endif() + # Firmware Update Library get_filename_component(IGSC_DIR_tmp "${NEO_SOURCE_DIR}/../igsc" ABSOLUTE) if(EXISTS "${IGSC_DIR_tmp}/lib/cmake") diff --git a/level_zero/doc/BUILD.md b/level_zero/doc/BUILD.md index 9ea053b6c9..ea71c5b4ea 100644 --- a/level_zero/doc/BUILD.md +++ b/level_zero/doc/BUILD.md @@ -15,6 +15,8 @@ These instructions have been tested on Ubuntu* and complement those existing for To use Fabric related APIs, please build and/or install [libnl-3.7.0](https://www.linuxfromscratch.org/blfs/view/svn/basicnet/libnl.html). If installing to a local folder, `-DLIBGENL_INCLUDE_DIR=/libnl/include/libnl3/` could be passed to the cmake line of NEO build(Please refer top-level BUILD.md). +To use Sysman events API, please build and/or install libudev-dev + 2. Install/build Level Zero loader and Level Zero headers Install Level Zero loader and headers from [https://github.com/oneapi-src/level-zero/releases](https://github.com/oneapi-src/level-zero/releases). diff --git a/level_zero/tools/source/sysman/events/events_imp.cpp b/level_zero/tools/source/sysman/events/events_imp.cpp index f6a9614b8c..89e6ecb56d 100644 --- a/level_zero/tools/source/sysman/events/events_imp.cpp +++ b/level_zero/tools/source/sysman/events/events_imp.cpp @@ -26,7 +26,9 @@ void EventsImp::initEvents() { }); } void EventsImp::init() { - pOsEvents = OsEvents::create(pOsSysman); + if (pOsEvents == nullptr) { + pOsEvents = OsEvents::create(pOsSysman); + } UNRECOVERABLE_IF(nullptr == pOsEvents); } diff --git a/level_zero/tools/source/sysman/events/linux/os_events_imp_prelim.cpp b/level_zero/tools/source/sysman/events/linux/os_events_imp_prelim.cpp index 56bdbb9ce4..14b3bbcdf0 100644 --- a/level_zero/tools/source/sysman/events/linux/os_events_imp_prelim.cpp +++ b/level_zero/tools/source/sysman/events/linux/os_events_imp_prelim.cpp @@ -7,17 +7,23 @@ #include "level_zero/tools/source/sysman/events/linux/os_events_imp_prelim.h" +#include "shared/source/debug_settings/debug_settings_manager.h" +#include "shared/source/utilities/directory.h" + #include "level_zero/tools/source/sysman/memory/linux/os_memory_imp_prelim.h" #include "sysman/events/events_imp.h" #include "sysman/linux/os_sysman_imp.h" +#include + namespace L0 { -const std::string LinuxEventsImp::deviceMemoryHealth("device_memory_health"); -const std::string LinuxEventsImp::varFs("/var/lib/libze_intel_gpu/"); -const std::string LinuxEventsImp::detachEvent("remove"); -const std::string LinuxEventsImp::attachEvent("add"); +const std::string LinuxEventsImp::add("add"); +const std::string LinuxEventsImp::remove("remove"); +const std::string LinuxEventsImp::change("change"); +const std::string LinuxEventsImp::unbind("unbind"); +const std::string LinuxEventsImp::bind("bind"); static bool checkRasEventOccured(Ras *rasHandle) { zes_ras_config_t config = {}; @@ -58,28 +64,24 @@ bool LinuxEventsImp::checkRasEvent(zes_event_type_flags_t &pEvent) { return false; } -bool LinuxEventsImp::isResetRequired(zes_event_type_flags_t &pEvent) { - zes_device_state_t pState = {}; - pLinuxSysmanImp->getSysmanDeviceImp()->deviceGetState(&pState); - if (pState.reset) { +bool LinuxEventsImp::isResetRequired(void *dev, zes_event_type_flags_t &pEvent) { + if (action.compare(change) != 0) { + return false; + } + const char *str; + str = pUdevLib->getEventPropertyValue(dev, "RESET_FAILED"); + if (str && atoi(str) == 1) { pEvent |= ZES_EVENT_TYPE_FLAG_DEVICE_RESET_REQUIRED; return true; } + + // ZES_EVENT_TYPE_FLAG_DEVICE_RESET_REQUIRED event could also be received when, reset Reason is + // ZES_RESET_REASON_FLAG_REPAIR. return false; } bool LinuxEventsImp::checkDeviceDetachEvent(zes_event_type_flags_t &pEvent) { - // When device detach uevent is generated, then L0 udev rules will create a file: - // /var/lib/libze_intel_gpu/remove- - // For , check comment in LinuxEventsImp::init() - const std::string deviceDetachFile = detachEvent + "-" + pciIdPathTag; - const std::string deviceDetachFileAbsolutePath = varFs + deviceDetachFile; - uint32_t val = 0; - auto result = pFsAccess->read(deviceDetachFileAbsolutePath, val); - if (result != ZE_RESULT_SUCCESS) { - return false; - } - if (val == 1) { + if (action.compare(remove) == 0) { pEvent |= ZES_EVENT_TYPE_FLAG_DEVICE_DETACH; return true; } @@ -87,68 +89,192 @@ bool LinuxEventsImp::checkDeviceDetachEvent(zes_event_type_flags_t &pEvent) { } bool LinuxEventsImp::checkDeviceAttachEvent(zes_event_type_flags_t &pEvent) { - // When device detach uevent is generated, then L0 udev rules will create a file: - // /var/lib/libze_intel_gpu/add- - // For , check comment in LinuxEventsImp::init() - const std::string deviceAttachFile = attachEvent + "-" + pciIdPathTag; - const std::string deviceAttachFileAbsolutePath = varFs + deviceAttachFile; - uint32_t val = 0; - auto result = pFsAccess->read(deviceAttachFileAbsolutePath, val); - if (result != ZE_RESULT_SUCCESS) { - return false; - } - if (val == 1) { + if (action.compare(add) == 0) { pEvent |= ZES_EVENT_TYPE_FLAG_DEVICE_ATTACH; return true; } return false; } -bool LinuxEventsImp::checkIfMemHealthChanged(zes_event_type_flags_t &pEvent) { - if (currentMemHealth() != memHealthAtEventRegister) { - pEvent |= ZES_EVENT_TYPE_FLAG_MEM_HEALTH; - return true; +bool LinuxEventsImp::checkIfMemHealthChanged(void *dev, zes_event_type_flags_t &pEvent) { + if (action.compare(change) != 0) { + return false; + } + std::vector properties{"MEM_HEALTH_ALARM", "REBOOT_ALARM", "EC_PENDING", "DEGRADED", "EC_FAILED", "SPARING_STATUS_UNKNOWN"}; + for (auto &property : properties) { + const char *propVal = nullptr; + propVal = pUdevLib->getEventPropertyValue(dev, property.c_str()); + if (propVal && atoi(propVal) == 1) { + pEvent |= ZES_EVENT_TYPE_FLAG_MEM_HEALTH; + return true; + } } return false; } -bool LinuxEventsImp::checkIfFabricPortStatusChanged(zes_event_type_flags_t &pEvent) { - uint32_t currentFabricEventStatusVal = 0; - if (currentFabricEventStatus(currentFabricEventStatusVal) != ZE_RESULT_SUCCESS) { +bool LinuxEventsImp::checkIfFabricPortStatusChanged(void *dev, zes_event_type_flags_t &pEvent) { + if (action.compare(change) != 0) { return false; } - if (currentFabricEventStatusVal != fabricEventTrackAtRegister) { + + const char *str = pUdevLib->getEventPropertyValue(dev, "TYPE"); + if (str == nullptr) { + return false; + } + const char *expectedStr = "PORT_CHANGE"; + const size_t expectedStrLen = 11; // length of "PORT_CHANGE" is 11 + if (strlen(str) != strlen(expectedStr)) { + return false; + } + if (strncmp(str, expectedStr, expectedStrLen) == 0) { pEvent |= ZES_EVENT_TYPE_FLAG_FABRIC_PORT_HEALTH; return true; } return false; } +ze_result_t LinuxEventsImp::readFabricDeviceStats(const std::string &devicePciPath, struct stat &iafStat) { + const std::string iafDirectoryLegacy = "iaf."; + const std::string iafDirectory = "i915.iaf."; + const std::string fabricIdFile = "/iaf_fabric_id"; + int fd = -1; + std::string path; + path.clear(); + std::vector list = NEO::Directory::getFiles(devicePciPath); + for (auto entry : list) { + if ((entry.find(iafDirectory) != std::string::npos) || + (entry.find(iafDirectoryLegacy) != std::string::npos)) { + path = entry + fabricIdFile; + break; + } + } + if (path.empty()) { + // This device does not have a fabric + return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + fd = NEO::SysCalls::open(path.c_str(), O_RDONLY); + if (fd < 0) { + return ZE_RESULT_ERROR_DEPENDENCY_UNAVAILABLE; + } + NEO::SysCalls::fstat(fd, &iafStat); + NEO::SysCalls::close(fd); + return ZE_RESULT_SUCCESS; +} + +bool LinuxEventsImp::listenSystemEvents(zes_event_type_flags_t &pEvent, uint64_t timeout) { + bool retval = false; + struct pollfd pfd; + struct stat iafStat; + std::vector subsystemList; + + if (pUdevLib == nullptr) { + NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, + "%s", "libudev library instantiation failed\n"); + return retval; + } + + // Get fabric device stats + const std::string iafPath = "device/"; + std::string iafRealPath = {}; + pLinuxSysmanImp->getSysfsAccess().getRealPath(iafPath, iafRealPath); + auto result = readFabricDeviceStats(iafRealPath, iafStat); + if (result == ZE_RESULT_SUCCESS) { + subsystemList.push_back("platform"); + } + + subsystemList.push_back("drm"); + pfd.fd = pUdevLib->registerEventsFromSubsystemAndGetFd(subsystemList); + pfd.events = POLLIN; + + auto pDrm = &pLinuxSysmanImp->getDrm(); + struct stat drmStat; + NEO::SysCalls::fstat(pDrm->getFileDescriptor(), &drmStat); + + auto start = std::chrono::steady_clock::now(); + std::chrono::duration timeElapsed; + while (NEO::SysCalls::poll(&pfd, 1, static_cast(timeout)) > 0) { + // Check again for registered events. Its possible while we are looping for events, registered events are cleared + if (!registeredEvents) { + return true; + } + + dev_t devnum; + void *dev = nullptr; + dev = pUdevLib->allocateDeviceToReceiveData(); + if (dev == nullptr) { + timeElapsed = std::chrono::steady_clock::now() - start; + if (timeout > timeElapsed.count()) { + timeout = timeout - timeElapsed.count(); + continue; + } else { + break; + } + } + + devnum = pUdevLib->getEventGenerationSourceDevice(dev); + + if ((memcmp(&drmStat.st_rdev, &devnum, sizeof(dev_t)) == 0) || + (memcmp(&iafStat.st_rdev, &devnum, sizeof(dev_t)) == 0)) { + auto eventTypePtr = pUdevLib->getEventType(dev); + + if (eventTypePtr != nullptr) { + action = std::string(eventTypePtr); + if (registeredEvents & ZES_EVENT_TYPE_FLAG_FABRIC_PORT_HEALTH) { + if (checkIfFabricPortStatusChanged(dev, pEvent)) { + registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_FABRIC_PORT_HEALTH); + retval = true; + } + } + if (registeredEvents & ZES_EVENT_TYPE_FLAG_DEVICE_DETACH) { + if (checkDeviceDetachEvent(pEvent)) { + registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_DEVICE_DETACH); // After receiving event unregister it + retval = true; + } + } + if (registeredEvents & ZES_EVENT_TYPE_FLAG_DEVICE_ATTACH) { + if (checkDeviceAttachEvent(pEvent)) { + registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_DEVICE_ATTACH); + retval = true; + } + } + if (registeredEvents & ZES_EVENT_TYPE_FLAG_DEVICE_RESET_REQUIRED) { + if (isResetRequired(dev, pEvent)) { + registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_DEVICE_RESET_REQUIRED); + retval = true; + } + } + if (registeredEvents & ZES_EVENT_TYPE_FLAG_MEM_HEALTH) { + if (checkIfMemHealthChanged(dev, pEvent)) { + registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_MEM_HEALTH); + retval = true; + } + } + } + } + + pUdevLib->dropDeviceReference(dev); + if (retval) { + break; + } + timeElapsed = std::chrono::steady_clock::now() - start; + if (timeout > timeElapsed.count()) { + timeout = timeout - timeElapsed.count(); + continue; + } else { + break; + } + } + + return retval; +} + bool LinuxEventsImp::eventListen(zes_event_type_flags_t &pEvent, uint64_t timeout) { - if (registeredEvents & ZES_EVENT_TYPE_FLAG_DEVICE_RESET_REQUIRED) { - if (isResetRequired(pEvent)) { - registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_DEVICE_RESET_REQUIRED); //After receiving event unregister it - return true; - } - } - if (registeredEvents & ZES_EVENT_TYPE_FLAG_DEVICE_DETACH) { - if (checkDeviceDetachEvent(pEvent)) { - registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_DEVICE_DETACH); - return true; - } - } - if (registeredEvents & ZES_EVENT_TYPE_FLAG_DEVICE_ATTACH) { - if (checkDeviceAttachEvent(pEvent)) { - registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_DEVICE_ATTACH); - return true; - } - } - if (registeredEvents & ZES_EVENT_TYPE_FLAG_MEM_HEALTH) { - if (checkIfMemHealthChanged(pEvent)) { - registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_MEM_HEALTH); - return true; - } + bool retval = false; + if (!registeredEvents) { + return retval; } + + pEvent = 0; if ((registeredEvents & ZES_EVENT_TYPE_FLAG_RAS_CORRECTABLE_ERRORS) || (registeredEvents & ZES_EVENT_TYPE_FLAG_RAS_UNCORRECTABLE_ERRORS)) { if (checkRasEvent(pEvent)) { if (pEvent & ZES_EVENT_TYPE_FLAG_RAS_CORRECTABLE_ERRORS) { @@ -159,74 +285,33 @@ bool LinuxEventsImp::eventListen(zes_event_type_flags_t &pEvent, uint64_t timeou return true; } } - if (registeredEvents & ZES_EVENT_TYPE_FLAG_FABRIC_PORT_HEALTH) { - if (checkIfFabricPortStatusChanged(pEvent)) { - registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_FABRIC_PORT_HEALTH); - return true; - } - } - return false; + + return listenSystemEvents(pEvent, timeout); } ze_result_t LinuxEventsImp::eventRegister(zes_event_type_flags_t events) { if (0x7fff < events) { return ZE_RESULT_ERROR_INVALID_ENUMERATION; } - registeredEvents |= events; - if (registeredEvents & ZES_EVENT_TYPE_FLAG_MEM_HEALTH) { - memHealthAtEventRegister = currentMemHealth(); - } - if (registeredEvents & ZES_EVENT_TYPE_FLAG_FABRIC_PORT_HEALTH) { - currentFabricEventStatus(fabricEventTrackAtRegister); + if (!events) { + // If user is trying to register events with empty events argument, then clear all the registered events + registeredEvents = events; + } else { + // supportedEventMask --> this mask checks for events that supported currently + zes_event_type_flags_t supportedEventMask = ZES_EVENT_TYPE_FLAG_FABRIC_PORT_HEALTH | ZES_EVENT_TYPE_FLAG_DEVICE_DETACH | + ZES_EVENT_TYPE_FLAG_DEVICE_ATTACH | ZES_EVENT_TYPE_FLAG_DEVICE_RESET_REQUIRED | + ZES_EVENT_TYPE_FLAG_MEM_HEALTH | ZES_EVENT_TYPE_FLAG_RAS_CORRECTABLE_ERRORS | + ZES_EVENT_TYPE_FLAG_RAS_UNCORRECTABLE_ERRORS; + registeredEvents |= (events & supportedEventMask); } return ZE_RESULT_SUCCESS; } -ze_result_t LinuxEventsImp::currentFabricEventStatus(uint32_t &val) { - // When Fabric port status change uevent is generated, then L0 udev rules will create a file: - // /var/lib/libze_intel_gpu/fabric- - // For , check comment in LinuxEventsImp::init() - const std::string fabric = "fabric"; - const std::string fabricEventFile = fabric + "-" + pciIdPathTag; - const std::string fabricEventFileAbsolutePath = varFs + fabricEventFile; - return pFsAccess->read(fabricEventFileAbsolutePath, val); -} - -zes_mem_health_t LinuxEventsImp::currentMemHealth() { - std::string memHealth; - ze_result_t result = pSysfsAccess->read(deviceMemoryHealth, memHealth); - if (ZE_RESULT_SUCCESS != result) { - return ZES_MEM_HEALTH_UNKNOWN; - } - - auto health = i915ToL0MemHealth.find(memHealth); - if (health != i915ToL0MemHealth.end()) { - return i915ToL0MemHealth.at(memHealth); - } - return ZES_MEM_HEALTH_UNKNOWN; -} - -void LinuxEventsImp::getPciIdPathTag() { - std::string bdfDir; - ze_result_t result = pSysfsAccess->readSymLink("device", bdfDir); - if (ZE_RESULT_SUCCESS != result) { - return; - } - const auto loc = bdfDir.find_last_of('/'); - auto bdf = bdfDir.substr(loc + 1); - std::replace(bdf.begin(), bdf.end(), ':', '_'); - std::replace(bdf.begin(), bdf.end(), '.', '_'); - // ID_PATH_TAG key is received when uevent related to device add/remove is generated. - // Example of ID_PATH_TAG is: - // ID_PATH_TAG=pci-0000_8c_00_0 - pciIdPathTag = "pci-" + bdf; -} - LinuxEventsImp::LinuxEventsImp(OsSysman *pOsSysman) { pLinuxSysmanImp = static_cast(pOsSysman); pSysfsAccess = &pLinuxSysmanImp->getSysfsAccess(); pFsAccess = &pLinuxSysmanImp->getFsAccess(); - getPciIdPathTag(); + pUdevLib = pLinuxSysmanImp->getUdevLibHandle(); } OsEvents *OsEvents::create(OsSysman *pOsSysman) { diff --git a/level_zero/tools/source/sysman/events/linux/os_events_imp_prelim.h b/level_zero/tools/source/sysman/events/linux/os_events_imp_prelim.h index 34f5e868fe..8403ca8301 100644 --- a/level_zero/tools/source/sysman/events/linux/os_events_imp_prelim.h +++ b/level_zero/tools/source/sysman/events/linux/os_events_imp_prelim.h @@ -8,6 +8,7 @@ #pragma once #include "level_zero/tools/source/sysman/events/os_events.h" #include "level_zero/tools/source/sysman/linux/os_sysman_imp.h" +#include "level_zero/tools/source/sysman/linux/udev/udev_lib.h" namespace L0 { @@ -21,27 +22,27 @@ class LinuxEventsImp : public OsEvents, NEO::NonCopyableOrMovableClass { protected: LinuxSysmanImp *pLinuxSysmanImp = nullptr; - void getPciIdPathTag(); - zes_mem_health_t currentMemHealth(); - ze_result_t currentFabricEventStatus(uint32_t &val); - bool isResetRequired(zes_event_type_flags_t &pEvent); + bool isResetRequired(void *dev, zes_event_type_flags_t &pEvent); bool checkDeviceDetachEvent(zes_event_type_flags_t &pEvent); bool checkDeviceAttachEvent(zes_event_type_flags_t &pEvent); - bool checkIfMemHealthChanged(zes_event_type_flags_t &pEvent); - bool checkIfFabricPortStatusChanged(zes_event_type_flags_t &pEvent); + bool checkIfMemHealthChanged(void *dev, zes_event_type_flags_t &pEvent); + bool checkIfFabricPortStatusChanged(void *dev, zes_event_type_flags_t &pEvent); bool checkRasEvent(zes_event_type_flags_t &pEvent); - std::string pciIdPathTag; - zes_mem_health_t memHealthAtEventRegister = ZES_MEM_HEALTH_UNKNOWN; + ze_result_t readFabricDeviceStats(const std::string &devicePciPath, struct stat &iafStat); + bool listenSystemEvents(zes_event_type_flags_t &pEvent, uint64_t timeout); uint32_t fabricEventTrackAtRegister = 0; + L0::UdevLib *pUdevLib = nullptr; + zes_event_type_flags_t registeredEvents = 0; private: FsAccess *pFsAccess = nullptr; SysfsAccess *pSysfsAccess = nullptr; - static const std::string varFs; - static const std::string detachEvent; - static const std::string attachEvent; - static const std::string deviceMemoryHealth; - zes_event_type_flags_t registeredEvents = 0; + std::string action; + static const std::string add; + static const std::string remove; + static const std::string change; + static const std::string unbind; + static const std::string bind; }; } // namespace L0 diff --git a/level_zero/tools/source/sysman/linux/CMakeLists.txt b/level_zero/tools/source/sysman/linux/CMakeLists.txt index e962e525b2..886509381e 100644 --- a/level_zero/tools/source/sysman/linux/CMakeLists.txt +++ b/level_zero/tools/source/sysman/linux/CMakeLists.txt @@ -1,5 +1,5 @@ # -# Copyright (C) 2020-2021 Intel Corporation +# Copyright (C) 2020-2022 Intel Corporation # # SPDX-License-Identifier: MIT # @@ -22,21 +22,3 @@ add_subdirectories() # Make our source files visible to parent set_property(GLOBAL PROPERTY L0_SRCS_TOOLS_SYSMAN_LINUX ${L0_SRCS_TOOLS_SYSMAN_LINUX}) - -if(L0_INSTALL_UDEV_RULES) - install( - FILES "${CMAKE_CURRENT_SOURCE_DIR}/udev/99-drm_ze_intel_gpu.rules" - DESTINATION ${UDEV_RULES_DIR} - COMPONENT ${PROJECT_NAME} - ) - install( - FILES "${CMAKE_CURRENT_SOURCE_DIR}/udev/wedged_file" - DESTINATION "/var/lib/libze_intel_gpu/" - COMPONENT ${PROJECT_NAME} - ) - install( - FILES "${CMAKE_CURRENT_SOURCE_DIR}/udev/pci_bind_status_file" - DESTINATION "/var/lib/libze_intel_gpu/" - COMPONENT ${PROJECT_NAME} - ) -endif() diff --git a/level_zero/tools/source/sysman/linux/os_sysman_imp.cpp b/level_zero/tools/source/sysman/linux/os_sysman_imp.cpp index 2d1c96d0b6..2d70d3673a 100644 --- a/level_zero/tools/source/sysman/linux/os_sysman_imp.cpp +++ b/level_zero/tools/source/sysman/linux/os_sysman_imp.cpp @@ -88,6 +88,13 @@ FirmwareUtil *LinuxSysmanImp::getFwUtilInterface() { return pFwUtilInterface; } +L0::UdevLib *LinuxSysmanImp::getUdevLibHandle() { + if (pUdevLib == nullptr) { + pUdevLib = UdevLib::create(); + } + return pUdevLib; +} + FsAccess &LinuxSysmanImp::getFsAccess() { UNRECOVERABLE_IF(nullptr == pFsAccess); return *pFsAccess; @@ -224,6 +231,10 @@ LinuxSysmanImp::~LinuxSysmanImp() { delete pPmuInterface; pPmuInterface = nullptr; } + if (nullptr != pUdevLib) { + delete pUdevLib; + pUdevLib = nullptr; + } releaseFwUtilInterface(); releasePmtObject(); } diff --git a/level_zero/tools/source/sysman/linux/os_sysman_imp.h b/level_zero/tools/source/sysman/linux/os_sysman_imp.h index cbdcb50972..a33db5e2a5 100644 --- a/level_zero/tools/source/sysman/linux/os_sysman_imp.h +++ b/level_zero/tools/source/sysman/linux/os_sysman_imp.h @@ -14,6 +14,7 @@ #include "level_zero/tools/source/sysman/linux/fs_access.h" #include "level_zero/tools/source/sysman/linux/pmt/pmt.h" #include "level_zero/tools/source/sysman/linux/pmu/pmu_imp.h" +#include "level_zero/tools/source/sysman/linux/udev/udev_lib.h" #include "level_zero/tools/source/sysman/sysman_imp.h" #include @@ -44,6 +45,7 @@ class LinuxSysmanImp : public OsSysman, NEO::NonCopyableOrMovableClass { ze_result_t init() override; + L0::UdevLib *getUdevLibHandle(); PmuInterface *getPmuInterface(); FirmwareUtil *getFwUtilInterface(); FsAccess &getFsAccess(); @@ -88,6 +90,7 @@ class LinuxSysmanImp : public OsSysman, NEO::NonCopyableOrMovableClass { NEO::Drm *pDrm = nullptr; PmuInterface *pPmuInterface = nullptr; FirmwareUtil *pFwUtilInterface = nullptr; + L0::UdevLib *pUdevLib = nullptr; std::map mapOfSubDeviceIdToPmtObject; ze_result_t initLocalDeviceAndDrmHandles(); diff --git a/level_zero/tools/source/sysman/linux/udev/99-drm_ze_intel_gpu.rules b/level_zero/tools/source/sysman/linux/udev/99-drm_ze_intel_gpu.rules deleted file mode 100644 index f078bb46df..0000000000 --- a/level_zero/tools/source/sysman/linux/udev/99-drm_ze_intel_gpu.rules +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright (C) 2020-2021 Intel Corporation -# -# SPDX-License-Identifier: MIT -# - -ACTION=="change", \ -SUBSYSTEM=="drm", \ -ENV{RESET_FAILED}=="1", \ -ENV{RESET_UNIT}=="gt", \ -RUN+="/bin/sh -c 'echo 1 > /var/lib/libze_intel_gpu/wedged_file'" -ACTION=="unbind", \ -SUBSYSTEM=="drm", \ -RUN+="/bin/sh -c 'echo 0 > /var/lib/libze_intel_gpu/pci_bind_status_file'" -ACTION=="bind", \ -SUBSYSTEM=="drm", \ -RUN+="/bin/sh -c 'echo 1 > /var/lib/libze_intel_gpu/pci_bind_status_file'", \ -RUN+="/bin/sh -c 'echo 0 > /var/lib/libze_intel_gpu/wedged_file'" -ACTION=="remove", \ -SUBSYSTEM=="drm", \ -ENV{DEVNAME}=="/dev/dri/card*", \ -ENV{ID_PATH_TAG}=="pci-*", \ -RUN+="/bin/sh -c 'touch /var/lib/libze_intel_gpu/remove-$env{ID_PATH_TAG}; echo 1 > /var/lib/libze_intel_gpu/remove-$env{ID_PATH_TAG}; rm /var/lib/libze_intel_gpu/add-$env{ID_PATH_TAG};'" -ACTION=="add", \ -SUBSYSTEM=="drm", \ -ENV{DEVNAME}=="/dev/dri/card*", \ -ENV{ID_PATH_TAG}=="pci-*", \ -RUN+="/bin/sh -c 'touch /var/lib/libze_intel_gpu/add-$env{ID_PATH_TAG}; echo 1 > /var/lib/libze_intel_gpu/add-$env{ID_PATH_TAG}; rm /var/lib/libze_intel_gpu/remove-$env{ID_PATH_TAG}; echo 0 > /var/lib/libze_intel_gpu/wedged_file;'" -ACTION=="change", \ -SUBSYSTEM=="platform", \ -ENV{DEVTYPE}=="mfd_device", \ -ENV{DRIVER}=="iaf", \ -ENV{TYPE}=="PORT_CHANGE", \ -ENV{ID_PATH_TAG}=="pci-*", \ -RUN+="/bin/sh -c 'pci_id_path_tag=${env{ID_PATH_TAG}::16}; filename=/var/lib/libze_intel_gpu/fabric-${pci_id_path_tag}; touch ${filename}; echo $(($(cat ${filename}) + 1)) > ${filename};'" diff --git a/level_zero/tools/source/sysman/linux/udev/CMakeLists.txt b/level_zero/tools/source/sysman/linux/udev/CMakeLists.txt new file mode 100644 index 0000000000..164e737430 --- /dev/null +++ b/level_zero/tools/source/sysman/linux/udev/CMakeLists.txt @@ -0,0 +1,29 @@ +# +# Copyright (C) 2022 Intel Corporation +# +# SPDX-License-Identifier: MIT +# + +if(LIBUDEV_FOUND) + set(L0_SRCS_TOOLS_SYSMAN_LINUX_UDEV_LIB + ${CMAKE_CURRENT_SOURCE_DIR}/udev_lib_imp.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/udev_lib_imp.h + ${CMAKE_CURRENT_SOURCE_DIR}/udev_lib.h + ) + add_subdirectories() +else() + set(L0_SRCS_TOOLS_SYSMAN_LINUX_UDEV_LIB + ${CMAKE_CURRENT_SOURCE_DIR}/udev_lib_imp_stub.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/udev_lib.h + ) +endif() + +if(UNIX) + target_sources(${L0_STATIC_LIB_NAME} + PRIVATE + ${L0_SRCS_TOOLS_SYSMAN_LINUX_UDEV_LIB} + ) +endif() + +# Make our source files visible to parent +set_property(GLOBAL PROPERTY L0_SRCS_TOOLS_SYSMAN_LINUX_UDEV_LIB ${L0_SRCS_TOOLS_SYSMAN_LINUX_UDEV_LIB}) diff --git a/level_zero/tools/source/sysman/linux/udev/pci_bind_status_file b/level_zero/tools/source/sysman/linux/udev/pci_bind_status_file deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/level_zero/tools/source/sysman/linux/udev/udev_lib.h b/level_zero/tools/source/sysman/linux/udev/udev_lib.h new file mode 100644 index 0000000000..bc031ff5eb --- /dev/null +++ b/level_zero/tools/source/sysman/linux/udev/udev_lib.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include +#include + +namespace L0 { +class UdevLib { + public: + UdevLib() = default; + static UdevLib *create(); + virtual int registerEventsFromSubsystemAndGetFd(std::vector &subsystemList) = 0; + virtual dev_t getEventGenerationSourceDevice(void *dev) = 0; + virtual const char *getEventType(void *dev) = 0; + virtual const char *getEventPropertyValue(void *dev, const char *key) = 0; + virtual void *allocateDeviceToReceiveData() = 0; + virtual void dropDeviceReference(void *dev) = 0; + virtual ~UdevLib() {} +}; +} // namespace L0 diff --git a/level_zero/tools/source/sysman/linux/udev/udev_lib_imp.cpp b/level_zero/tools/source/sysman/linux/udev/udev_lib_imp.cpp new file mode 100644 index 0000000000..72ffef5850 --- /dev/null +++ b/level_zero/tools/source/sysman/linux/udev/udev_lib_imp.cpp @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "level_zero/tools/source/sysman/linux/udev/udev_lib_imp.h" + +#include "shared/source/helpers/debug_helpers.h" +#include "shared/source/utilities/directory.h" + +namespace L0 { +static constexpr std::string_view libUdevlFile = "libudev.so"; +const std::string udevNewRoutine = "udev_new"; +const std::string udevMonitorNewFromNetlinkRoutine = "udev_monitor_new_from_netlink"; +const std::string udevMonitorFilterAddMatchSubsystemDevtypeRoutine = "udev_monitor_filter_add_match_subsystem_devtype"; +const std::string udevMonitorEnableReceivingRoutine = "udev_monitor_enable_receiving"; +const std::string udevMonitorGetFdRoutine = "udev_monitor_get_fd"; +const std::string udevMonitorReceiveDeviceRoutine = "udev_monitor_receive_device"; +const std::string udevDeviceGetDevnumRoutine = "udev_device_get_devnum"; +const std::string udevDeviceGetActionRoutine = "udev_device_get_action"; +const std::string udevDeviceGetPropertyValueRoutine = "udev_device_get_property_value"; +const std::string udevDeviceUnrefRoutine = "udev_device_unref"; + +bool UdevLibImp::loadEntryPoints() { + bool ok = getSymbolAddr(udevMonitorNewFromNetlinkRoutine, pUdevMonitorNewFromNetlinkEntry); + ok = ok && getSymbolAddr(udevNewRoutine, pUdevNewEntry); + ok = ok && getSymbolAddr(udevMonitorFilterAddMatchSubsystemDevtypeRoutine, pUdevMonitorFilterAddMatchSubsystemDevtypeEntry); + ok = ok && getSymbolAddr(udevMonitorEnableReceivingRoutine, pUdevMonitorEnableReceivingEntry); + ok = ok && getSymbolAddr(udevMonitorGetFdRoutine, pUdevMonitorGetFdEntry); + ok = ok && getSymbolAddr(udevMonitorReceiveDeviceRoutine, pUdevMonitorReceiveDeviceEntry); + ok = ok && getSymbolAddr(udevDeviceGetDevnumRoutine, pUdevDeviceGetDevnumEntry); + ok = ok && getSymbolAddr(udevDeviceGetActionRoutine, pUdevDeviceGetActionEntry); + ok = ok && getSymbolAddr(udevDeviceGetPropertyValueRoutine, pUdevDeviceGetPropertyValueEntry); + ok = ok && getSymbolAddr(udevDeviceUnrefRoutine, pUdevDeviceUnrefEntry); + + return ok; +} + +const char *UdevLibImp::getEventPropertyValue(void *dev, const char *key) { + return pUdevDeviceGetPropertyValueEntry(reinterpret_cast(dev), key); +} + +const char *UdevLibImp::getEventType(void *dev) { + return pUdevDeviceGetActionEntry(reinterpret_cast(dev)); +} + +void *UdevLibImp::allocateDeviceToReceiveData() { + struct udev_device *dev = pUdevMonitorReceiveDeviceEntry(mon); + return reinterpret_cast(dev); +} + +dev_t UdevLibImp::getEventGenerationSourceDevice(void *dev) { + return pUdevDeviceGetDevnumEntry(reinterpret_cast(dev)); +} + +int UdevLibImp::registerEventsFromSubsystemAndGetFd(std::vector &subsystemList) { + for (const auto &subsystem : subsystemList) { + pUdevMonitorFilterAddMatchSubsystemDevtypeEntry(mon, subsystem.c_str(), NULL); + } + if (pUdevMonitorEnableReceivingEntry(mon) != 0) { + return -1; + } + return pUdevMonitorGetFdEntry(mon); +} + +void UdevLibImp::dropDeviceReference(void *dev) { + pUdevDeviceUnrefEntry(reinterpret_cast(dev)); +} + +bool UdevLibImp::init() { + mon = pUdevMonitorNewFromNetlinkEntry(pUdevNewEntry(), "kernel"); + return (mon != nullptr); +} + +UdevLibImp::~UdevLibImp(){}; + +UdevLib *UdevLib::create() { + UdevLibImp *pUdevLib = new UdevLibImp(); + UNRECOVERABLE_IF(nullptr == pUdevLib); + pUdevLib->libraryHandle.reset(NEO::OsLibrary::load(std::string(libUdevlFile))); + if (pUdevLib->libraryHandle == nullptr || pUdevLib->loadEntryPoints() == false || !pUdevLib->init()) { + delete pUdevLib; + return nullptr; + } + return static_cast(pUdevLib); +} +} // namespace L0 diff --git a/level_zero/tools/source/sysman/linux/udev/udev_lib_imp.h b/level_zero/tools/source/sysman/linux/udev/udev_lib_imp.h new file mode 100644 index 0000000000..1bd3e6074c --- /dev/null +++ b/level_zero/tools/source/sysman/linux/udev/udev_lib_imp.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "shared/source/helpers/non_copyable_or_moveable.h" +#include "shared/source/os_interface/os_library.h" + +#include "level_zero/tools/source/sysman/linux/udev/udev_lib.h" + +#include +#include +#include +#include +#include + +namespace L0 { + +typedef struct udev *(*pUdevNew)(void); +typedef struct udev_monitor *(*pUdevMonitorNewFromNetlink)(struct udev *, const char *); +typedef int (*pUdevMonitorFilterAddMatchSubsystemDevtype)(struct udev_monitor *udev_monitor, const char *subsystem, const char *devtype); +typedef int (*pUdevMonitorEnableReceiving)(struct udev_monitor *udev_monitor); +typedef int (*pUdevMonitorGetFd)(struct udev_monitor *udev_monitor); +typedef struct udev_device *(*pUdevMonitorReceiveDevice)(struct udev_monitor *udev_monitor); +typedef dev_t (*pUdevDeviceGetDevnum)(struct udev_device *udev_device); +typedef const char *(*pUdevDeviceGetAction)(struct udev_device *udev_device); +typedef const char *(*pUdevDeviceGetPropertyValue)(struct udev_device *udev_device, const char *key); +typedef struct udev_device *(*pUdevDeviceUnref)(struct udev_device *udev_device); + +class UdevLibImp : public UdevLib { + public: + UdevLibImp() = default; + ~UdevLibImp() override; + std::unique_ptr libraryHandle; + bool loadEntryPoints(); + template + bool getSymbolAddr(const std::string name, T &proc) { + void *addr = libraryHandle->getProcAddress(name); + proc = reinterpret_cast(addr); + return nullptr != proc; + } + bool init(); + int registerEventsFromSubsystemAndGetFd(std::vector &subsystemList) override; + dev_t getEventGenerationSourceDevice(void *dev) override; + const char *getEventType(void *dev) override; + const char *getEventPropertyValue(void *dev, const char *key) override; + void *allocateDeviceToReceiveData() override; + void dropDeviceReference(void *dev) override; + + protected: + pUdevNew pUdevNewEntry = nullptr; + pUdevMonitorNewFromNetlink pUdevMonitorNewFromNetlinkEntry = nullptr; + pUdevMonitorFilterAddMatchSubsystemDevtype pUdevMonitorFilterAddMatchSubsystemDevtypeEntry = nullptr; + pUdevMonitorEnableReceiving pUdevMonitorEnableReceivingEntry = nullptr; + pUdevMonitorGetFd pUdevMonitorGetFdEntry = nullptr; + pUdevMonitorReceiveDevice pUdevMonitorReceiveDeviceEntry = nullptr; + pUdevDeviceGetDevnum pUdevDeviceGetDevnumEntry = nullptr; + pUdevDeviceGetAction pUdevDeviceGetActionEntry = nullptr; + pUdevDeviceGetPropertyValue pUdevDeviceGetPropertyValueEntry = nullptr; + pUdevDeviceUnref pUdevDeviceUnrefEntry = nullptr; + + private: + struct udev_monitor *mon = nullptr; +}; + +} // namespace L0 diff --git a/level_zero/tools/source/sysman/linux/udev/udev_lib_imp_stub.cpp b/level_zero/tools/source/sysman/linux/udev/udev_lib_imp_stub.cpp new file mode 100644 index 0000000000..7d66df747b --- /dev/null +++ b/level_zero/tools/source/sysman/linux/udev/udev_lib_imp_stub.cpp @@ -0,0 +1,15 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "level_zero/tools/source/sysman/linux/udev/udev_lib.h" + +namespace L0 { + +UdevLib *UdevLib::create() { + return nullptr; +} +} // namespace L0 diff --git a/level_zero/tools/source/sysman/linux/udev/wedged_file b/level_zero/tools/source/sysman/linux/udev/wedged_file deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/level_zero/tools/test/unit_tests/sources/sysman/linux/mock_sysman_fixture.h b/level_zero/tools/test/unit_tests/sources/sysman/linux/mock_sysman_fixture.h index e36b37be6b..2013d1d6d0 100644 --- a/level_zero/tools/test/unit_tests/sources/sysman/linux/mock_sysman_fixture.h +++ b/level_zero/tools/test/unit_tests/sources/sysman/linux/mock_sysman_fixture.h @@ -46,6 +46,7 @@ class PublicLinuxSysmanImp : public L0::LinuxSysmanImp { using LinuxSysmanImp::pPmuInterface; using LinuxSysmanImp::pProcfsAccess; using LinuxSysmanImp::pSysfsAccess; + using LinuxSysmanImp::pUdevLib; }; class SysmanDeviceFixture : public DeviceFixture, public ::testing::Test { diff --git a/scripts/packaging/l0_gpu_driver/rhel_8/SPECS/l0_gpu_driver.spec b/scripts/packaging/l0_gpu_driver/rhel_8/SPECS/l0_gpu_driver.spec index 9ad78cd203..cc4f038ee9 100644 --- a/scripts/packaging/l0_gpu_driver/rhel_8/SPECS/l0_gpu_driver.spec +++ b/scripts/packaging/l0_gpu_driver/rhel_8/SPECS/l0_gpu_driver.spec @@ -84,9 +84,6 @@ cp -pvR %{_sourcedir}/copyright %{buildroot}/usr/share/doc/intel-level-zero-gpu/ %files %defattr(-,root,root) %{_libdir}/libze_intel_gpu.so.* -%{_sharedstatedir}/libze_intel_gpu/pci_bind_status_file -%{_sharedstatedir}/libze_intel_gpu/wedged_file -%{_sysconfdir}/udev/rules.d/99-drm_ze_intel_gpu.rules /usr/share/doc/intel-level-zero-gpu/copyright %config(noreplace) diff --git a/scripts/packaging/l0_gpu_driver/sles_15.3/SPECS/l0_gpu_driver.spec b/scripts/packaging/l0_gpu_driver/sles_15.3/SPECS/l0_gpu_driver.spec index 95970ff346..d2765f7bca 100644 --- a/scripts/packaging/l0_gpu_driver/sles_15.3/SPECS/l0_gpu_driver.spec +++ b/scripts/packaging/l0_gpu_driver/sles_15.3/SPECS/l0_gpu_driver.spec @@ -94,9 +94,6 @@ cp -pvR %{_sourcedir}/copyright %{buildroot}/usr/share/doc/intel-level-zero-gpu% %files -n intel-level-zero-gpu%{?name_suffix} %defattr(-,root,root) %{_libdir}/libze_intel_gpu.so.* -%{_sharedstatedir}/libze_intel_gpu/pci_bind_status_file -%{_sharedstatedir}/libze_intel_gpu/wedged_file -%{_sysconfdir}/udev/rules.d/99-drm_ze_intel_gpu.rules /usr/share/doc/intel-level-zero-gpu%{?name_suffix}/copyright %config(noreplace) diff --git a/shared/test/common/os_interface/linux/sys_calls_linux_ult.cpp b/shared/test/common/os_interface/linux/sys_calls_linux_ult.cpp index 685f227cf5..88b65d3438 100644 --- a/shared/test/common/os_interface/linux/sys_calls_linux_ult.cpp +++ b/shared/test/common/os_interface/linux/sys_calls_linux_ult.cpp @@ -56,6 +56,7 @@ int (*sysCallsReadlink)(const char *path, char *buf, size_t bufsize) = nullptr; int (*sysCallsIoctl)(int fileDescriptor, unsigned long int request, void *arg) = nullptr; int (*sysCallsPoll)(struct pollfd *pollFd, unsigned long int numberOfFds, int timeout) = nullptr; ssize_t (*sysCallsRead)(int fd, void *buf, size_t count) = nullptr; +int (*sysCallsFstat)(int fd, struct stat *buf) = nullptr; int close(int fileDescriptor) { closeFuncCalled++; @@ -169,6 +170,9 @@ int poll(struct pollfd *pollFd, unsigned long int numberOfFds, int timeout) { } int fstat(int fd, struct stat *buf) { + if (sysCallsFstat != nullptr) { + return sysCallsFstat(fd, buf); + } return fstatFuncRetVal; } diff --git a/shared/test/common/os_interface/linux/sys_calls_linux_ult.h b/shared/test/common/os_interface/linux/sys_calls_linux_ult.h index b216146ba0..2553ebbd5f 100644 --- a/shared/test/common/os_interface/linux/sys_calls_linux_ult.h +++ b/shared/test/common/os_interface/linux/sys_calls_linux_ult.h @@ -9,6 +9,7 @@ #include #include +#include namespace NEO { namespace SysCalls { @@ -19,6 +20,7 @@ extern int (*sysCallsReadlink)(const char *path, char *buf, size_t bufsize); extern int (*sysCallsIoctl)(int fileDescriptor, unsigned long int request, void *arg); extern int (*sysCallsPoll)(struct pollfd *pollFd, unsigned long int numberOfFds, int timeout); extern ssize_t (*sysCallsRead)(int fd, void *buf, size_t count); +extern int (*sysCallsFstat)(int fd, struct stat *buf); extern const char *drmVersion; constexpr int fakeFileDescriptor = 123;