Sysman: Redesign event API to effectively use uevents

Earlier implementation of sysman events API was based on file
creation in the filesystem. Whenever a uevent for some event
which needs to be monitored arrive, at that time a file was
created in the filesystem based on some preinstalled udev rules.
This approach was inefficient as it heavily depends over file
system and second with this approach losing events is always a
possibility.

Now with this change, we are removing our dependency over file
creation in filesystem. Rather we will be using libudev library
to monitor the uevents. This approach could also be extended,
when we want to listen to all the uevents for all the gpu
devices present in the system.

Related-To: LOCI-2140
Signed-off-by: Jitendra Sharma <jitendra.sharma@intel.com>

level_zero/CMakeLists.txt

@@ -75,6 +75,18 @@ if(BUILD_WITH_L0)
     endif()
   endif()
 
+  if(UNIX)
+    # Udev library interface
+    find_path(LIBUDEV_INCLUDE_DIR libudev.h)
+    if(LIBUDEV_INCLUDE_DIR)
+      message(STATUS "libudev headers directory: ${LIBUDEV_INCLUDE_DIR}")
+      include_directories(SYSTEM ${LIBUDEV_INCLUDE_DIR})
+      set(LIBUDEV_FOUND TRUE)
+    else()
+      message(STATUS "libudev headers not available. Building without")
+    endif()
+  endif()
+
   # Firmware Update Library
   get_filename_component(IGSC_DIR_tmp "${NEO_SOURCE_DIR}/../igsc" ABSOLUTE)
   if(EXISTS "${IGSC_DIR_tmp}/lib/cmake")

level_zero/doc/BUILD.md

@@ -15,6 +15,8 @@ These instructions have been tested on Ubuntu* and complement those existing for
 To use Fabric related APIs, please build and/or install [libnl-3.7.0](https://www.linuxfromscratch.org/blfs/view/svn/basicnet/libnl.html).
 If installing to a local folder, `-DLIBGENL_INCLUDE_DIR=<local install folder path>/libnl/include/libnl3/` could be passed to the cmake line of NEO build(Please refer top-level BUILD.md).
 
+To use Sysman events API, please build and/or install libudev-dev 
+
 2. Install/build Level Zero loader and Level Zero headers
 
 Install Level Zero loader and headers from [https://github.com/oneapi-src/level-zero/releases](https://github.com/oneapi-src/level-zero/releases).

level_zero/tools/source/sysman/events/events_imp.cpp

@@ -26,7 +26,9 @@ void EventsImp::initEvents() {
     });
 }
 void EventsImp::init() {
-    pOsEvents = OsEvents::create(pOsSysman);
+    if (pOsEvents == nullptr) {
+        pOsEvents = OsEvents::create(pOsSysman);
+    }
     UNRECOVERABLE_IF(nullptr == pOsEvents);
 }
 

level_zero/tools/source/sysman/events/linux/os_events_imp_prelim.cpp

@@ -7,17 +7,23 @@
 
 #include "level_zero/tools/source/sysman/events/linux/os_events_imp_prelim.h"
 
+#include "shared/source/debug_settings/debug_settings_manager.h"
+#include "shared/source/utilities/directory.h"
+
 #include "level_zero/tools/source/sysman/memory/linux/os_memory_imp_prelim.h"
 
 #include "sysman/events/events_imp.h"
 #include "sysman/linux/os_sysman_imp.h"
 
+#include <sys/stat.h>
+
 namespace L0 {
 
-const std::string LinuxEventsImp::deviceMemoryHealth("device_memory_health");
-const std::string LinuxEventsImp::varFs("/var/lib/libze_intel_gpu/");
-const std::string LinuxEventsImp::detachEvent("remove");
-const std::string LinuxEventsImp::attachEvent("add");
+const std::string LinuxEventsImp::add("add");
+const std::string LinuxEventsImp::remove("remove");
+const std::string LinuxEventsImp::change("change");
+const std::string LinuxEventsImp::unbind("unbind");
+const std::string LinuxEventsImp::bind("bind");
 
 static bool checkRasEventOccured(Ras *rasHandle) {
     zes_ras_config_t config = {};
@@ -58,28 +64,24 @@ bool LinuxEventsImp::checkRasEvent(zes_event_type_flags_t &pEvent) {
     return false;
 }
 
-bool LinuxEventsImp::isResetRequired(zes_event_type_flags_t &pEvent) {
-    zes_device_state_t pState = {};
-    pLinuxSysmanImp->getSysmanDeviceImp()->deviceGetState(&pState);
-    if (pState.reset) {
+bool LinuxEventsImp::isResetRequired(void *dev, zes_event_type_flags_t &pEvent) {
+    if (action.compare(change) != 0) {
+        return false;
+    }
+    const char *str;
+    str = pUdevLib->getEventPropertyValue(dev, "RESET_FAILED");
+    if (str && atoi(str) == 1) {
         pEvent |= ZES_EVENT_TYPE_FLAG_DEVICE_RESET_REQUIRED;
         return true;
     }
+
+    // ZES_EVENT_TYPE_FLAG_DEVICE_RESET_REQUIRED event could also be received when, reset Reason is
+    // ZES_RESET_REASON_FLAG_REPAIR.
     return false;
 }
 
 bool LinuxEventsImp::checkDeviceDetachEvent(zes_event_type_flags_t &pEvent) {
-    // When device detach uevent is generated, then L0 udev rules will create a file:
-    // /var/lib/libze_intel_gpu/remove-<ID_PATH_TAG>
-    // For <ID_PATH_TAG>, check comment in LinuxEventsImp::init()
-    const std::string deviceDetachFile = detachEvent + "-" + pciIdPathTag;
-    const std::string deviceDetachFileAbsolutePath = varFs + deviceDetachFile;
-    uint32_t val = 0;
-    auto result = pFsAccess->read(deviceDetachFileAbsolutePath, val);
-    if (result != ZE_RESULT_SUCCESS) {
-        return false;
-    }
-    if (val == 1) {
+    if (action.compare(remove) == 0) {
         pEvent |= ZES_EVENT_TYPE_FLAG_DEVICE_DETACH;
         return true;
     }
@@ -87,68 +89,192 @@ bool LinuxEventsImp::checkDeviceDetachEvent(zes_event_type_flags_t &pEvent) {
 }
 
 bool LinuxEventsImp::checkDeviceAttachEvent(zes_event_type_flags_t &pEvent) {
-    // When device detach uevent is generated, then L0 udev rules will create a file:
-    // /var/lib/libze_intel_gpu/add-<ID_PATH_TAG>
-    // For <ID_PATH_TAG>, check comment in LinuxEventsImp::init()
-    const std::string deviceAttachFile = attachEvent + "-" + pciIdPathTag;
-    const std::string deviceAttachFileAbsolutePath = varFs + deviceAttachFile;
-    uint32_t val = 0;
-    auto result = pFsAccess->read(deviceAttachFileAbsolutePath, val);
-    if (result != ZE_RESULT_SUCCESS) {
-        return false;
-    }
-    if (val == 1) {
+    if (action.compare(add) == 0) {
         pEvent |= ZES_EVENT_TYPE_FLAG_DEVICE_ATTACH;
         return true;
     }
     return false;
 }
 
-bool LinuxEventsImp::checkIfMemHealthChanged(zes_event_type_flags_t &pEvent) {
-    if (currentMemHealth() != memHealthAtEventRegister) {
-        pEvent |= ZES_EVENT_TYPE_FLAG_MEM_HEALTH;
-        return true;
+bool LinuxEventsImp::checkIfMemHealthChanged(void *dev, zes_event_type_flags_t &pEvent) {
+    if (action.compare(change) != 0) {
+        return false;
+    }
+    std::vector<std::string> properties{"MEM_HEALTH_ALARM", "REBOOT_ALARM", "EC_PENDING", "DEGRADED", "EC_FAILED", "SPARING_STATUS_UNKNOWN"};
+    for (auto &property : properties) {
+        const char *propVal = nullptr;
+        propVal = pUdevLib->getEventPropertyValue(dev, property.c_str());
+        if (propVal && atoi(propVal) == 1) {
+            pEvent |= ZES_EVENT_TYPE_FLAG_MEM_HEALTH;
+            return true;
+        }
     }
     return false;
 }
 
-bool LinuxEventsImp::checkIfFabricPortStatusChanged(zes_event_type_flags_t &pEvent) {
-    uint32_t currentFabricEventStatusVal = 0;
-    if (currentFabricEventStatus(currentFabricEventStatusVal) != ZE_RESULT_SUCCESS) {
+bool LinuxEventsImp::checkIfFabricPortStatusChanged(void *dev, zes_event_type_flags_t &pEvent) {
+    if (action.compare(change) != 0) {
         return false;
     }
-    if (currentFabricEventStatusVal != fabricEventTrackAtRegister) {
+
+    const char *str = pUdevLib->getEventPropertyValue(dev, "TYPE");
+    if (str == nullptr) {
+        return false;
+    }
+    const char *expectedStr = "PORT_CHANGE";
+    const size_t expectedStrLen = 11; // length of "PORT_CHANGE" is 11
+    if (strlen(str) != strlen(expectedStr)) {
+        return false;
+    }
+    if (strncmp(str, expectedStr, expectedStrLen) == 0) {
         pEvent |= ZES_EVENT_TYPE_FLAG_FABRIC_PORT_HEALTH;
         return true;
     }
     return false;
 }
 
+ze_result_t LinuxEventsImp::readFabricDeviceStats(const std::string &devicePciPath, struct stat &iafStat) {
+    const std::string iafDirectoryLegacy = "iaf.";
+    const std::string iafDirectory = "i915.iaf.";
+    const std::string fabricIdFile = "/iaf_fabric_id";
+    int fd = -1;
+    std::string path;
+    path.clear();
+    std::vector<std::string> list = NEO::Directory::getFiles(devicePciPath);
+    for (auto entry : list) {
+        if ((entry.find(iafDirectory) != std::string::npos) ||
+            (entry.find(iafDirectoryLegacy) != std::string::npos)) {
+            path = entry + fabricIdFile;
+            break;
+        }
+    }
+    if (path.empty()) {
+        // This device does not have a fabric
+        return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
+    }
+    fd = NEO::SysCalls::open(path.c_str(), O_RDONLY);
+    if (fd < 0) {
+        return ZE_RESULT_ERROR_DEPENDENCY_UNAVAILABLE;
+    }
+    NEO::SysCalls::fstat(fd, &iafStat);
+    NEO::SysCalls::close(fd);
+    return ZE_RESULT_SUCCESS;
+}
+
+bool LinuxEventsImp::listenSystemEvents(zes_event_type_flags_t &pEvent, uint64_t timeout) {
+    bool retval = false;
+    struct pollfd pfd;
+    struct stat iafStat;
+    std::vector<std::string> subsystemList;
+
+    if (pUdevLib == nullptr) {
+        NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr,
+                              "%s", "libudev library instantiation failed\n");
+        return retval;
+    }
+
+    // Get fabric device stats
+    const std::string iafPath = "device/";
+    std::string iafRealPath = {};
+    pLinuxSysmanImp->getSysfsAccess().getRealPath(iafPath, iafRealPath);
+    auto result = readFabricDeviceStats(iafRealPath, iafStat);
+    if (result == ZE_RESULT_SUCCESS) {
+        subsystemList.push_back("platform");
+    }
+
+    subsystemList.push_back("drm");
+    pfd.fd = pUdevLib->registerEventsFromSubsystemAndGetFd(subsystemList);
+    pfd.events = POLLIN;
+
+    auto pDrm = &pLinuxSysmanImp->getDrm();
+    struct stat drmStat;
+    NEO::SysCalls::fstat(pDrm->getFileDescriptor(), &drmStat);
+
+    auto start = std::chrono::steady_clock::now();
+    std::chrono::duration<double, std::milli> timeElapsed;
+    while (NEO::SysCalls::poll(&pfd, 1, static_cast<int>(timeout)) > 0) {
+        // Check again for registered events. Its possible while we are looping for events, registered events are cleared
+        if (!registeredEvents) {
+            return true;
+        }
+
+        dev_t devnum;
+        void *dev = nullptr;
+        dev = pUdevLib->allocateDeviceToReceiveData();
+        if (dev == nullptr) {
+            timeElapsed = std::chrono::steady_clock::now() - start;
+            if (timeout > timeElapsed.count()) {
+                timeout = timeout - timeElapsed.count();
+                continue;
+            } else {
+                break;
+            }
+        }
+
+        devnum = pUdevLib->getEventGenerationSourceDevice(dev);
+
+        if ((memcmp(&drmStat.st_rdev, &devnum, sizeof(dev_t)) == 0) ||
+            (memcmp(&iafStat.st_rdev, &devnum, sizeof(dev_t)) == 0)) {
+            auto eventTypePtr = pUdevLib->getEventType(dev);
+
+            if (eventTypePtr != nullptr) {
+                action = std::string(eventTypePtr);
+                if (registeredEvents & ZES_EVENT_TYPE_FLAG_FABRIC_PORT_HEALTH) {
+                    if (checkIfFabricPortStatusChanged(dev, pEvent)) {
+                        registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_FABRIC_PORT_HEALTH);
+                        retval = true;
+                    }
+                }
+                if (registeredEvents & ZES_EVENT_TYPE_FLAG_DEVICE_DETACH) {
+                    if (checkDeviceDetachEvent(pEvent)) {
+                        registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_DEVICE_DETACH); // After receiving event unregister it
+                        retval = true;
+                    }
+                }
+                if (registeredEvents & ZES_EVENT_TYPE_FLAG_DEVICE_ATTACH) {
+                    if (checkDeviceAttachEvent(pEvent)) {
+                        registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_DEVICE_ATTACH);
+                        retval = true;
+                    }
+                }
+                if (registeredEvents & ZES_EVENT_TYPE_FLAG_DEVICE_RESET_REQUIRED) {
+                    if (isResetRequired(dev, pEvent)) {
+                        registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_DEVICE_RESET_REQUIRED);
+                        retval = true;
+                    }
+                }
+                if (registeredEvents & ZES_EVENT_TYPE_FLAG_MEM_HEALTH) {
+                    if (checkIfMemHealthChanged(dev, pEvent)) {
+                        registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_MEM_HEALTH);
+                        retval = true;
+                    }
+                }
+            }
+        }
+
+        pUdevLib->dropDeviceReference(dev);
+        if (retval) {
+            break;
+        }
+        timeElapsed = std::chrono::steady_clock::now() - start;
+        if (timeout > timeElapsed.count()) {
+            timeout = timeout - timeElapsed.count();
+            continue;
+        } else {
+            break;
+        }
+    }
+
+    return retval;
+}
+
 bool LinuxEventsImp::eventListen(zes_event_type_flags_t &pEvent, uint64_t timeout) {
-    if (registeredEvents & ZES_EVENT_TYPE_FLAG_DEVICE_RESET_REQUIRED) {
-        if (isResetRequired(pEvent)) {
-            registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_DEVICE_RESET_REQUIRED); //After receiving event unregister it
-            return true;
-        }
-    }
-    if (registeredEvents & ZES_EVENT_TYPE_FLAG_DEVICE_DETACH) {
-        if (checkDeviceDetachEvent(pEvent)) {
-            registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_DEVICE_DETACH);
-            return true;
-        }
-    }
-    if (registeredEvents & ZES_EVENT_TYPE_FLAG_DEVICE_ATTACH) {
-        if (checkDeviceAttachEvent(pEvent)) {
-            registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_DEVICE_ATTACH);
-            return true;
-        }
-    }
-    if (registeredEvents & ZES_EVENT_TYPE_FLAG_MEM_HEALTH) {
-        if (checkIfMemHealthChanged(pEvent)) {
-            registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_MEM_HEALTH);
-            return true;
-        }
+    bool retval = false;
+    if (!registeredEvents) {
+        return retval;
     }
+
+    pEvent = 0;
     if ((registeredEvents & ZES_EVENT_TYPE_FLAG_RAS_CORRECTABLE_ERRORS) || (registeredEvents & ZES_EVENT_TYPE_FLAG_RAS_UNCORRECTABLE_ERRORS)) {
         if (checkRasEvent(pEvent)) {
             if (pEvent & ZES_EVENT_TYPE_FLAG_RAS_CORRECTABLE_ERRORS) {
@@ -159,74 +285,33 @@ bool LinuxEventsImp::eventListen(zes_event_type_flags_t &pEvent, uint64_t timeou
             return true;
         }
     }
-    if (registeredEvents & ZES_EVENT_TYPE_FLAG_FABRIC_PORT_HEALTH) {
-        if (checkIfFabricPortStatusChanged(pEvent)) {
-            registeredEvents &= ~(ZES_EVENT_TYPE_FLAG_FABRIC_PORT_HEALTH);
-            return true;
-        }
-    }
-    return false;
+
+    return listenSystemEvents(pEvent, timeout);
 }
 
 ze_result_t LinuxEventsImp::eventRegister(zes_event_type_flags_t events) {
     if (0x7fff < events) {
         return ZE_RESULT_ERROR_INVALID_ENUMERATION;
     }
-    registeredEvents |= events;
-    if (registeredEvents & ZES_EVENT_TYPE_FLAG_MEM_HEALTH) {
-        memHealthAtEventRegister = currentMemHealth();
-    }
-    if (registeredEvents & ZES_EVENT_TYPE_FLAG_FABRIC_PORT_HEALTH) {
-        currentFabricEventStatus(fabricEventTrackAtRegister);
+    if (!events) {
+        // If user is trying to register events with empty events argument, then clear all the registered events
+        registeredEvents = events;
+    } else {
+        // supportedEventMask --> this mask checks for events that supported currently
+        zes_event_type_flags_t supportedEventMask = ZES_EVENT_TYPE_FLAG_FABRIC_PORT_HEALTH | ZES_EVENT_TYPE_FLAG_DEVICE_DETACH |
+                                                    ZES_EVENT_TYPE_FLAG_DEVICE_ATTACH | ZES_EVENT_TYPE_FLAG_DEVICE_RESET_REQUIRED |
+                                                    ZES_EVENT_TYPE_FLAG_MEM_HEALTH | ZES_EVENT_TYPE_FLAG_RAS_CORRECTABLE_ERRORS |
+                                                    ZES_EVENT_TYPE_FLAG_RAS_UNCORRECTABLE_ERRORS;
+        registeredEvents |= (events & supportedEventMask);
     }
     return ZE_RESULT_SUCCESS;
 }
 
-ze_result_t LinuxEventsImp::currentFabricEventStatus(uint32_t &val) {
-    // When Fabric port status change uevent is generated, then L0 udev rules will create a file:
-    // /var/lib/libze_intel_gpu/fabric-<ID_PATH_TAG>
-    // For <ID_PATH_TAG>, check comment in LinuxEventsImp::init()
-    const std::string fabric = "fabric";
-    const std::string fabricEventFile = fabric + "-" + pciIdPathTag;
-    const std::string fabricEventFileAbsolutePath = varFs + fabricEventFile;
-    return pFsAccess->read(fabricEventFileAbsolutePath, val);
-}
-
-zes_mem_health_t LinuxEventsImp::currentMemHealth() {
-    std::string memHealth;
-    ze_result_t result = pSysfsAccess->read(deviceMemoryHealth, memHealth);
-    if (ZE_RESULT_SUCCESS != result) {
-        return ZES_MEM_HEALTH_UNKNOWN;
-    }
-
-    auto health = i915ToL0MemHealth.find(memHealth);
-    if (health != i915ToL0MemHealth.end()) {
-        return i915ToL0MemHealth.at(memHealth);
-    }
-    return ZES_MEM_HEALTH_UNKNOWN;
-}
-
-void LinuxEventsImp::getPciIdPathTag() {
-    std::string bdfDir;
-    ze_result_t result = pSysfsAccess->readSymLink("device", bdfDir);
-    if (ZE_RESULT_SUCCESS != result) {
-        return;
-    }
-    const auto loc = bdfDir.find_last_of('/');
-    auto bdf = bdfDir.substr(loc + 1);
-    std::replace(bdf.begin(), bdf.end(), ':', '_');
-    std::replace(bdf.begin(), bdf.end(), '.', '_');
-    // ID_PATH_TAG key is received when uevent related to device add/remove is generated.
-    // Example of ID_PATH_TAG is:
-    // ID_PATH_TAG=pci-0000_8c_00_0
-    pciIdPathTag = "pci-" + bdf;
-}
-
 LinuxEventsImp::LinuxEventsImp(OsSysman *pOsSysman) {
     pLinuxSysmanImp = static_cast<LinuxSysmanImp *>(pOsSysman);
     pSysfsAccess = &pLinuxSysmanImp->getSysfsAccess();
     pFsAccess = &pLinuxSysmanImp->getFsAccess();
-    getPciIdPathTag();
+    pUdevLib = pLinuxSysmanImp->getUdevLibHandle();
 }
 
 OsEvents *OsEvents::create(OsSysman *pOsSysman) {

level_zero/tools/source/sysman/events/linux/os_events_imp_prelim.h

@@ -8,6 +8,7 @@
 #pragma once
 #include "level_zero/tools/source/sysman/events/os_events.h"
 #include "level_zero/tools/source/sysman/linux/os_sysman_imp.h"
+#include "level_zero/tools/source/sysman/linux/udev/udev_lib.h"
 
 namespace L0 {
 
@@ -21,27 +22,27 @@ class LinuxEventsImp : public OsEvents, NEO::NonCopyableOrMovableClass {
 
   protected:
     LinuxSysmanImp *pLinuxSysmanImp = nullptr;
-    void getPciIdPathTag();
-    zes_mem_health_t currentMemHealth();
-    ze_result_t currentFabricEventStatus(uint32_t &val);
-    bool isResetRequired(zes_event_type_flags_t &pEvent);
+    bool isResetRequired(void *dev, zes_event_type_flags_t &pEvent);
     bool checkDeviceDetachEvent(zes_event_type_flags_t &pEvent);
     bool checkDeviceAttachEvent(zes_event_type_flags_t &pEvent);
-    bool checkIfMemHealthChanged(zes_event_type_flags_t &pEvent);
-    bool checkIfFabricPortStatusChanged(zes_event_type_flags_t &pEvent);
+    bool checkIfMemHealthChanged(void *dev, zes_event_type_flags_t &pEvent);
+    bool checkIfFabricPortStatusChanged(void *dev, zes_event_type_flags_t &pEvent);
     bool checkRasEvent(zes_event_type_flags_t &pEvent);
-    std::string pciIdPathTag;
-    zes_mem_health_t memHealthAtEventRegister = ZES_MEM_HEALTH_UNKNOWN;
+    ze_result_t readFabricDeviceStats(const std::string &devicePciPath, struct stat &iafStat);
+    bool listenSystemEvents(zes_event_type_flags_t &pEvent, uint64_t timeout);
     uint32_t fabricEventTrackAtRegister = 0;
+    L0::UdevLib *pUdevLib = nullptr;
+    zes_event_type_flags_t registeredEvents = 0;
 
   private:
     FsAccess *pFsAccess = nullptr;
     SysfsAccess *pSysfsAccess = nullptr;
-    static const std::string varFs;
-    static const std::string detachEvent;
-    static const std::string attachEvent;
-    static const std::string deviceMemoryHealth;
-    zes_event_type_flags_t registeredEvents = 0;
+    std::string action;
+    static const std::string add;
+    static const std::string remove;
+    static const std::string change;
+    static const std::string unbind;
+    static const std::string bind;
 };
 
 } // namespace L0

level_zero/tools/source/sysman/linux/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2020-2021 Intel Corporation
+# Copyright (C) 2020-2022 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 #
@@ -22,21 +22,3 @@ add_subdirectories()
 
 # Make our source files visible to parent
 set_property(GLOBAL PROPERTY L0_SRCS_TOOLS_SYSMAN_LINUX ${L0_SRCS_TOOLS_SYSMAN_LINUX})
-
-if(L0_INSTALL_UDEV_RULES)
-  install(
-          FILES "${CMAKE_CURRENT_SOURCE_DIR}/udev/99-drm_ze_intel_gpu.rules"
-          DESTINATION ${UDEV_RULES_DIR}
-          COMPONENT ${PROJECT_NAME}
-  )
-  install(
-          FILES "${CMAKE_CURRENT_SOURCE_DIR}/udev/wedged_file"
-          DESTINATION "/var/lib/libze_intel_gpu/"
-          COMPONENT ${PROJECT_NAME}
-  )
-  install(
-          FILES "${CMAKE_CURRENT_SOURCE_DIR}/udev/pci_bind_status_file"
-          DESTINATION "/var/lib/libze_intel_gpu/"
-          COMPONENT ${PROJECT_NAME}
-  )
-endif()

level_zero/tools/source/sysman/linux/os_sysman_imp.cpp

@@ -88,6 +88,13 @@ FirmwareUtil *LinuxSysmanImp::getFwUtilInterface() {
     return pFwUtilInterface;
 }
 
+L0::UdevLib *LinuxSysmanImp::getUdevLibHandle() {
+    if (pUdevLib == nullptr) {
+        pUdevLib = UdevLib::create();
+    }
+    return pUdevLib;
+}
+
 FsAccess &LinuxSysmanImp::getFsAccess() {
     UNRECOVERABLE_IF(nullptr == pFsAccess);
     return *pFsAccess;
@@ -224,6 +231,10 @@ LinuxSysmanImp::~LinuxSysmanImp() {
         delete pPmuInterface;
         pPmuInterface = nullptr;
     }
+    if (nullptr != pUdevLib) {
+        delete pUdevLib;
+        pUdevLib = nullptr;
+    }
     releaseFwUtilInterface();
     releasePmtObject();
 }

level_zero/tools/source/sysman/linux/os_sysman_imp.h

@@ -14,6 +14,7 @@
 #include "level_zero/tools/source/sysman/linux/fs_access.h"
 #include "level_zero/tools/source/sysman/linux/pmt/pmt.h"
 #include "level_zero/tools/source/sysman/linux/pmu/pmu_imp.h"
+#include "level_zero/tools/source/sysman/linux/udev/udev_lib.h"
 #include "level_zero/tools/source/sysman/sysman_imp.h"
 
 #include <linux/pci_regs.h>
@@ -44,6 +45,7 @@ class LinuxSysmanImp : public OsSysman, NEO::NonCopyableOrMovableClass {
 
     ze_result_t init() override;
 
+    L0::UdevLib *getUdevLibHandle();
     PmuInterface *getPmuInterface();
     FirmwareUtil *getFwUtilInterface();
     FsAccess &getFsAccess();
@@ -88,6 +90,7 @@ class LinuxSysmanImp : public OsSysman, NEO::NonCopyableOrMovableClass {
     NEO::Drm *pDrm = nullptr;
     PmuInterface *pPmuInterface = nullptr;
     FirmwareUtil *pFwUtilInterface = nullptr;
+    L0::UdevLib *pUdevLib = nullptr;
     std::map<uint32_t, L0::PlatformMonitoringTech *> mapOfSubDeviceIdToPmtObject;
     ze_result_t initLocalDeviceAndDrmHandles();
 

level_zero/tools/source/sysman/linux/udev/99-drm_ze_intel_gpu.rules

@@ -1,35 +0,0 @@
-#
-# Copyright (C) 2020-2021 Intel Corporation
-#
-# SPDX-License-Identifier: MIT
-#
-
-ACTION=="change", \
-SUBSYSTEM=="drm", \
-ENV{RESET_FAILED}=="1", \
-ENV{RESET_UNIT}=="gt", \
-RUN+="/bin/sh -c 'echo 1 > /var/lib/libze_intel_gpu/wedged_file'"
-ACTION=="unbind", \
-SUBSYSTEM=="drm", \
-RUN+="/bin/sh -c 'echo 0 > /var/lib/libze_intel_gpu/pci_bind_status_file'"
-ACTION=="bind", \
-SUBSYSTEM=="drm", \
-RUN+="/bin/sh -c 'echo 1 > /var/lib/libze_intel_gpu/pci_bind_status_file'", \
-RUN+="/bin/sh -c 'echo 0 > /var/lib/libze_intel_gpu/wedged_file'"
-ACTION=="remove", \
-SUBSYSTEM=="drm", \
-ENV{DEVNAME}=="/dev/dri/card*", \
-ENV{ID_PATH_TAG}=="pci-*", \
-RUN+="/bin/sh -c 'touch /var/lib/libze_intel_gpu/remove-$env{ID_PATH_TAG}; echo 1 > /var/lib/libze_intel_gpu/remove-$env{ID_PATH_TAG}; rm /var/lib/libze_intel_gpu/add-$env{ID_PATH_TAG};'"
-ACTION=="add", \
-SUBSYSTEM=="drm", \
-ENV{DEVNAME}=="/dev/dri/card*", \
-ENV{ID_PATH_TAG}=="pci-*", \
-RUN+="/bin/sh -c 'touch /var/lib/libze_intel_gpu/add-$env{ID_PATH_TAG}; echo 1 > /var/lib/libze_intel_gpu/add-$env{ID_PATH_TAG}; rm /var/lib/libze_intel_gpu/remove-$env{ID_PATH_TAG}; echo 0 > /var/lib/libze_intel_gpu/wedged_file;'"
-ACTION=="change", \
-SUBSYSTEM=="platform", \
-ENV{DEVTYPE}=="mfd_device", \
-ENV{DRIVER}=="iaf", \
-ENV{TYPE}=="PORT_CHANGE", \
-ENV{ID_PATH_TAG}=="pci-*", \
-RUN+="/bin/sh -c 'pci_id_path_tag=${env{ID_PATH_TAG}::16}; filename=/var/lib/libze_intel_gpu/fabric-${pci_id_path_tag}; touch ${filename}; echo $(($(cat ${filename}) + 1)) > ${filename};'"

level_zero/tools/source/sysman/linux/udev/CMakeLists.txt

@@ -0,0 +1,29 @@
+#
+# Copyright (C) 2022 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+if(LIBUDEV_FOUND)
+  set(L0_SRCS_TOOLS_SYSMAN_LINUX_UDEV_LIB
+      ${CMAKE_CURRENT_SOURCE_DIR}/udev_lib_imp.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/udev_lib_imp.h
+      ${CMAKE_CURRENT_SOURCE_DIR}/udev_lib.h
+  )
+  add_subdirectories()
+else()
+  set(L0_SRCS_TOOLS_SYSMAN_LINUX_UDEV_LIB
+      ${CMAKE_CURRENT_SOURCE_DIR}/udev_lib_imp_stub.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/udev_lib.h
+  )
+endif()
+
+if(UNIX)
+  target_sources(${L0_STATIC_LIB_NAME}
+                 PRIVATE
+                 ${L0_SRCS_TOOLS_SYSMAN_LINUX_UDEV_LIB}
+  )
+endif()
+
+# Make our source files visible to parent
+set_property(GLOBAL PROPERTY L0_SRCS_TOOLS_SYSMAN_LINUX_UDEV_LIB ${L0_SRCS_TOOLS_SYSMAN_LINUX_UDEV_LIB})

level_zero/tools/source/sysman/linux/udev/udev_lib.h

@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+namespace L0 {
+class UdevLib {
+  public:
+    UdevLib() = default;
+    static UdevLib *create();
+    virtual int registerEventsFromSubsystemAndGetFd(std::vector<std::string> &subsystemList) = 0;
+    virtual dev_t getEventGenerationSourceDevice(void *dev) = 0;
+    virtual const char *getEventType(void *dev) = 0;
+    virtual const char *getEventPropertyValue(void *dev, const char *key) = 0;
+    virtual void *allocateDeviceToReceiveData() = 0;
+    virtual void dropDeviceReference(void *dev) = 0;
+    virtual ~UdevLib() {}
+};
+} // namespace L0

level_zero/tools/source/sysman/linux/udev/udev_lib_imp.cpp

@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/tools/source/sysman/linux/udev/udev_lib_imp.h"
+
+#include "shared/source/helpers/debug_helpers.h"
+#include "shared/source/utilities/directory.h"
+
+namespace L0 {
+static constexpr std::string_view libUdevlFile = "libudev.so";
+const std::string udevNewRoutine = "udev_new";
+const std::string udevMonitorNewFromNetlinkRoutine = "udev_monitor_new_from_netlink";
+const std::string udevMonitorFilterAddMatchSubsystemDevtypeRoutine = "udev_monitor_filter_add_match_subsystem_devtype";
+const std::string udevMonitorEnableReceivingRoutine = "udev_monitor_enable_receiving";
+const std::string udevMonitorGetFdRoutine = "udev_monitor_get_fd";
+const std::string udevMonitorReceiveDeviceRoutine = "udev_monitor_receive_device";
+const std::string udevDeviceGetDevnumRoutine = "udev_device_get_devnum";
+const std::string udevDeviceGetActionRoutine = "udev_device_get_action";
+const std::string udevDeviceGetPropertyValueRoutine = "udev_device_get_property_value";
+const std::string udevDeviceUnrefRoutine = "udev_device_unref";
+
+bool UdevLibImp::loadEntryPoints() {
+    bool ok = getSymbolAddr(udevMonitorNewFromNetlinkRoutine, pUdevMonitorNewFromNetlinkEntry);
+    ok = ok && getSymbolAddr(udevNewRoutine, pUdevNewEntry);
+    ok = ok && getSymbolAddr(udevMonitorFilterAddMatchSubsystemDevtypeRoutine, pUdevMonitorFilterAddMatchSubsystemDevtypeEntry);
+    ok = ok && getSymbolAddr(udevMonitorEnableReceivingRoutine, pUdevMonitorEnableReceivingEntry);
+    ok = ok && getSymbolAddr(udevMonitorGetFdRoutine, pUdevMonitorGetFdEntry);
+    ok = ok && getSymbolAddr(udevMonitorReceiveDeviceRoutine, pUdevMonitorReceiveDeviceEntry);
+    ok = ok && getSymbolAddr(udevDeviceGetDevnumRoutine, pUdevDeviceGetDevnumEntry);
+    ok = ok && getSymbolAddr(udevDeviceGetActionRoutine, pUdevDeviceGetActionEntry);
+    ok = ok && getSymbolAddr(udevDeviceGetPropertyValueRoutine, pUdevDeviceGetPropertyValueEntry);
+    ok = ok && getSymbolAddr(udevDeviceUnrefRoutine, pUdevDeviceUnrefEntry);
+
+    return ok;
+}
+
+const char *UdevLibImp::getEventPropertyValue(void *dev, const char *key) {
+    return pUdevDeviceGetPropertyValueEntry(reinterpret_cast<struct udev_device *>(dev), key);
+}
+
+const char *UdevLibImp::getEventType(void *dev) {
+    return pUdevDeviceGetActionEntry(reinterpret_cast<struct udev_device *>(dev));
+}
+
+void *UdevLibImp::allocateDeviceToReceiveData() {
+    struct udev_device *dev = pUdevMonitorReceiveDeviceEntry(mon);
+    return reinterpret_cast<void *>(dev);
+}
+
+dev_t UdevLibImp::getEventGenerationSourceDevice(void *dev) {
+    return pUdevDeviceGetDevnumEntry(reinterpret_cast<struct udev_device *>(dev));
+}
+
+int UdevLibImp::registerEventsFromSubsystemAndGetFd(std::vector<std::string> &subsystemList) {
+    for (const auto &subsystem : subsystemList) {
+        pUdevMonitorFilterAddMatchSubsystemDevtypeEntry(mon, subsystem.c_str(), NULL);
+    }
+    if (pUdevMonitorEnableReceivingEntry(mon) != 0) {
+        return -1;
+    }
+    return pUdevMonitorGetFdEntry(mon);
+}
+
+void UdevLibImp::dropDeviceReference(void *dev) {
+    pUdevDeviceUnrefEntry(reinterpret_cast<struct udev_device *>(dev));
+}
+
+bool UdevLibImp::init() {
+    mon = pUdevMonitorNewFromNetlinkEntry(pUdevNewEntry(), "kernel");
+    return (mon != nullptr);
+}
+
+UdevLibImp::~UdevLibImp(){};
+
+UdevLib *UdevLib::create() {
+    UdevLibImp *pUdevLib = new UdevLibImp();
+    UNRECOVERABLE_IF(nullptr == pUdevLib);
+    pUdevLib->libraryHandle.reset(NEO::OsLibrary::load(std::string(libUdevlFile)));
+    if (pUdevLib->libraryHandle == nullptr || pUdevLib->loadEntryPoints() == false || !pUdevLib->init()) {
+        delete pUdevLib;
+        return nullptr;
+    }
+    return static_cast<UdevLib *>(pUdevLib);
+}
+} // namespace L0

level_zero/tools/source/sysman/linux/udev/udev_lib_imp.h

@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "shared/source/helpers/non_copyable_or_moveable.h"
+#include "shared/source/os_interface/os_library.h"
+
+#include "level_zero/tools/source/sysman/linux/udev/udev_lib.h"
+
+#include <cinttypes>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+namespace L0 {
+
+typedef struct udev *(*pUdevNew)(void);
+typedef struct udev_monitor *(*pUdevMonitorNewFromNetlink)(struct udev *, const char *);
+typedef int (*pUdevMonitorFilterAddMatchSubsystemDevtype)(struct udev_monitor *udev_monitor, const char *subsystem, const char *devtype);
+typedef int (*pUdevMonitorEnableReceiving)(struct udev_monitor *udev_monitor);
+typedef int (*pUdevMonitorGetFd)(struct udev_monitor *udev_monitor);
+typedef struct udev_device *(*pUdevMonitorReceiveDevice)(struct udev_monitor *udev_monitor);
+typedef dev_t (*pUdevDeviceGetDevnum)(struct udev_device *udev_device);
+typedef const char *(*pUdevDeviceGetAction)(struct udev_device *udev_device);
+typedef const char *(*pUdevDeviceGetPropertyValue)(struct udev_device *udev_device, const char *key);
+typedef struct udev_device *(*pUdevDeviceUnref)(struct udev_device *udev_device);
+
+class UdevLibImp : public UdevLib {
+  public:
+    UdevLibImp() = default;
+    ~UdevLibImp() override;
+    std::unique_ptr<NEO::OsLibrary> libraryHandle;
+    bool loadEntryPoints();
+    template <class T>
+    bool getSymbolAddr(const std::string name, T &proc) {
+        void *addr = libraryHandle->getProcAddress(name);
+        proc = reinterpret_cast<T>(addr);
+        return nullptr != proc;
+    }
+    bool init();
+    int registerEventsFromSubsystemAndGetFd(std::vector<std::string> &subsystemList) override;
+    dev_t getEventGenerationSourceDevice(void *dev) override;
+    const char *getEventType(void *dev) override;
+    const char *getEventPropertyValue(void *dev, const char *key) override;
+    void *allocateDeviceToReceiveData() override;
+    void dropDeviceReference(void *dev) override;
+
+  protected:
+    pUdevNew pUdevNewEntry = nullptr;
+    pUdevMonitorNewFromNetlink pUdevMonitorNewFromNetlinkEntry = nullptr;
+    pUdevMonitorFilterAddMatchSubsystemDevtype pUdevMonitorFilterAddMatchSubsystemDevtypeEntry = nullptr;
+    pUdevMonitorEnableReceiving pUdevMonitorEnableReceivingEntry = nullptr;
+    pUdevMonitorGetFd pUdevMonitorGetFdEntry = nullptr;
+    pUdevMonitorReceiveDevice pUdevMonitorReceiveDeviceEntry = nullptr;
+    pUdevDeviceGetDevnum pUdevDeviceGetDevnumEntry = nullptr;
+    pUdevDeviceGetAction pUdevDeviceGetActionEntry = nullptr;
+    pUdevDeviceGetPropertyValue pUdevDeviceGetPropertyValueEntry = nullptr;
+    pUdevDeviceUnref pUdevDeviceUnrefEntry = nullptr;
+
+  private:
+    struct udev_monitor *mon = nullptr;
+};
+
+} // namespace L0

level_zero/tools/source/sysman/linux/udev/udev_lib_imp_stub.cpp

@@ -0,0 +1,15 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/tools/source/sysman/linux/udev/udev_lib.h"
+
+namespace L0 {
+
+UdevLib *UdevLib::create() {
+    return nullptr;
+}
+} // namespace L0

level_zero/tools/test/unit_tests/sources/sysman/linux/mock_sysman_fixture.h

@@ -46,6 +46,7 @@ class PublicLinuxSysmanImp : public L0::LinuxSysmanImp {
     using LinuxSysmanImp::pPmuInterface;
     using LinuxSysmanImp::pProcfsAccess;
     using LinuxSysmanImp::pSysfsAccess;
+    using LinuxSysmanImp::pUdevLib;
 };
 
 class SysmanDeviceFixture : public DeviceFixture, public ::testing::Test {

scripts/packaging/l0_gpu_driver/rhel_8/SPECS/l0_gpu_driver.spec

@@ -84,9 +84,6 @@ cp -pvR %{_sourcedir}/copyright %{buildroot}/usr/share/doc/intel-level-zero-gpu/
 %files
 %defattr(-,root,root)
 %{_libdir}/libze_intel_gpu.so.*
-%{_sharedstatedir}/libze_intel_gpu/pci_bind_status_file
-%{_sharedstatedir}/libze_intel_gpu/wedged_file
-%{_sysconfdir}/udev/rules.d/99-drm_ze_intel_gpu.rules
 /usr/share/doc/intel-level-zero-gpu/copyright
 %config(noreplace)
 

scripts/packaging/l0_gpu_driver/sles_15.3/SPECS/l0_gpu_driver.spec

@@ -94,9 +94,6 @@ cp -pvR %{_sourcedir}/copyright %{buildroot}/usr/share/doc/intel-level-zero-gpu%
 %files -n intel-level-zero-gpu%{?name_suffix}
 %defattr(-,root,root)
 %{_libdir}/libze_intel_gpu.so.*
-%{_sharedstatedir}/libze_intel_gpu/pci_bind_status_file
-%{_sharedstatedir}/libze_intel_gpu/wedged_file
-%{_sysconfdir}/udev/rules.d/99-drm_ze_intel_gpu.rules
 /usr/share/doc/intel-level-zero-gpu%{?name_suffix}/copyright
 %config(noreplace)
 

shared/test/common/os_interface/linux/sys_calls_linux_ult.cpp

@@ -56,6 +56,7 @@ int (*sysCallsReadlink)(const char *path, char *buf, size_t bufsize) = nullptr;
 int (*sysCallsIoctl)(int fileDescriptor, unsigned long int request, void *arg) = nullptr;
 int (*sysCallsPoll)(struct pollfd *pollFd, unsigned long int numberOfFds, int timeout) = nullptr;
 ssize_t (*sysCallsRead)(int fd, void *buf, size_t count) = nullptr;
+int (*sysCallsFstat)(int fd, struct stat *buf) = nullptr;
 
 int close(int fileDescriptor) {
     closeFuncCalled++;
@@ -169,6 +170,9 @@ int poll(struct pollfd *pollFd, unsigned long int numberOfFds, int timeout) {
 }
 
 int fstat(int fd, struct stat *buf) {
+    if (sysCallsFstat != nullptr) {
+        return sysCallsFstat(fd, buf);
+    }
     return fstatFuncRetVal;
 }
 

shared/test/common/os_interface/linux/sys_calls_linux_ult.h

@@ -9,6 +9,7 @@
 
 #include <iostream>
 #include <poll.h>
+#include <sys/stat.h>
 
 namespace NEO {
 namespace SysCalls {
@@ -19,6 +20,7 @@ extern int (*sysCallsReadlink)(const char *path, char *buf, size_t bufsize);
 extern int (*sysCallsIoctl)(int fileDescriptor, unsigned long int request, void *arg);
 extern int (*sysCallsPoll)(struct pollfd *pollFd, unsigned long int numberOfFds, int timeout);
 extern ssize_t (*sysCallsRead)(int fd, void *buf, size_t count);
+extern int (*sysCallsFstat)(int fd, struct stat *buf);
 
 extern const char *drmVersion;
 constexpr int fakeFileDescriptor = 123;