Implement zetSysmanDeviceReset

Implemented function level reset.

Implementation is:
Make sure we are root (otherwise, return insufficient permissions)
Make sure no one has the device open
    (otherwise, return hande object in use)
Close our file handle
Unbind the device from the kernel driver
Make sure no one still has the device open
    (otherwise, kill them)
Perform function level reset (FLR)
Rebind the device to the kernel driver

Change-Id: Ic57b95487e73b5a5f2d03e619d813bf4199adf40
Signed-off-by: Bill Jordan <bill.jordan@intel.com>

level_zero/tools/source/sysman/frequency/linux/os_frequency_imp.cpp

@@ -12,8 +12,8 @@
 
 #include "sysman/frequency/frequency_imp.h"
 #include "sysman/frequency/os_frequency.h"
+#include "sysman/linux/fs_access.h"
 #include "sysman/linux/os_sysman_imp.h"
-#include "sysman/linux/sysfs_access.h"
 
 namespace L0 {
 

level_zero/tools/source/sysman/linux/CMakeLists.txt

@@ -8,7 +8,7 @@ set(L0_SRCS_TOOLS_SYSMAN_LINUX
     ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
     ${CMAKE_CURRENT_SOURCE_DIR}/os_sysman_imp.h
     ${CMAKE_CURRENT_SOURCE_DIR}/os_sysman_imp.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/sysfs_access.cpp)
+    ${CMAKE_CURRENT_SOURCE_DIR}/fs_access.cpp)
 
 if(UNIX)
 target_sources(${L0_STATIC_LIB_NAME}
@@ -18,4 +18,4 @@ target_sources(${L0_STATIC_LIB_NAME}
 endif()
 
 # Make our source files visible to parent
-set_property(GLOBAL PROPERTY L0_SRCS_TOOLS_SYSMAN_LINUX ${L0_SRCS_TOOLS_SYSMAN_LINUX})
+set_property(GLOBAL PROPERTY L0_SRCS_TOOLS_SYSMAN_LINUX ${L0_SRCS_TOOLS_SYSMAN_LINUX})

level_zero/tools/source/sysman/linux/fs_access.cpp

@@ -0,0 +1,414 @@
+/*
+ * Copyright (C) 2020 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/tools/source/sysman/linux/fs_access.h"
+
+#include <climits>
+
+#include <array>
+#include <cerrno>
+#include <cstdio>
+#include <cstdlib>
+#include <dirent.h>
+#include <unistd.h>
+
+namespace L0 {
+
+static ze_result_t getResult(int err) {
+    if ((EPERM == err) || (EACCES == err)) {
+        return ZE_RESULT_ERROR_INSUFFICIENT_PERMISSIONS;
+    } else if (ENOENT == err) {
+        return ZE_RESULT_ERROR_UNKNOWN;
+    } else {
+        return ZE_RESULT_ERROR_UNKNOWN;
+    }
+}
+
+// Generic Filesystem Access
+FsAccess::FsAccess() {
+}
+
+FsAccess *FsAccess::create() {
+    return new FsAccess();
+}
+
+ze_result_t FsAccess::read(const std::string file, std::string &val) {
+    // Read a single line from text file without trailing newline
+    std::ifstream fs;
+    val.clear();
+
+    fs.open(file.c_str());
+    if (fs.fail()) {
+        return getResult(errno);
+    }
+    fs >> val;
+    if (fs.fail()) {
+        fs.close();
+        return getResult(errno);
+    }
+    fs.close();
+    // Strip trailing newline
+    if (val.back() == '\n') {
+        val.pop_back();
+    }
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t FsAccess::read(const std::string file, std::vector<std::string> &val) {
+    // Read a entire text file, one line per vector entry
+    std::string line;
+    std::ifstream fs;
+    val.clear();
+
+    fs.open(file.c_str());
+    if (fs.fail()) {
+        return getResult(errno);
+    }
+    while (std::getline(fs, line)) {
+        if (fs.fail()) {
+            fs.close();
+            return getResult(errno);
+        }
+        if (line.back() == '\n') {
+            line.pop_back();
+        }
+        val.push_back(line);
+    }
+    fs.close();
+
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t FsAccess::write(const std::string file, const std::string val) {
+    std::ofstream sysfs;
+
+    sysfs.open(file.c_str());
+    if (sysfs.fail()) {
+        return getResult(errno);
+    }
+    sysfs << val << std::endl;
+    if (sysfs.fail()) {
+        sysfs.close();
+        return getResult(errno);
+    }
+    sysfs.close();
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t FsAccess::canRead(const std::string file) {
+    if (access(file.c_str(), F_OK)) {
+        return ZE_RESULT_ERROR_UNKNOWN;
+    }
+    if (access(file.c_str(), R_OK)) {
+        return ZE_RESULT_ERROR_INSUFFICIENT_PERMISSIONS;
+    }
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t FsAccess::canWrite(const std::string file) {
+    if (access(file.c_str(), F_OK)) {
+        return ZE_RESULT_ERROR_UNKNOWN;
+    }
+    if (access(file.c_str(), W_OK)) {
+        return ZE_RESULT_ERROR_INSUFFICIENT_PERMISSIONS;
+    }
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t FsAccess::readSymLink(const std::string path, std::string &val) {
+    // returns the value of symlink at path
+    char buf[PATH_MAX];
+    ssize_t len = ::readlink(path.c_str(), buf, PATH_MAX - 1);
+    if (len < 0) {
+        return getResult(errno);
+    }
+    buf[len] = '\0';
+    val = std::string(buf);
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t FsAccess::getRealPath(const std::string path, std::string &val) {
+    // returns the real file path after resolving all symlinks in path
+    char buf[PATH_MAX];
+    char *realPath = ::realpath(path.c_str(), buf);
+    if (!realPath) {
+        return getResult(errno);
+    }
+    val = std::string(buf);
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t FsAccess::listDirectory(const std::string path, std::vector<std::string> &list) {
+    list.clear();
+    ::DIR *procDir = ::opendir(path.c_str());
+    if (!procDir) {
+        return ZE_RESULT_ERROR_UNKNOWN;
+    }
+    struct ::dirent *ent;
+    while (NULL != (ent = ::readdir(procDir))) {
+        // Ignore . and ..
+        std::string name = std::string(ent->d_name);
+        if (!name.compare(".") || !name.compare("..")) {
+            continue;
+        }
+        list.push_back(std::string(ent->d_name));
+    }
+    ::closedir(procDir);
+    return ZE_RESULT_SUCCESS;
+}
+
+std::string FsAccess::getBaseName(const std::string path) {
+    size_t pos = path.rfind("/");
+    if (std::string::npos == pos) {
+        return path;
+    }
+    return path.substr(pos + 1, std::string::npos);
+}
+
+std::string FsAccess::getDirName(const std::string path) {
+    size_t pos = path.rfind("/");
+    if (std::string::npos == pos) {
+        return std::string("");
+    }
+    // Include trailing slash
+    return path.substr(0, pos);
+}
+
+ze_bool_t FsAccess::fileExists(const std::string file) {
+    struct stat sb;
+    if (stat(file.c_str(), &sb) == 0) {
+        return true;
+    }
+    return false;
+}
+
+// Procfs Access
+const std::string ProcfsAccess::procDir = "/proc/";
+const std::string ProcfsAccess::fdDir = "/fd/";
+
+std::string ProcfsAccess::fullPath(const ::pid_t pid) {
+    // Returns the full path for proc entry for process pid
+    return std::string(procDir + std::to_string(pid));
+}
+
+std::string ProcfsAccess::fdDirPath(const ::pid_t pid) {
+    // Returns the full path to file descritpor directory
+    // for process pid
+    return std::string(fullPath(pid) + fdDir);
+}
+
+std::string ProcfsAccess::fullFdPath(const ::pid_t pid, const int fd) {
+    // Returns the full path for filedescriptor fd
+    // for process pid
+    return std::string(fdDirPath(pid) + std::to_string(fd));
+}
+
+ProcfsAccess *ProcfsAccess::create() {
+    return new ProcfsAccess();
+}
+
+ze_result_t ProcfsAccess::listProcesses(std::vector<::pid_t> &list) {
+    // Returns a vector with all the active process ids in the system
+    list.clear();
+    std::vector<std::string> dir;
+    ze_result_t result = FsAccess::listDirectory(procDir, dir);
+    if (ZE_RESULT_SUCCESS != result) {
+        return result;
+    }
+    for (auto &&file : dir) {
+        ::pid_t pid;
+        std::istringstream stream(file);
+        stream >> pid;
+        if (stream.fail()) {
+            // Non numeric filename, not a process, skip
+            continue;
+        }
+        list.push_back(pid);
+    }
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t ProcfsAccess::getFileDescriptors(const ::pid_t pid, std::vector<int> &list) {
+    // Returns a vector with all the filedescriptor numbers opened by a pid
+    list.clear();
+    std::vector<std::string> dir;
+    ze_result_t result = FsAccess::listDirectory(fdDirPath(pid), dir);
+    if (ZE_RESULT_SUCCESS != result) {
+        return result;
+    }
+    for (auto &&file : dir) {
+        int fd;
+        std::istringstream stream(file);
+        stream >> fd;
+        if (stream.fail()) {
+            // Non numeric filename, not a file descriptor
+            continue;
+        }
+        list.push_back(fd);
+    }
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t ProcfsAccess::getFileName(const ::pid_t pid, const int fd, std::string &val) {
+    // Given a process id and a file descriptor number
+    // return full name of the open file.
+    // NOTE: For sockets, the name will be of the format "socket:[nnnnnnn]"
+    return FsAccess::readSymLink(fullFdPath(pid, fd), val);
+}
+
+::pid_t ProcfsAccess::myProcessId() {
+    return ::getpid();
+}
+
+// Sysfs Access
+const std::string SysfsAccess::drmPath = "/sys/class/drm/";
+const std::string SysfsAccess::devicesPath = "device/drm/";
+const std::string SysfsAccess::primaryDevName = "card";
+const std::string SysfsAccess::drmDriverDevNodeDir = "/dev/dri/";
+const std::string SysfsAccess::intelGpuBindEntry = "/sys/bus/pci/drivers/i915/bind";
+const std::string SysfsAccess::intelGpuUnbindEntry = "/sys/bus/pci/drivers/i915/unbind";
+
+std::string SysfsAccess::fullPath(const std::string file) {
+    // Prepend sysfs directory path for this device
+    return std::string(dirname + file);
+}
+
+SysfsAccess::SysfsAccess(const std::string dev) {
+    // dev could be either /dev/dri/cardX or /dev/dri/renderDX
+    std::string fileName = FsAccess::getBaseName(dev);
+    std::string devicesDir = drmPath + fileName + std::string("/") + devicesPath;
+
+    FsAccess::listDirectory(devicesDir, deviceNames);
+    for (auto &&next : deviceNames) {
+        if (!next.compare(0, primaryDevName.length(), primaryDevName)) {
+            dirname = drmPath + next + std::string("/");
+            break;
+        }
+    }
+}
+
+SysfsAccess *SysfsAccess::create(const std::string dev) {
+    return new SysfsAccess(dev);
+}
+
+ze_result_t SysfsAccess::canRead(const std::string file) {
+    // Prepend sysfs directory path and call the base canRead
+    return FsAccess::canRead(fullPath(file));
+}
+
+ze_result_t SysfsAccess::canWrite(const std::string file) {
+    // Prepend sysfs directory path and call the base canWrite
+    return FsAccess::canWrite(fullPath(file));
+}
+
+ze_result_t SysfsAccess::read(const std::string file, std::string &val) {
+    // Prepend sysfs directory path and call the base read
+    return FsAccess::read(fullPath(file).c_str(), val);
+}
+
+ze_result_t SysfsAccess::read(const std::string file, int &val) {
+    std::string str;
+    ze_result_t result;
+
+    result = FsAccess::read(fullPath(file), str);
+    if (ZE_RESULT_SUCCESS != result) {
+        return result;
+    }
+
+    std::istringstream stream(str);
+    stream >> val;
+
+    if (stream.fail()) {
+        return ZE_RESULT_ERROR_UNKNOWN;
+    }
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t SysfsAccess::read(const std::string file, double &val) {
+    std::string str;
+    ze_result_t result;
+
+    result = FsAccess::read(fullPath(file), str);
+    if (ZE_RESULT_SUCCESS != result) {
+        return result;
+    }
+
+    std::istringstream stream(str);
+    stream >> val;
+
+    if (stream.fail()) {
+        return ZE_RESULT_ERROR_UNKNOWN;
+    }
+    return ZE_RESULT_SUCCESS;
+}
+
+ze_result_t SysfsAccess::read(const std::string file, std::vector<std::string> &val) {
+    // Prepend sysfs directory path and call the base read
+    return FsAccess::read(fullPath(file), val);
+}
+
+ze_result_t SysfsAccess::write(const std::string file, const std::string val) {
+    // Prepend sysfs directory path and call the base write
+    return FsAccess::write(fullPath(file).c_str(), val);
+}
+
+ze_result_t SysfsAccess::write(const std::string file, const int val) {
+    std::ostringstream stream;
+    stream << val;
+
+    if (stream.fail()) {
+        return ZE_RESULT_ERROR_UNKNOWN;
+    }
+    return FsAccess::write(fullPath(file), stream.str());
+}
+
+ze_result_t SysfsAccess::write(const std::string file, const double val) {
+    std::ostringstream stream;
+    stream << val;
+
+    if (stream.fail()) {
+        return ZE_RESULT_ERROR_UNKNOWN;
+    }
+    return FsAccess::write(fullPath(file), stream.str());
+}
+
+ze_result_t SysfsAccess::readSymLink(const std::string path, std::string &val) {
+    // Prepend sysfs directory path and call the base readSymLink
+    return FsAccess::readSymLink(fullPath(path).c_str(), val);
+}
+
+ze_result_t SysfsAccess::getRealPath(const std::string path, std::string &val) {
+    // Prepend sysfs directory path and call the base getRealPath
+    return FsAccess::getRealPath(fullPath(path).c_str(), val);
+}
+
+ze_result_t SysfsAccess::bindDevice(std::string device) {
+    return FsAccess::write(intelGpuBindEntry, device);
+}
+
+ze_result_t SysfsAccess::unbindDevice(std::string device) {
+    return FsAccess::write(intelGpuUnbindEntry, device);
+}
+
+ze_bool_t SysfsAccess::fileExists(const std::string file) {
+    // Prepend sysfs directory path and call the base fileExists
+    return FsAccess::fileExists(fullPath(file).c_str());
+}
+
+ze_bool_t SysfsAccess::isMyDeviceFile(const std::string dev) {
+    // dev is a full pathname.
+    if (getDirName(dev).compare(drmDriverDevNodeDir)) {
+        for (auto &&next : deviceNames) {
+            if (!getBaseName(dev).compare(next)) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+} // namespace L0

level_zero/tools/source/sysman/linux/fs_access.h

@@ -0,0 +1,110 @@
+/*
+ * Copyright (C) 2020 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "level_zero/ze_api.h"
+#include "level_zero/zet_api.h"
+
+#include <fstream>
+#include <iostream>
+#include <list>
+#include <sstream>
+#include <string>
+#include <sys/stat.h>
+#include <sys/sysmacros.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <vector>
+
+namespace L0 {
+
+class FsAccess {
+  public:
+    static FsAccess *create();
+    ~FsAccess() = default;
+
+    ze_result_t canRead(const std::string file);
+    ze_result_t canWrite(const std::string file);
+
+    ze_result_t read(const std::string file, std::string &val);
+    ze_result_t read(const std::string file, std::vector<std::string> &val);
+
+    ze_result_t write(const std::string file, const std::string val);
+
+    ze_result_t readSymLink(const std::string path, std::string &buf);
+    ze_result_t getRealPath(const std::string path, std::string &buf);
+    ze_result_t listDirectory(const std::string path, std::vector<std::string> &list);
+    std::string getBaseName(const std::string path);
+    std::string getDirName(const std::string path);
+    ze_bool_t fileExists(const std::string file);
+
+  protected:
+    FsAccess();
+};
+
+class ProcfsAccess : private FsAccess {
+  public:
+    static ProcfsAccess *create();
+    ~ProcfsAccess() = default;
+
+    ze_result_t listProcesses(std::vector<::pid_t> &list);
+    ::pid_t myProcessId();
+    ze_result_t getFileDescriptors(const ::pid_t pid, std::vector<int> &list);
+    ze_result_t getFileName(const ::pid_t pid, const int fd, std::string &val);
+
+  private:
+    ProcfsAccess() = default;
+
+    std::string fullPath(const ::pid_t pid);
+    std::string fdDirPath(const ::pid_t pid);
+    std::string fullFdPath(const ::pid_t pid, const int fd);
+    static const std::string procDir;
+    static const std::string fdDir;
+};
+
+class SysfsAccess : private FsAccess {
+  public:
+    static SysfsAccess *create(const std::string file);
+    ~SysfsAccess() = default;
+
+    ze_result_t canRead(const std::string file);
+    ze_result_t canWrite(const std::string file);
+
+    ze_result_t read(const std::string file, std::string &val);
+    ze_result_t read(const std::string file, int &val);
+    ze_result_t read(const std::string file, double &val);
+    ze_result_t read(const std::string file, std::vector<std::string> &val);
+
+    ze_result_t write(const std::string file, const std::string val);
+    ze_result_t write(const std::string file, const int val);
+    ze_result_t write(const std::string file, const double val);
+
+    ze_result_t readSymLink(const std::string path, std::string &buf);
+    ze_result_t getRealPath(const std::string path, std::string &buf);
+    ze_result_t bindDevice(const std::string device);
+    ze_result_t unbindDevice(const std::string device);
+    ze_bool_t fileExists(const std::string file);
+    ze_bool_t isMyDeviceFile(const std::string dev);
+
+  private:
+    SysfsAccess() = delete;
+    SysfsAccess(const std::string file);
+
+    std::string fullPath(const std::string file);
+
+    std::vector<std::string> deviceNames;
+    std::string dirname;
+    static const std::string drmPath;
+    static const std::string devicesPath;
+    static const std::string primaryDevName;
+    static const std::string drmDriverDevNodeDir;
+    static const std::string intelGpuBindEntry;
+    static const std::string intelGpuUnbindEntry;
+};
+
+} // namespace L0

level_zero/tools/source/sysman/linux/os_sysman_imp.cpp

@@ -7,30 +7,62 @@
 
 #include "level_zero/tools/source/sysman/linux/os_sysman_imp.h"
 
-#include "level_zero/tools/source/sysman/linux/sysfs_access.h"
+#include "level_zero/tools/source/sysman/linux/fs_access.h"
 
 namespace L0 {
 
 ze_result_t LinuxSysmanImp::init() {
+    pFsAccess = FsAccess::create();
+    UNRECOVERABLE_IF(nullptr == pFsAccess);
+
+    pProcfsAccess = ProcfsAccess::create();
+    UNRECOVERABLE_IF(nullptr == pProcfsAccess);
+
     Device *pDevice = Device::fromHandle(pParentSysmanImp->hCoreDevice);
     NEO::OSInterface &OsInterface = pDevice->getOsInterface();
     NEO::Drm *pDrm = OsInterface.get()->getDrm();
-    int fd = pDrm->getFileDescriptor();
+    int myDeviceFd = pDrm->getFileDescriptor();
+    std::string myDeviceName;
+    ze_result_t result = pProcfsAccess->getFileName(pProcfsAccess->myProcessId(), myDeviceFd, myDeviceName);
+    if (ZE_RESULT_SUCCESS != result) {
+        return result;
+    }
 
-    pSysfsAccess = SysfsAccess::create(fd);
+    pSysfsAccess = SysfsAccess::create(myDeviceName);
     UNRECOVERABLE_IF(nullptr == pSysfsAccess);
+
     return ZE_RESULT_SUCCESS;
 }
 
+FsAccess &LinuxSysmanImp::getFsAccess() {
+    UNRECOVERABLE_IF(nullptr == pFsAccess);
+    return *pFsAccess;
+}
+
+ProcfsAccess &LinuxSysmanImp::getProcfsAccess() {
+    UNRECOVERABLE_IF(nullptr == pProcfsAccess);
+    return *pProcfsAccess;
+}
+
 SysfsAccess &LinuxSysmanImp::getSysfsAccess() {
     UNRECOVERABLE_IF(nullptr == pSysfsAccess);
     return *pSysfsAccess;
 }
 
+LinuxSysmanImp::LinuxSysmanImp(SysmanImp *pParentSysmanImp) {
+    this->pParentSysmanImp = pParentSysmanImp;
+}
+
 LinuxSysmanImp::~LinuxSysmanImp() {
     if (nullptr != pSysfsAccess) {
         delete pSysfsAccess;
     }
+    if (nullptr != pProcfsAccess) {
+        delete pProcfsAccess;
+    }
+    if (nullptr != pFsAccess) {
+        delete pFsAccess;
+    }
 }
 
 OsSysman *OsSysman::create(SysmanImp *pParentSysmanImp) {

level_zero/tools/source/sysman/linux/os_sysman_imp.h

@@ -9,14 +9,14 @@
 #include "shared/source/os_interface/linux/os_interface.h"
 
 #include "level_zero/core/source/device.h"
-#include "level_zero/tools/source/sysman/linux/sysfs_access.h"
+#include "level_zero/tools/source/sysman/linux/fs_access.h"
 #include "level_zero/tools/source/sysman/sysman_imp.h"
 
 namespace L0 {
 
 class LinuxSysmanImp : public OsSysman {
   public:
-    LinuxSysmanImp(SysmanImp *pParentSysmanImp) : pParentSysmanImp(pParentSysmanImp), pSysfsAccess(nullptr){};
+    LinuxSysmanImp(SysmanImp *pParentSysmanImp);
     ~LinuxSysmanImp() override;
 
     // Don't allow copies of the LinuxSysmanImp object
@@ -25,11 +25,16 @@ class LinuxSysmanImp : public OsSysman {
 
     ze_result_t init() override;
 
+    FsAccess &getFsAccess();
+    ProcfsAccess &getProcfsAccess();
     SysfsAccess &getSysfsAccess();
 
   private:
-    SysmanImp *pParentSysmanImp;
-    SysfsAccess *pSysfsAccess;
+    LinuxSysmanImp() = delete;
+    SysmanImp *pParentSysmanImp = nullptr;
+    FsAccess *pFsAccess = nullptr;
+    ProcfsAccess *pProcfsAccess = nullptr;
+    SysfsAccess *pSysfsAccess = nullptr;
 };
 
 } // namespace L0

level_zero/tools/source/sysman/linux/sysfs_access.cpp

@@ -1,212 +0,0 @@
-/*
- * Copyright (C) 2020 Intel Corporation
- *
- * SPDX-License-Identifier: MIT
- *
- */
-
-#include "sysfs_access.h"
-
-#include <climits>
-
-#include <cerrno>
-#include <cstdio>
-
-namespace L0 {
-
-static ze_result_t getResult(int err) {
-    if ((EPERM == err) || (EACCES == err)) {
-        return ZE_RESULT_ERROR_INSUFFICIENT_PERMISSIONS;
-    } else if (ENOENT == err) {
-        return ZE_RESULT_ERROR_UNKNOWN;
-    } else {
-        return ZE_RESULT_ERROR_UNKNOWN;
-    }
-}
-
-const std::string SysfsAccess::charDevPath = "/sys/dev/char/";
-
-std::string SysfsAccess::fullPath(const std::string file) {
-    return std::string(dirname + file);
-}
-
-SysfsAccess::SysfsAccess(const int maj, const int min) {
-    dirname = charDevPath + std::to_string(maj) + ":" + std::to_string(min) + "/";
-}
-
-SysfsAccess::SysfsAccess(const std::string path) {
-    dirname = path;
-}
-
-SysfsAccess *SysfsAccess::create(const int fd) {
-    struct stat st;
-
-    if (fd < 0 || fstat(fd, &st) || !S_ISCHR(st.st_mode)) {
-        return NULL;
-    }
-
-    return new SysfsAccess(std::string("/sys/class/drm/card0/"));
-}
-
-SysfsAccess *SysfsAccess::create(const std::string path) {
-    return new SysfsAccess(path);
-}
-
-ze_result_t SysfsAccess::readAsString(const std::string file, std::string &val) {
-    std::ifstream sysfs;
-    val.clear();
-
-    sysfs.open(fullPath(file).c_str());
-    if (sysfs.fail()) {
-        return getResult(errno);
-    }
-    sysfs >> val;
-    if (sysfs.fail()) {
-        sysfs.close();
-        return getResult(errno);
-    }
-    sysfs.close();
-
-    if (val.back() == '\n') {
-        val.pop_back();
-    }
-    return ZE_RESULT_SUCCESS;
-}
-
-ze_result_t SysfsAccess::readAsLines(const std::string file, std::vector<std::string> &val) {
-    std::string line;
-    std::ifstream sysfs;
-    val.clear();
-
-    sysfs.open(fullPath(file).c_str());
-    if (sysfs.fail()) {
-        return getResult(errno);
-    }
-    while (std::getline(sysfs, line)) {
-        if (sysfs.fail()) {
-            sysfs.close();
-            return getResult(errno);
-        }
-        if (line.back() == '\n') {
-            line.pop_back();
-        }
-        val.push_back(line);
-    }
-    sysfs.close();
-
-    return ZE_RESULT_SUCCESS;
-}
-
-ze_result_t SysfsAccess::writeAsString(const std::string file, const std::string val) {
-    std::ofstream sysfs;
-
-    sysfs.open(fullPath(file).c_str());
-    if (sysfs.fail()) {
-        return getResult(errno);
-    }
-    sysfs << val << std::endl;
-    if (sysfs.fail()) {
-        sysfs.close();
-        return getResult(errno);
-    }
-    sysfs.close();
-    return ZE_RESULT_SUCCESS;
-}
-
-ze_result_t SysfsAccess::canRead(const std::string file) {
-    if (access(fullPath(file).c_str(), F_OK)) {
-        return ZE_RESULT_ERROR_UNKNOWN;
-    }
-    if (access(fullPath(file).c_str(), R_OK)) {
-        return ZE_RESULT_ERROR_INSUFFICIENT_PERMISSIONS;
-    }
-    return ZE_RESULT_SUCCESS;
-}
-
-ze_result_t SysfsAccess::canWrite(const std::string file) {
-    if (access(fullPath(file).c_str(), F_OK)) {
-        return ZE_RESULT_ERROR_UNKNOWN;
-    }
-    if (access(fullPath(file).c_str(), W_OK)) {
-        return ZE_RESULT_ERROR_INSUFFICIENT_PERMISSIONS;
-    }
-    return ZE_RESULT_SUCCESS;
-}
-
-ze_result_t SysfsAccess::read(const std::string file, int &val) {
-    std::string str;
-    ze_result_t result;
-
-    result = readAsString(file, str);
-    if (ZE_RESULT_SUCCESS != result) {
-        return result;
-    }
-
-    std::istringstream stream(str);
-    stream >> val;
-
-    if (stream.fail()) {
-        return ZE_RESULT_ERROR_UNKNOWN;
-    }
-    return ZE_RESULT_SUCCESS;
-}
-
-ze_result_t SysfsAccess::read(const std::string file, double &val) {
-    std::string str;
-    ze_result_t result;
-
-    result = readAsString(file, str);
-    if (ZE_RESULT_SUCCESS != result) {
-        return result;
-    }
-
-    std::istringstream stream(str);
-    stream >> val;
-
-    if (stream.fail()) {
-        return ZE_RESULT_ERROR_UNKNOWN;
-    }
-    return ZE_RESULT_SUCCESS;
-}
-
-ze_result_t SysfsAccess::write(const std::string file, const int val) {
-    std::ostringstream stream;
-    stream << val;
-
-    if (stream.fail()) {
-        return ZE_RESULT_ERROR_UNKNOWN;
-    }
-    return writeAsString(file, stream.str());
-}
-
-ze_result_t SysfsAccess::write(const std::string file, const double val) {
-    std::ostringstream stream;
-    stream << val;
-
-    if (stream.fail()) {
-        return ZE_RESULT_ERROR_UNKNOWN;
-    }
-    return writeAsString(file, stream.str());
-}
-
-ze_result_t SysfsAccess::readSymLink(const std::string path, std::string &val) {
-    char buf[PATH_MAX];
-    ssize_t len = ::readlink(fullPath(path).c_str(), buf, PATH_MAX - 1);
-    if (len < 0) {
-        return getResult(errno);
-    }
-    buf[len] = '\0';
-    val = std::string(buf);
-    return ZE_RESULT_SUCCESS;
-}
-
-ze_bool_t SysfsAccess::fileExists(const std::string file) {
-    struct stat sb;
-    if (stat(fullPath(file).c_str(), &sb) == 0) {
-        return true;
-    }
-
-    return false;
-}
-
-} // namespace L0

level_zero/tools/source/sysman/linux/sysfs_access.h

@@ -1,57 +0,0 @@
-/*
- * Copyright (C) 2020 Intel Corporation
- *
- * SPDX-License-Identifier: MIT
- *
- */
-
-#pragma once
-
-#include <level_zero/ze_api.h>
-
-#include <fstream>
-#include <iostream>
-#include <list>
-#include <sstream>
-#include <string>
-#include <sys/stat.h>
-#include <sys/sysmacros.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <vector>
-
-namespace L0 {
-
-class SysfsAccess {
-  public:
-    static SysfsAccess *create(const int fd);
-    static SysfsAccess *create(const std::string path);
-
-    ze_result_t canRead(const std::string file);
-    ze_result_t canWrite(const std::string file);
-
-    ze_result_t read(const std::string file, std::string &val) { return this->readAsString(file, val); }
-    ze_result_t read(const std::string file, std::vector<std::string> &val) { return this->readAsLines(file, val); }
-    ze_result_t read(const std::string file, int &val);
-    ze_result_t read(const std::string file, double &val);
-
-    ze_result_t write(const std::string file, const std::string val) { return this->writeAsString(file, val); }
-    ze_result_t write(const std::string file, const int val);
-    ze_result_t write(const std::string file, const double val);
-    ze_result_t readSymLink(const std::string path, std::string &buf);
-    ze_bool_t fileExists(const std::string file);
-
-  private:
-    ze_result_t readAsString(const std::string file, std::string &val);
-    ze_result_t readAsLines(const std::string file, std::vector<std::string> &val);
-    ze_result_t writeAsString(const std::string file, const std::string val);
-    std::string fullPath(const std::string file);
-
-    SysfsAccess(const int maj, const int min);
-    SysfsAccess(const std::string path);
-
-    std::string dirname;
-    static const std::string charDevPath;
-};
-
-} // namespace L0

level_zero/tools/source/sysman/pci/linux/os_pci_imp.cpp

@@ -11,7 +11,6 @@
 #include "level_zero/core/source/device.h"
 
 #include "sysman/linux/os_sysman_imp.h"
-#include "sysman/linux/sysfs_access.h"
 #include "sysman/pci/os_pci.h"
 #include "sysman/pci/pci_imp.h"
 

level_zero/tools/source/sysman/standby/linux/os_standby_imp.cpp

@@ -11,7 +11,6 @@
 #include "level_zero/core/source/device.h"
 
 #include "sysman/linux/os_sysman_imp.h"
-#include "sysman/linux/sysfs_access.h"
 #include "sysman/standby/os_standby.h"
 #include "sysman/standby/standby_imp.h"
 

level_zero/tools/source/sysman/sysman_device/linux/os_sysman_device_imp.cpp

@@ -9,12 +9,12 @@
 #include "shared/source/os_interface/linux/os_interface.h"
 
 #include "level_zero/core/source/device.h"
+#include "level_zero/tools/source/sysman/linux/fs_access.h"
 #include "level_zero/tools/source/sysman/linux/os_sysman_imp.h"
-#include "level_zero/tools/source/sysman/linux/sysfs_access.h"
 #include "level_zero/tools/source/sysman/sysman_device/os_sysman_device.h"
 #include "level_zero/tools/source/sysman/sysman_device/sysman_device_imp.h"
 
-#include <unistd.h>
+#include <csignal>
 
 namespace L0 {
 
@@ -26,6 +26,7 @@ class LinuxSysmanDeviceImp : public OsSysmanDevice {
     void getModelName(int8_t (&modelName)[ZET_STRING_PROPERTY_SIZE]) override;
     void getVendorName(int8_t (&vendorName)[ZET_STRING_PROPERTY_SIZE]) override;
     void getDriverVersion(int8_t (&driverVersion)[ZET_STRING_PROPERTY_SIZE]) override;
+    ze_result_t reset() override;
     LinuxSysmanDeviceImp(OsSysman *pOsSysman);
     ~LinuxSysmanDeviceImp() override = default;
 
@@ -41,15 +42,18 @@ class LinuxSysmanDeviceImp : public OsSysmanDevice {
     static const std::string deviceFile;
     static const std::string subsystemVendorFile;
     static const std::string driverFile;
+    static const std::string functionLevelReset;
 };
 
 const std::string vendorIntel("Intel(R) Corporation");
 const std::string unknown("Unknown");
 const std::string intelPciId("0x8086");
+const std::string LinuxSysmanDeviceImp::deviceDir("device");
 const std::string LinuxSysmanDeviceImp::vendorFile("device/vendor");
 const std::string LinuxSysmanDeviceImp::deviceFile("device/device");
 const std::string LinuxSysmanDeviceImp::subsystemVendorFile("device/subsystem_vendor");
 const std::string LinuxSysmanDeviceImp::driverFile("device/driver");
+const std::string LinuxSysmanDeviceImp::functionLevelReset("device/reset");
 
 void LinuxSysmanDeviceImp::getSerialNumber(int8_t (&serialNumber)[ZET_STRING_PROPERTY_SIZE]) {
     std::copy(unknown.begin(), unknown.end(), serialNumber);
@@ -113,6 +117,117 @@ void LinuxSysmanDeviceImp::getDriverVersion(int8_t (&driverVersion)[ZET_STRING_P
     driverVersion[unknown.size()] = '\0';
 }
 
+static void getPidFdsForOpenDevice(ProcfsAccess *pProcfsAccess, SysfsAccess *pSysfsAccess, const ::pid_t pid, std::vector<int> &deviceFds) {
+    // Return a list of all the file descriptors of this process that point to this device
+    std::vector<int> fds;
+    deviceFds.clear();
+    if (ZE_RESULT_SUCCESS != pProcfsAccess->getFileDescriptors(pid, fds)) {
+        // Process exited. Not an error. Just ignore.
+        return;
+    }
+    for (auto &&fd : fds) {
+        std::string file;
+        if (pProcfsAccess->getFileName(pid, fd, file) != ZE_RESULT_SUCCESS) {
+            // Process closed this file. Not an error. Just ignore.
+            continue;
+        }
+        if (pSysfsAccess->isMyDeviceFile(file)) {
+            deviceFds.push_back(fd);
+        }
+    }
+}
+
+ze_result_t LinuxSysmanDeviceImp::reset() {
+    FsAccess *pFsAccess = &pLinuxSysmanImp->getFsAccess();
+    ProcfsAccess *pProcfsAccess = &pLinuxSysmanImp->getProcfsAccess();
+    SysfsAccess *pSysfsAccess = &pLinuxSysmanImp->getSysfsAccess();
+
+    std::string resetPath;
+    std::string resetName;
+    ze_result_t result = ZE_RESULT_SUCCESS;
+
+    pSysfsAccess->getRealPath(functionLevelReset, resetPath);
+    // Must run as root. Verify permission to perform reset.
+    result = pFsAccess->canWrite(resetPath);
+    if (ZE_RESULT_SUCCESS != result) {
+        return result;
+    }
+    pSysfsAccess->getRealPath(deviceDir, resetName);
+    resetName = pFsAccess->getBaseName(resetName);
+
+    ::pid_t myPid = pProcfsAccess->myProcessId();
+    std::vector<int> myPidFds;
+    std::vector<::pid_t> processes;
+
+    // For all processes in the system, see if the process
+    // has this device open.
+    result = pProcfsAccess->listProcesses(processes);
+    if (ZE_RESULT_SUCCESS != result) {
+        return result;
+    }
+    for (auto &&pid : processes) {
+        std::vector<int> fds;
+        getPidFdsForOpenDevice(pProcfsAccess, pSysfsAccess, pid, fds);
+        if (pid == myPid) {
+            // L0 is expected to have this file open.
+            // Keep list of fds. Close before unbind.
+            myPidFds = fds;
+        } else if (!fds.empty()) {
+            // Device is in use by another process.
+            // Don't reset while in use.
+            return ZE_RESULT_ERROR_HANDLE_OBJECT_IN_USE;
+        }
+    }
+
+    for (auto &&fd : myPidFds) {
+        // Close open filedescriptors to the device
+        // before unbinding device.
+        // From this point forward, there is
+        // graceful way to fail the reset call.
+        // All future ze calls by this process for this
+        // device will fail.
+        ::close(fd);
+    }
+
+    // Unbind the device from the kernel driver.
+    result = pSysfsAccess->unbindDevice(resetName);
+    if (ZE_RESULT_SUCCESS != result) {
+        return result;
+    }
+
+    // Verify no other process has the device open.
+    // This addresses the window between checking
+    // file handles above, and unbinding the device.
+    result = pProcfsAccess->listProcesses(processes);
+    if (ZE_RESULT_SUCCESS != result) {
+        return result;
+    }
+    for (auto &&pid : processes) {
+        std::vector<int> fds;
+        getPidFdsForOpenDevice(pProcfsAccess, pSysfsAccess, pid, fds);
+        if (!fds.empty()) {
+            // Process is using this device after we unbound it.
+            // Send a sigkill to the process to force it to close
+            // the device. Otherwise, the device cannot be rebound.
+            ::kill(pid, SIGKILL);
+        }
+    }
+
+    // Reset the device.
+    result = pFsAccess->write(resetPath, "1");
+    if (ZE_RESULT_SUCCESS != result) {
+        return result;
+    }
+
+    // Rebind the device to the kernel driver.
+    result = pSysfsAccess->bindDevice(resetName);
+    if (ZE_RESULT_SUCCESS != result) {
+        return result;
+    }
+
+    return ZE_RESULT_SUCCESS;
+}
+
 LinuxSysmanDeviceImp::LinuxSysmanDeviceImp(OsSysman *pOsSysman) {
     pLinuxSysmanImp = static_cast<LinuxSysmanImp *>(pOsSysman);
 

level_zero/tools/source/sysman/sysman_device/os_sysman_device.h

@@ -22,6 +22,7 @@ class OsSysmanDevice {
     virtual void getModelName(int8_t (&modelName)[ZET_STRING_PROPERTY_SIZE]) = 0;
     virtual void getVendorName(int8_t (&vendorName)[ZET_STRING_PROPERTY_SIZE]) = 0;
     virtual void getDriverVersion(int8_t (&driverVersion)[ZET_STRING_PROPERTY_SIZE]) = 0;
+    virtual ze_result_t reset() = 0;
     static OsSysmanDevice *create(OsSysman *pOsSysman);
     virtual ~OsSysmanDevice() {}
 };

level_zero/tools/source/sysman/sysman_device/sysman_device.h

@@ -10,10 +10,11 @@
 
 namespace L0 {
 
-class sysmanDevice {
+class SysmanDevice {
   public:
-    virtual ~sysmanDevice(){};
+    virtual ~SysmanDevice(){};
     virtual ze_result_t deviceGetProperties(zet_sysman_properties_t *pProperties) = 0;
+    virtual ze_result_t reset() = 0;
 
     virtual void init() = 0;
 };

level_zero/tools/source/sysman/sysman_device/sysman_device_imp.cpp

@@ -27,6 +27,10 @@ ze_result_t SysmanDeviceImp::deviceGetProperties(zet_sysman_properties_t *pPrope
     return ZE_RESULT_SUCCESS;
 }
 
+ze_result_t SysmanDeviceImp::reset() {
+    return pOsSysmanDevice->reset();
+}
+
 void SysmanDeviceImp::init() {
     if (pOsSysmanDevice == nullptr) {
         pOsSysmanDevice = OsSysmanDevice::create(pOsSysman);

level_zero/tools/source/sysman/sysman_device/sysman_device_imp.h

@@ -15,10 +15,11 @@
 
 namespace L0 {
 
-class SysmanDeviceImp : public sysmanDevice {
+class SysmanDeviceImp : public SysmanDevice {
   public:
     void init() override;
     ze_result_t deviceGetProperties(zet_sysman_properties_t *pProperties) override;
+    ze_result_t reset() override;
 
     SysmanDeviceImp(OsSysman *pOsSysman, ze_device_handle_t hCoreDevice) : pOsSysman(pOsSysman),
                                                                            hCoreDevice(hCoreDevice) { pOsSysmanDevice = nullptr; };
@@ -27,7 +28,7 @@ class SysmanDeviceImp : public sysmanDevice {
     SysmanDeviceImp(OsSysmanDevice *pOsSysmanDevice, ze_device_handle_t hCoreDevice) : pOsSysmanDevice(pOsSysmanDevice),
                                                                                        hCoreDevice(hCoreDevice) { init(); };
 
-    // Don't allow copies of the sysmanDeviceImp object
+    // Don't allow copies of the SysmanDeviceImp object
     SysmanDeviceImp(const SysmanDeviceImp &obj) = delete;
     SysmanDeviceImp &operator=(const SysmanDeviceImp &obj) = delete;
 

level_zero/tools/source/sysman/sysman_device/windows/CMakeLists.txt

@@ -17,4 +17,5 @@ target_sources(${L0_STATIC_LIB_NAME}
 endif()
 
 # Make our source files visible to parent
-set_property(GLOBAL PROPERTY L0_SRCS_TOOLS_SYSMAN_DEVICE_WINDOWS ${L0_SRCS_TOOLS_SYSMAN_DEVICE_WINDOWS})
+set_property(GLOBAL PROPERTY L0_SRCS_TOOLS_SYSMAN_DEVICE_WINDOWS ${L0_SRCS_TOOLS_SYSMAN_DEVICE_WINDOWS})
+

level_zero/tools/source/sysman/sysman_device/windows/os_sysman_device_imp.cpp

@@ -18,6 +18,14 @@ class WddmSysmanDeviceImp : public OsSysmanDevice {
     void getModelName(int8_t (&modelName)[ZET_STRING_PROPERTY_SIZE]) override;
     void getVendorName(int8_t (&vendorName)[ZET_STRING_PROPERTY_SIZE]) override;
     void getDriverVersion(int8_t (&driverVersion)[ZET_STRING_PROPERTY_SIZE]) override;
+    ze_result_t reset() override;
+
+    WddmSysmanDeviceImp(OsSysman *pOsSysman);
+    ~WddmSysmanDeviceImp() = default;
+
+    // Don't allow copies of the WddmSysmanDeviceImp object
+    WddmSysmanDeviceImp(const WddmSysmanDeviceImp &obj) = delete;
+    WddmSysmanDeviceImp &operator=(const WddmSysmanDeviceImp &obj) = delete;
 };
 
 void WddmSysmanDeviceImp::getSerialNumber(int8_t (&serialNumber)[ZET_STRING_PROPERTY_SIZE]) {
@@ -38,8 +46,15 @@ void WddmSysmanDeviceImp::getVendorName(int8_t (&vendorName)[ZET_STRING_PROPERTY
 void WddmSysmanDeviceImp::getDriverVersion(int8_t (&driverVersion)[ZET_STRING_PROPERTY_SIZE]) {
 }
 
+ze_result_t WddmSysmanDeviceImp::reset() {
+    return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
+}
+
+WddmSysmanDeviceImp::WddmSysmanDeviceImp(OsSysman *pOsSysman) {
+}
+
 OsSysmanDevice *OsSysmanDevice::create(OsSysman *pOsSysman) {
-    WddmSysmanDeviceImp *pWddmSysmanDeviceImp = new WddmSysmanDeviceImp();
+    WddmSysmanDeviceImp *pWddmSysmanDeviceImp = new WddmSysmanDeviceImp(pOsSysman);
     return static_cast<OsSysmanDevice *>(pWddmSysmanDeviceImp);
 }
 

level_zero/tools/source/sysman/sysman_imp.cpp

@@ -29,8 +29,8 @@ SysmanImp::SysmanImp(ze_device_handle_t hDevice) {
 SysmanImp::~SysmanImp() {
     delete pStandbyHandleContext;
     delete pFrequencyHandleContext;
-    delete pPci;
     delete pSysmanDevice;
+    delete pPci;
     delete pOsSysman;
 }
 
@@ -79,7 +79,7 @@ ze_result_t SysmanImp::processesGetState(uint32_t *pCount, zet_process_state_t *
 }
 
 ze_result_t SysmanImp::deviceReset() {
-    return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
+    return pSysmanDevice->reset();
 }
 
 ze_result_t SysmanImp::deviceGetRepairStatus(zet_repair_status_t *pRepairStatus) {

level_zero/tools/source/sysman/sysman_imp.h

@@ -30,7 +30,7 @@ struct SysmanImp : Sysman {
 
     OsSysman *pOsSysman;
     Pci *pPci;
-    sysmanDevice *pSysmanDevice;
+    SysmanDevice *pSysmanDevice;
     FrequencyHandleContext *pFrequencyHandleContext;
     StandbyHandleContext *pStandbyHandleContext;
 

level_zero/tools/source/sysman/windows/os_sysman_imp.h

@@ -21,6 +21,7 @@ class WddmSysmanImp : public OsSysman {
     ze_result_t init() override;
 
   private:
+    WddmSysmanImp() = delete;
     SysmanImp *pParentSysmanImp;
 };