Implement zetSysmanProcessesGetState API

This change does the following:
 - Implement API zetSysmanProcessesGetState.
 - Implement ULT to validate zetSysmanProcessesGetState's implementation
 - Redesign ULTs for zetSysmanDeviceGetProperties to mock only OS specific
   classes, and validate zetSysmanDeviceGetProperties's implementation end
   to end.
 - Some bug fixes in code that are caught by these new ULTs

Change-Id: I4a83789771d32978576ff62859628df2c0c795ad
Signed-off-by: Jitendra Sharma <jitendra.sharma@intel.com>

level_zero/tools/source/sysman/engine/linux/os_engine_imp.h

@@ -23,6 +23,8 @@ class LinuxEngineImp : public OsEngine, public NEO::NonCopyableClass {
 
   protected:
     SysfsAccess *pSysfsAccess = nullptr;
+
+  private:
     static const std::string computeEngineGroupFile;
     static const std::string computeEngineGroupName;
 };

level_zero/tools/source/sysman/linux/fs_access.cpp

@@ -435,6 +435,11 @@ ze_result_t SysfsAccess::write(const std::string file, const uint64_t val) {
     return FsAccess::write(fullPath(file), stream.str());
 }
 
+ze_result_t SysfsAccess::scanDirEntries(const std::string path, std::vector<std::string> &list) {
+    list.clear();
+    return FsAccess::listDirectory(fullPath(path).c_str(), list);
+}
+
 ze_result_t SysfsAccess::readSymLink(const std::string path, std::string &val) {
     // Prepend sysfs directory path and call the base readSymLink
     return FsAccess::readSymLink(fullPath(path).c_str(), val);

level_zero/tools/source/sysman/linux/fs_access.h

@@ -90,6 +90,7 @@ class SysfsAccess : private FsAccess {
     MOCKABLE_VIRTUAL ze_result_t write(const std::string file, const uint64_t val);
     ze_result_t write(const std::string file, const double val);
 
+    MOCKABLE_VIRTUAL ze_result_t scanDirEntries(const std::string path, std::vector<std::string> &list);
     ze_result_t readSymLink(const std::string path, std::string &buf);
     ze_result_t getRealPath(const std::string path, std::string &buf);
     ze_result_t bindDevice(const std::string device);

level_zero/tools/source/sysman/scheduler/linux/os_scheduler_imp.h

@@ -26,6 +26,8 @@ class LinuxSchedulerImp : public NEO::NonCopyableClass, public OsScheduler {
     LinuxSchedulerImp() = default;
     LinuxSchedulerImp(OsSysman *pOsSysman);
     ~LinuxSchedulerImp() override = default;
+
+  protected:
     SysfsAccess *pSysfsAccess = nullptr;
 
   private:

level_zero/tools/source/sysman/sysman_device/linux/CMakeLists.txt

@@ -7,6 +7,7 @@
 set(L0_SRCS_TOOLS_SYSMAN_DEVICE_LINUX
     ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
     ${CMAKE_CURRENT_SOURCE_DIR}/os_sysman_device_imp.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/os_sysman_device_imp.h
 )
 
 if(UNIX)

level_zero/tools/source/sysman/sysman_device/linux/os_sysman_device_imp.cpp

@@ -5,46 +5,18 @@
  *
  */
 
+#include "level_zero/tools/source/sysman/sysman_device/linux/os_sysman_device_imp.h"
+
 #include "shared/source/os_interface/linux/drm_neo.h"
 #include "shared/source/os_interface/linux/os_interface.h"
 
 #include "level_zero/core/source/device/device.h"
-#include "level_zero/tools/source/sysman/linux/fs_access.h"
-#include "level_zero/tools/source/sysman/linux/os_sysman_imp.h"
-#include "level_zero/tools/source/sysman/sysman_device/os_sysman_device.h"
-#include "level_zero/tools/source/sysman/sysman_device/sysman_device_imp.h"
+#include <level_zero/zet_api.h>
 
 #include <csignal>
 
 namespace L0 {
 
-class LinuxSysmanDeviceImp : public OsSysmanDevice {
-  public:
-    void getSerialNumber(int8_t (&serialNumber)[ZET_STRING_PROPERTY_SIZE]) override;
-    void getBoardNumber(int8_t (&boardNumber)[ZET_STRING_PROPERTY_SIZE]) override;
-    void getBrandName(int8_t (&brandName)[ZET_STRING_PROPERTY_SIZE]) override;
-    void getModelName(int8_t (&modelName)[ZET_STRING_PROPERTY_SIZE]) override;
-    void getVendorName(int8_t (&vendorName)[ZET_STRING_PROPERTY_SIZE]) override;
-    void getDriverVersion(int8_t (&driverVersion)[ZET_STRING_PROPERTY_SIZE]) override;
-    ze_result_t reset() override;
-    LinuxSysmanDeviceImp(OsSysman *pOsSysman);
-    ~LinuxSysmanDeviceImp() override = default;
-
-    // Don't allow copies of the LinuxSysmanDeviceImp object
-    LinuxSysmanDeviceImp(const LinuxSysmanDeviceImp &obj) = delete;
-    LinuxSysmanDeviceImp &operator=(const LinuxSysmanDeviceImp &obj) = delete;
-
-  private:
-    SysfsAccess *pSysfsAccess;
-    LinuxSysmanImp *pLinuxSysmanImp;
-    static const std::string deviceDir;
-    static const std::string vendorFile;
-    static const std::string deviceFile;
-    static const std::string subsystemVendorFile;
-    static const std::string driverFile;
-    static const std::string functionLevelReset;
-};
-
 const std::string vendorIntel("Intel(R) Corporation");
 const std::string unknown("Unknown");
 const std::string intelPciId("0x8086");
@@ -54,15 +26,27 @@ const std::string LinuxSysmanDeviceImp::deviceFile("device/device");
 const std::string LinuxSysmanDeviceImp::subsystemVendorFile("device/subsystem_vendor");
 const std::string LinuxSysmanDeviceImp::driverFile("device/driver");
 const std::string LinuxSysmanDeviceImp::functionLevelReset("device/reset");
+const std::string LinuxSysmanDeviceImp::clientsDir("clients");
+
+// Map engine entries(numeric values) present in /sys/class/drm/card<n>/clients/<client_n>/busy,
+// with engine enum defined in leve-zero spec
+// Note that entries with int 2 and 3(represented by i915 as CLASS_VIDEO and CLASS_VIDEO_ENHANCE)
+// are both mapped to MEDIA, as CLASS_VIDEO represents any media fixed-function hardware.
+const std::map<int, zet_engine_type_t> engineMap = {
+    {0, ZET_ENGINE_TYPE_3D},
+    {1, ZET_ENGINE_TYPE_DMA},
+    {2, ZET_ENGINE_TYPE_MEDIA},
+    {3, ZET_ENGINE_TYPE_MEDIA},
+    {4, ZET_ENGINE_TYPE_COMPUTE}};
 
 void LinuxSysmanDeviceImp::getSerialNumber(int8_t (&serialNumber)[ZET_STRING_PROPERTY_SIZE]) {
     std::copy(unknown.begin(), unknown.end(), serialNumber);
-    serialNumber[unknown.size()] = '\0';
+    serialNumber[unknown.size() - 1] = '\0';
 }
 
 void LinuxSysmanDeviceImp::getBoardNumber(int8_t (&boardNumber)[ZET_STRING_PROPERTY_SIZE]) {
     std::copy(unknown.begin(), unknown.end(), boardNumber);
-    boardNumber[unknown.size()] = '\0';
+    boardNumber[unknown.size() - 1] = '\0';
 }
 
 void LinuxSysmanDeviceImp::getBrandName(int8_t (&brandName)[ZET_STRING_PROPERTY_SIZE]) {
@@ -70,16 +54,16 @@ void LinuxSysmanDeviceImp::getBrandName(int8_t (&brandName)[ZET_STRING_PROPERTY_
     ze_result_t result = pSysfsAccess->read(subsystemVendorFile, strVal);
     if (ZE_RESULT_SUCCESS != result) {
         std::copy(unknown.begin(), unknown.end(), brandName);
-        brandName[unknown.size()] = '\0';
+        brandName[unknown.size() - 1] = '\0';
         return;
     }
     if (strVal.compare(intelPciId) == 0) {
         std::copy(vendorIntel.begin(), vendorIntel.end(), brandName);
-        brandName[vendorIntel.size()] = '\0';
+        brandName[vendorIntel.size() - 1] = '\0';
         return;
     }
     std::copy(unknown.begin(), unknown.end(), brandName);
-    brandName[unknown.size()] = '\0';
+    brandName[unknown.size() - 1] = '\0';
 }
 
 void LinuxSysmanDeviceImp::getModelName(int8_t (&modelName)[ZET_STRING_PROPERTY_SIZE]) {
@@ -87,12 +71,12 @@ void LinuxSysmanDeviceImp::getModelName(int8_t (&modelName)[ZET_STRING_PROPERTY_
     ze_result_t result = pSysfsAccess->read(deviceFile, strVal);
     if (ZE_RESULT_SUCCESS != result) {
         std::copy(unknown.begin(), unknown.end(), modelName);
-        modelName[unknown.size()] = '\0';
+        modelName[unknown.size() - 1] = '\0';
         return;
     }
 
     std::copy(strVal.begin(), strVal.end(), modelName);
-    modelName[strVal.size()] = '\0';
+    modelName[strVal.size() - 1] = '\0';
 }
 
 void LinuxSysmanDeviceImp::getVendorName(int8_t (&vendorName)[ZET_STRING_PROPERTY_SIZE]) {
@@ -100,21 +84,21 @@ void LinuxSysmanDeviceImp::getVendorName(int8_t (&vendorName)[ZET_STRING_PROPERT
     ze_result_t result = pSysfsAccess->read(vendorFile, strVal);
     if (ZE_RESULT_SUCCESS != result) {
         std::copy(unknown.begin(), unknown.end(), vendorName);
-        vendorName[unknown.size()] = '\0';
+        vendorName[unknown.size() - 1] = '\0';
         return;
     }
     if (strVal.compare(intelPciId) == 0) {
         std::copy(vendorIntel.begin(), vendorIntel.end(), vendorName);
-        vendorName[vendorIntel.size()] = '\0';
+        vendorName[vendorIntel.size() - 1] = '\0';
         return;
     }
     std::copy(unknown.begin(), unknown.end(), vendorName);
-    vendorName[unknown.size()] = '\0';
+    vendorName[unknown.size() - 1] = '\0';
 }
 
 void LinuxSysmanDeviceImp::getDriverVersion(int8_t (&driverVersion)[ZET_STRING_PROPERTY_SIZE]) {
     std::copy(unknown.begin(), unknown.end(), driverVersion);
-    driverVersion[unknown.size()] = '\0';
+    driverVersion[unknown.size() - 1] = '\0';
 }
 
 static void getPidFdsForOpenDevice(ProcfsAccess *pProcfsAccess, SysfsAccess *pSysfsAccess, const ::pid_t pid, std::vector<int> &deviceFds) {
@@ -228,6 +212,94 @@ ze_result_t LinuxSysmanDeviceImp::reset() {
     return ZE_RESULT_SUCCESS;
 }
 
+// Processes in the form of clients are present in sysfs like this:
+// # /sys/class/drm/card0/clients$ ls
+// 4  5
+// # /sys/class/drm/card0/clients/4$ ls
+// busy  name  pid
+// # /sys/class/drm/card0/clients/4/busy$ ls
+// 0  1  2  3
+//
+// Number of processes(If one process opened drm device multiple times, then multiple entries will be
+// present for same process in clients directory) will be the number of clients
+// (For example from above example, processes dirs are 4,5)
+// Thus total number of times drm connection opened with this device will be 2.
+// process.pid = pid (from above example)
+// process.engines -> For each client's busy dir, numbers 0,1,2,3 represent engines and they contain
+// accumulated nanoseconds each client spent on engines.
+// Thus we traverse each file in busy dir for non-zero time and if we find that file say 0,then we could say that
+// this engine 0 is used by process.
+ze_result_t LinuxSysmanDeviceImp::scanProcessesState(std::vector<zet_process_state_t> &pProcessList) {
+    std::vector<std::string> clientIds;
+    ze_result_t result = pSysfsAccess->scanDirEntries(clientsDir, clientIds);
+    if (ZE_RESULT_SUCCESS != result) {
+        return result;
+    }
+
+    // Create a map with unique pid as key and engineType as value
+    std::map<uint64_t, int64_t> pidClientMap;
+    for (auto clientId : clientIds) {
+        // realClientPidPath will be something like: clients/<clientId>/pid
+        std::string realClientPidPath = clientsDir + "/" + clientId + "/" + "pid";
+        uint64_t pid;
+        result = pSysfsAccess->read(realClientPidPath, pid);
+        if (ZE_RESULT_SUCCESS != result) {
+            return result;
+        }
+
+        // Traverse the clients/<clientId>/busy directory to get accelerator engines used by process
+        std::vector<std::string> engineNums;
+        std::string busyDirForEngines = clientsDir + "/" + clientId + "/" + "busy";
+        result = pSysfsAccess->scanDirEntries(busyDirForEngines, engineNums);
+        if (ZE_RESULT_SUCCESS != result) {
+            return result;
+        }
+        int64_t engineType = 0;
+        // Scan all engine files present in /sys/class/drm/card0/clients/<ClientId>/busy and check
+        // whether that engine is used by process
+        for (auto engineNum : engineNums) {
+            uint64_t timeSpent = 0;
+            std::string engine = busyDirForEngines + "/" + engineNum;
+            result = pSysfsAccess->read(engine, timeSpent);
+            if (ZE_RESULT_SUCCESS != result) {
+                return result;
+            }
+            if (timeSpent > 0) {
+                int i915EnginNumber = stoi(engineNum);
+                auto i915MapToL0EngineType = engineMap.find(i915EnginNumber);
+                zet_engine_type_t val = ZET_ENGINE_TYPE_OTHER;
+                if (i915MapToL0EngineType != engineMap.end()) {
+                    // Found a valid map
+                    val = i915MapToL0EngineType->second;
+                }
+                // In this for loop we want to retrieve the overall engines used by process
+                engineType = engineType | (1 << val);
+            }
+        }
+
+        auto ret = pidClientMap.insert(std::make_pair(pid, engineType));
+        if (ret.second == false) {
+            // insertion failed as entry with same pid already exists in map
+            // Now update the engineType field of the existing pid entry
+            auto pidEntryFromMap = pidClientMap.find(pid);
+            auto existingEngineType = pidEntryFromMap->second;
+            engineType = existingEngineType | engineType;
+            pidClientMap[pid] = engineType;
+        }
+    }
+
+    // iterate through all elements of pidClientMap
+    for (auto itr = pidClientMap.begin(); itr != pidClientMap.end(); ++itr) {
+        zet_process_state_t process;
+        process.processId = static_cast<uint32_t>(itr->first);
+        process.memSize = 0; // Unsupported
+        process.engines = itr->second;
+        pProcessList.push_back(process);
+    }
+
+    return result;
+}
+
 LinuxSysmanDeviceImp::LinuxSysmanDeviceImp(OsSysman *pOsSysman) {
     pLinuxSysmanImp = static_cast<LinuxSysmanImp *>(pOsSysman);
 
@@ -239,4 +311,4 @@ OsSysmanDevice *OsSysmanDevice::create(OsSysman *pOsSysman) {
     return static_cast<OsSysmanDevice *>(pLinuxSysmanDeviceImp);
 }
 
-} // namespace L0
+} // namespace L0

level_zero/tools/source/sysman/sysman_device/linux/os_sysman_device_imp.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2020 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+#include "level_zero/tools/source/sysman/linux/fs_access.h"
+#include "level_zero/tools/source/sysman/linux/os_sysman_imp.h"
+#include "level_zero/tools/source/sysman/sysman_device/os_sysman_device.h"
+#include "level_zero/tools/source/sysman/sysman_device/sysman_device_imp.h"
+
+namespace L0 {
+
+class LinuxSysmanDeviceImp : public OsSysmanDevice, public NEO::NonCopyableClass {
+  public:
+    void getSerialNumber(int8_t (&serialNumber)[ZET_STRING_PROPERTY_SIZE]) override;
+    void getBoardNumber(int8_t (&boardNumber)[ZET_STRING_PROPERTY_SIZE]) override;
+    void getBrandName(int8_t (&brandName)[ZET_STRING_PROPERTY_SIZE]) override;
+    void getModelName(int8_t (&modelName)[ZET_STRING_PROPERTY_SIZE]) override;
+    void getVendorName(int8_t (&vendorName)[ZET_STRING_PROPERTY_SIZE]) override;
+    void getDriverVersion(int8_t (&driverVersion)[ZET_STRING_PROPERTY_SIZE]) override;
+    ze_result_t reset() override;
+    ze_result_t scanProcessesState(std::vector<zet_process_state_t> &pProcessList) override;
+    LinuxSysmanDeviceImp() = default;
+    LinuxSysmanDeviceImp(OsSysman *pOsSysman);
+    ~LinuxSysmanDeviceImp() override = default;
+
+  protected:
+    SysfsAccess *pSysfsAccess = nullptr;
+    LinuxSysmanImp *pLinuxSysmanImp = nullptr;
+
+  private:
+    static const std::string deviceDir;
+    static const std::string vendorFile;
+    static const std::string deviceFile;
+    static const std::string subsystemVendorFile;
+    static const std::string driverFile;
+    static const std::string functionLevelReset;
+    static const std::string clientsDir;
+};
+
+} // namespace L0

level_zero/tools/source/sysman/sysman_device/os_sysman_device.h

@@ -11,6 +11,7 @@
 #include <level_zero/zet_api.h>
 
 #include <string>
+#include <vector>
 
 namespace L0 {
 
@@ -23,6 +24,7 @@ class OsSysmanDevice {
     virtual void getVendorName(int8_t (&vendorName)[ZET_STRING_PROPERTY_SIZE]) = 0;
     virtual void getDriverVersion(int8_t (&driverVersion)[ZET_STRING_PROPERTY_SIZE]) = 0;
     virtual ze_result_t reset() = 0;
+    virtual ze_result_t scanProcessesState(std::vector<zet_process_state_t> &pProcessList) = 0;
     static OsSysmanDevice *create(OsSysman *pOsSysman);
     virtual ~OsSysmanDevice() {}
 };

level_zero/tools/source/sysman/sysman_device/sysman_device.h

@@ -15,6 +15,7 @@ class SysmanDevice {
     virtual ~SysmanDevice(){};
     virtual ze_result_t deviceGetProperties(zet_sysman_properties_t *pProperties) = 0;
     virtual ze_result_t reset() = 0;
+    virtual ze_result_t processesGetState(uint32_t *pCount, zet_process_state_t *pProcesses) = 0;
 
     virtual void init() = 0;
 };

level_zero/tools/source/sysman/sysman_device/sysman_device_imp.cpp

@@ -14,6 +14,29 @@
 
 namespace L0 {
 
+ze_result_t SysmanDeviceImp::processesGetState(uint32_t *pCount, zet_process_state_t *pProcesses) {
+    std::vector<zet_process_state_t> pProcessList;
+    ze_result_t result = pOsSysmanDevice->scanProcessesState(pProcessList);
+    if (result != ZE_RESULT_SUCCESS) {
+        return result;
+    }
+
+    if ((*pCount > 0) && (*pCount < pProcessList.size())) {
+        result = ZE_RESULT_ERROR_INVALID_SIZE;
+    }
+    if (pProcesses != nullptr) {
+        uint32_t limit = std::min(*pCount, static_cast<uint32_t>(pProcessList.size()));
+        for (uint32_t i = 0; i < limit; i++) {
+            pProcesses[i].processId = pProcessList[i].processId;
+            pProcesses[i].engines = pProcessList[i].engines;
+            pProcesses[i].memSize = pProcessList[i].memSize;
+        }
+    }
+    *pCount = static_cast<uint32_t>(pProcessList.size());
+
+    return result;
+}
+
 ze_result_t SysmanDeviceImp::deviceGetProperties(zet_sysman_properties_t *pProperties) {
     Device *device = L0::Device::fromHandle(hCoreDevice);
     ze_device_properties_t deviceProperties;

level_zero/tools/source/sysman/sysman_device/sysman_device_imp.h

@@ -6,6 +6,8 @@
  */
 
 #pragma once
+#include "shared/source/helpers/non_copyable_or_moveable.h"
+
 #include <level_zero/zet_api.h>
 
 #include "os_sysman_device.h"
@@ -15,11 +17,12 @@
 
 namespace L0 {
 
-class SysmanDeviceImp : public SysmanDevice {
+class SysmanDeviceImp : public SysmanDevice, public NEO::NonCopyableClass {
   public:
     void init() override;
     ze_result_t deviceGetProperties(zet_sysman_properties_t *pProperties) override;
     ze_result_t reset() override;
+    ze_result_t processesGetState(uint32_t *pCount, zet_process_state_t *pProcesses) override;
 
     OsSysmanDevice *pOsSysmanDevice = nullptr;
     ze_device_handle_t hCoreDevice = {};
@@ -29,10 +32,6 @@ class SysmanDeviceImp : public SysmanDevice {
                                                                            pOsSysman(pOsSysman){};
     ~SysmanDeviceImp() override;
 
-    // Don't allow copies of the SysmanDeviceImp object
-    SysmanDeviceImp(const SysmanDeviceImp &obj) = delete;
-    SysmanDeviceImp &operator=(const SysmanDeviceImp &obj) = delete;
-
   private:
     OsSysman *pOsSysman = nullptr;
     zet_sysman_properties_t sysmanProperties = {};

level_zero/tools/source/sysman/sysman_device/windows/os_sysman_device_imp.cpp

@@ -19,6 +19,7 @@ class WddmSysmanDeviceImp : public OsSysmanDevice {
     void getVendorName(int8_t (&vendorName)[ZET_STRING_PROPERTY_SIZE]) override;
     void getDriverVersion(int8_t (&driverVersion)[ZET_STRING_PROPERTY_SIZE]) override;
     ze_result_t reset() override;
+    ze_result_t scanProcessesState(std::vector<zet_process_state_t> &pProcessList) override;
 
     WddmSysmanDeviceImp(OsSysman *pOsSysman);
     ~WddmSysmanDeviceImp() = default;
@@ -50,6 +51,10 @@ ze_result_t WddmSysmanDeviceImp::reset() {
     return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
 }
 
+ze_result_t WddmSysmanDeviceImp::scanProcessesState(std::vector<zet_process_state_t> &pProcessList) {
+    return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
+}
+
 WddmSysmanDeviceImp::WddmSysmanDeviceImp(OsSysman *pOsSysman) {
 }
 

level_zero/tools/source/sysman/sysman_imp.cpp

@@ -138,7 +138,7 @@ ze_result_t SysmanImp::schedulerSetComputeUnitDebugMode(ze_bool_t *pNeedReboot)
 }
 
 ze_result_t SysmanImp::processesGetState(uint32_t *pCount, zet_process_state_t *pProcesses) {
-    return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
+    return pSysmanDevice->processesGetState(pCount, pProcesses);
 }
 
 ze_result_t SysmanImp::deviceReset() {