[Sysman] Add support to control lifetime of file descriptor

This patch adds support to open and close the drm fd,
so that more granular control of drm fd could be
achieved.

Related-To: LOCI-3986

Signed-off-by: Joshua Santosh Ranjan <joshua.santosh.ranjan@intel.com>

level_zero/sysman/source/CMakeLists.txt

@@ -16,6 +16,8 @@ set(L0_SRCS_SYSMAN
     ${CMAKE_CURRENT_SOURCE_DIR}/sysman_device_imp.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/sysman_device_imp.h
     ${CMAKE_CURRENT_SOURCE_DIR}/os_sysman.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/sysman_hw_device_id.h
+
 )
 
 target_sources(${L0_STATIC_LIB_NAME}

level_zero/sysman/source/linux/CMakeLists.txt

@@ -10,6 +10,8 @@ set(L0_SRCS_SYSMAN_LINUX
     ${CMAKE_CURRENT_SOURCE_DIR}/os_sysman_imp.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/fs_access.h
     ${CMAKE_CURRENT_SOURCE_DIR}/fs_access.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id_linux.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id_linux.h
 )
 
 if(UNIX)

level_zero/sysman/source/linux/hw_device_id_linux.cpp

@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/sysman/source/linux/hw_device_id_linux.h"
+
+#include "shared/source/os_interface/linux/sys_calls.h"
+
+#include "level_zero/sysman/source/sysman_hw_device_id.h"
+
+#include <fcntl.h>
+
+namespace L0 {
+namespace Sysman {
+
+int SysmanHwDeviceIdDrm::openFileDescriptor() {
+
+    std::unique_lock<std::mutex> lock(fdMutex);
+    if (fileDescriptor == -1) {
+        fileDescriptor = NEO::SysCalls::open(devNodePath.c_str(), O_RDWR);
+    }
+    ++fdRefCounter;
+    return fileDescriptor;
+}
+
+int SysmanHwDeviceIdDrm::closeFileDescriptor() {
+
+    int closeStatus = 0;
+    std::unique_lock<std::mutex> lock(fdMutex);
+
+    DEBUG_BREAK_IF(fdRefCounter == 0u);
+    if (fdRefCounter > 0u) {
+        --fdRefCounter;
+        if (fdRefCounter == 0u && fileDescriptor >= 0) {
+            closeStatus = NEO::SysCalls::close(fileDescriptor);
+            fileDescriptor = -1;
+        }
+    }
+    return closeStatus;
+}
+
+std::unique_ptr<NEO::HwDeviceId> createSysmanHwDeviceId(std::unique_ptr<NEO::HwDeviceId> &hwDeviceId) {
+
+    const auto hwDeviceIdDrm = static_cast<NEO::HwDeviceIdDrm *>(hwDeviceId.get());
+    return std::make_unique<SysmanHwDeviceIdDrm>(hwDeviceIdDrm->getFileDescriptor(), hwDeviceIdDrm->getPciPath(), hwDeviceIdDrm->getDeviceNode());
+}
+
+} // namespace Sysman
+} // namespace L0

level_zero/sysman/source/linux/hw_device_id_linux.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+#include "shared/source/os_interface/linux/hw_device_id.h"
+
+#include <mutex>
+
+namespace L0 {
+namespace Sysman {
+class SysmanHwDeviceIdDrm : public NEO::HwDeviceIdDrm {
+
+  public:
+    using NEO::HwDeviceIdDrm::HwDeviceIdDrm;
+    SysmanHwDeviceIdDrm() = delete;
+    int openFileDescriptor();
+    int closeFileDescriptor();
+
+  private:
+    std::mutex fdMutex{};
+    uint32_t fdRefCounter = 0;
+};
+
+} // namespace Sysman
+} // namespace L0

level_zero/sysman/source/linux/os_sysman_imp.cpp

@@ -33,8 +33,9 @@ ze_result_t LinuxSysmanImp::init() {
     if (osInterface.getDriverModel()->getDriverModelType() != NEO::DriverModelType::DRM) {
         return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
     }
-    auto pDrm = osInterface.getDriverModel()->as<NEO::Drm>();
+    auto sysmanHwDeviceId = getSysmanHwDeviceId();
-    int myDeviceFd = pDrm->getFileDescriptor();
+    sysmanHwDeviceId->openFileDescriptor();
+    int myDeviceFd = sysmanHwDeviceId->getFileDescriptor();
     std::string myDeviceName;
     result = pProcfsAccess->getFileName(pProcfsAccess->myProcessId(), myDeviceFd, myDeviceName);
     if (ZE_RESULT_SUCCESS != result) {
@@ -53,10 +54,19 @@ ze_result_t LinuxSysmanImp::init() {
 
     osInterface.getDriverModel()->as<NEO::Drm>()->cleanup();
     // Close Drm handles
-    pDrm->closeFileDescriptor();
+    sysmanHwDeviceId->closeFileDescriptor();
     return createPmtHandles();
 }
 
+SysmanHwDeviceIdDrm *LinuxSysmanImp::getSysmanHwDeviceId() {
+    return static_cast<SysmanHwDeviceIdDrm *>(getDrm()->getHwDeviceId().get());
+}
+
+NEO::Drm *LinuxSysmanImp::getDrm() {
+    const auto &osInterface = *pParentSysmanDeviceImp->getRootDeviceEnvironment().osInterface;
+    return osInterface.getDriverModel()->as<NEO::Drm>();
+}
+
 ze_result_t LinuxSysmanImp::createPmtHandles() {
     std::string gtDevicePCIPath;
     auto result = pSysfsAccess->getRealPath("device", gtDevicePCIPath);

level_zero/sysman/source/linux/os_sysman_imp.h

@@ -9,11 +9,16 @@
 #include "shared/source/execution_environment/execution_environment.h"
 #include "shared/source/helpers/non_copyable_or_moveable.h"
 
+#include "level_zero/sysman/source/linux/hw_device_id_linux.h"
 #include "level_zero/sysman/source/os_sysman.h"
 #include "level_zero/sysman/source/sysman_device_imp.h"
 
 #include <map>
 
+namespace NEO {
+class Drm;
+} // namespace NEO
+
 namespace L0 {
 namespace Sysman {
 class PlatformMonitoringTech;
@@ -36,6 +41,8 @@ class LinuxSysmanImp : public OsSysman, NEO::NonCopyableOrMovableClass {
     std::string getPciCardBusDirectoryPath(std::string realPciPath);
     PlatformMonitoringTech *getPlatformMonitoringTechAccess(uint32_t subDeviceId);
     PRODUCT_FAMILY getProductFamily() const { return pParentSysmanDeviceImp->getProductFamily(); }
+    SysmanHwDeviceIdDrm *getSysmanHwDeviceId();
+    NEO::Drm *getDrm();
     void releasePmtObject();
     ze_result_t createPmtHandles();
     std::string devicePciBdf = "";

level_zero/sysman/source/sysman_driver.cpp

@@ -14,6 +14,7 @@
 #include "level_zero/core/source/driver/driver.h"
 #include "level_zero/sysman/source/sysman_driver_handle_imp.h"
 #include "level_zero/sysman/source/sysman_driver_imp.h"
+#include "level_zero/sysman/source/sysman_hw_device_id.h"
 
 #include <cstring>
 #include <vector>
@@ -43,7 +44,12 @@ void SysmanDriverImp::initialize(ze_result_t *result) {
         executionEnvironment->prepareRootDeviceEnvironments(static_cast<uint32_t>(hwDeviceIds.size()));
         uint32_t rootDeviceIndex = 0u;
         for (auto &hwDeviceId : hwDeviceIds) {
-            if (!executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->initOsInterface(std::move(hwDeviceId), rootDeviceIndex)) {
+
+            auto sysmanHwDeviceId = createSysmanHwDeviceId(hwDeviceId);
+            auto initStatus = sysmanHwDeviceId != nullptr &&
+                              executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->initOsInterface(std::move(sysmanHwDeviceId), rootDeviceIndex);
+
+            if (!initStatus) {
                 NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr,
                                       "OsInterface initialization failed for device : %d\n", rootDeviceIndex);
                 *result = ZE_RESULT_ERROR_UNINITIALIZED;

level_zero/sysman/source/sysman_hw_device_id.h

@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+#include <memory>
+#include <vector>
+
+namespace NEO {
+class HwDeviceId;
+} // namespace NEO
+
+namespace L0 {
+namespace Sysman {
+std::unique_ptr<NEO::HwDeviceId> createSysmanHwDeviceId(std::unique_ptr<NEO::HwDeviceId> &hwDeviceId);
+} // namespace Sysman
+} // namespace L0

level_zero/sysman/source/windows/CMakeLists.txt

@@ -8,6 +8,7 @@ set(L0_SRCS_SYSMAN_WINDOWS
     ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
     ${CMAKE_CURRENT_SOURCE_DIR}/os_sysman_imp.h
     ${CMAKE_CURRENT_SOURCE_DIR}/os_sysman_imp.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id_windows.cpp
 )
 
 if(WIN32)

level_zero/sysman/source/windows/hw_device_id_windows.cpp

@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/os_interface/os_interface.h"
+
+#include "level_zero/sysman/source/sysman_hw_device_id.h"
+
+namespace L0 {
+namespace Sysman {
+
+std::unique_ptr<NEO::HwDeviceId> createSysmanHwDeviceId(std::unique_ptr<NEO::HwDeviceId> &hwDeviceId) {
+    return std::unique_ptr<NEO::HwDeviceId>{};
+}
+
+} // namespace Sysman
+} // namespace L0

level_zero/sysman/test/unit_tests/sources/linux/CMakeLists.txt

@@ -12,6 +12,7 @@ if(UNIX)
                  ${CMAKE_CURRENT_SOURCE_DIR}/mock_sysman_drm.h
                  ${CMAKE_CURRENT_SOURCE_DIR}/mock_sysman_driver.h
                  ${CMAKE_CURRENT_SOURCE_DIR}/test_sysman_driver.cpp
+                 ${CMAKE_CURRENT_SOURCE_DIR}/test_sysman_hw_device_id.cpp
   )
 endif()
 

level_zero/sysman/test/unit_tests/sources/linux/mock_sysman_drm.h

@@ -12,12 +12,14 @@
 #include "shared/source/os_interface/linux/drm_neo.h"
 #include "shared/source/os_interface/os_interface.h"
 
+#include "level_zero/sysman/source/linux/hw_device_id_linux.h"
+
 namespace L0 {
 namespace ult {
-constexpr int mockFd = 0;
+constexpr int mockFd = -1;
 class SysmanMockDrm : public NEO::Drm {
   public:
-    SysmanMockDrm(NEO::RootDeviceEnvironment &rootDeviceEnvironment) : Drm(std::make_unique<NEO::HwDeviceIdDrm>(mockFd, ""), rootDeviceEnvironment) {
+    SysmanMockDrm(NEO::RootDeviceEnvironment &rootDeviceEnvironment) : Drm(std::make_unique<L0::Sysman::SysmanHwDeviceIdDrm>(mockFd, ""), rootDeviceEnvironment) {
         setupIoctlHelper(rootDeviceEnvironment.getHardwareInfo()->platform.eProductFamily);
     }
 };

level_zero/sysman/test/unit_tests/sources/linux/mock_sysman_fixture.h

@@ -18,6 +18,7 @@
 #include "level_zero/sysman/source/linux/os_sysman_imp.h"
 #include "level_zero/sysman/source/sysman_device.h"
 #include "level_zero/sysman/source/sysman_driver_handle_imp.h"
+#include "level_zero/sysman/test/unit_tests/sources/linux/mock_sysman_drm.h"
 
 #include "gtest/gtest.h"
 
@@ -25,14 +26,6 @@ using namespace NEO;
 
 namespace L0 {
 namespace ult {
-constexpr int mockFd = 0;
-class SysmanMockDrm : public Drm {
-  public:
-    SysmanMockDrm(RootDeviceEnvironment &rootDeviceEnvironment) : Drm(std::make_unique<HwDeviceIdDrm>(mockFd, ""), rootDeviceEnvironment) {
-        setupIoctlHelper(rootDeviceEnvironment.getHardwareInfo()->platform.eProductFamily);
-    }
-};
-
 class PublicLinuxSysmanImp : public L0::Sysman::LinuxSysmanImp {
   public:
     using LinuxSysmanImp::mapOfSubDeviceIdToPmtObject;

level_zero/sysman/test/unit_tests/sources/linux/test_sysman_hw_device_id.cpp

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/test/common/helpers/variable_backup.h"
+#include "shared/test/common/os_interface/linux/sys_calls_linux_ult.h"
+
+#include "level_zero/sysman/source/linux/hw_device_id_linux.h"
+
+#include "gtest/gtest.h"
+
+#include <bitset>
+#include <cstring>
+
+namespace L0 {
+namespace ult {
+
+TEST(sysmanHwDeviceIdTest, whenOpenFileDescriptorIsCalledMultipleTimesThenOpenIsCalledOnlyOnce) {
+
+    L0::Sysman::SysmanHwDeviceIdDrm hwDeviceId(-1, "mockPciPath", "mockDevNodePath");
+    static uint32_t openCallCount = 0;
+
+    VariableBackup<decltype(NEO::SysCalls::sysCallsOpen)> mockOpen(&NEO::SysCalls::sysCallsOpen, [](const char *pathname, int flags) -> int {
+        openCallCount++;
+        return 1;
+    });
+
+    openCallCount = 0;
+    hwDeviceId.openFileDescriptor();
+    EXPECT_EQ(openCallCount, 1u);
+    hwDeviceId.openFileDescriptor();
+    EXPECT_EQ(openCallCount, 1u);
+}
+
+TEST(sysmanHwDeviceIdTest, whenOpenFileDescriptorIsCalledMultipleTimesThenCloseIsCalledOnlyAfterAllReferencesAreClosed) {
+
+    L0::Sysman::SysmanHwDeviceIdDrm hwDeviceId(-1, "mockPciPath", "mockDevNodePath");
+    VariableBackup<decltype(NEO::SysCalls::sysCallsOpen)> mockOpen(&NEO::SysCalls::sysCallsOpen, [](const char *pathname, int flags) -> int {
+        return 1;
+    });
+
+    const auto referenceCloseCount = NEO::SysCalls::closeFuncCalled;
+    hwDeviceId.openFileDescriptor();
+    hwDeviceId.openFileDescriptor();
+    hwDeviceId.openFileDescriptor();
+    hwDeviceId.closeFileDescriptor();
+    EXPECT_EQ(NEO::SysCalls::closeFuncCalled, referenceCloseCount);
+    hwDeviceId.closeFileDescriptor();
+    EXPECT_EQ(NEO::SysCalls::closeFuncCalled, referenceCloseCount);
+    hwDeviceId.closeFileDescriptor();
+    EXPECT_EQ(NEO::SysCalls::closeFuncCalled, referenceCloseCount + 1u);
+}
+
+} // namespace ult
+} // namespace L0

shared/source/os_interface/linux/drm_neo.cpp

@@ -484,10 +484,10 @@ int Drm::setupHardwareInfo(const DeviceDescriptor *device, bool setupFeatureTabl
     return 0;
 }
 
-void appendHwDeviceId(std::vector<std::unique_ptr<HwDeviceId>> &hwDeviceIds, int fileDescriptor, const char *pciPath) {
+void appendHwDeviceId(std::vector<std::unique_ptr<HwDeviceId>> &hwDeviceIds, int fileDescriptor, const char *pciPath, const char *devNodePath) {
     if (fileDescriptor >= 0) {
         if (Drm::isDrmSupported(fileDescriptor)) {
-            hwDeviceIds.push_back(std::make_unique<HwDeviceIdDrm>(fileDescriptor, pciPath));
+            hwDeviceIds.push_back(std::make_unique<HwDeviceIdDrm>(fileDescriptor, pciPath, devNodePath));
         } else {
             SysCalls::close(fileDescriptor);
         }
@@ -524,7 +524,7 @@ std::vector<std::unique_ptr<HwDeviceId>> Drm::discoverDevices(ExecutionEnvironme
 
             auto pciPath = NEO::getPciPath(fileDescriptor);
 
-            appendHwDeviceId(hwDeviceIds, fileDescriptor, pciPath.value_or("0000:00:02.0").c_str());
+            appendHwDeviceId(hwDeviceIds, fileDescriptor, pciPath.value_or("0000:00:02.0").c_str(), path.c_str());
             if (!hwDeviceIds.empty() && hwDeviceIds.size() == numRootDevices) {
                 break;
             }
@@ -564,7 +564,7 @@ std::vector<std::unique_ptr<HwDeviceId>> Drm::discoverDevices(ExecutionEnvironme
                 }
             }
             int fileDescriptor = SysCalls::open(file->c_str(), O_RDWR);
-            appendHwDeviceId(hwDeviceIds, fileDescriptor, pciPath.c_str());
+            appendHwDeviceId(hwDeviceIds, fileDescriptor, pciPath.c_str(), file->c_str());
             if (!hwDeviceIds.empty() && hwDeviceIds.size() == numRootDevices) {
                 break;
             }

shared/source/os_interface/linux/drm_neo.h

@@ -102,7 +102,6 @@ class Drm : public DriverModel {
 
     MOCKABLE_VIRTUAL void checkPreemptionSupport();
     inline int getFileDescriptor() const { return hwDeviceId->getFileDescriptor(); }
-    inline void closeFileDescriptor() const { return hwDeviceId->closeFileDescriptor(); }
     ADAPTER_BDF getAdapterBDF() const {
         return adapterBDF;
     }
@@ -259,6 +258,7 @@ class Drm : public DriverModel {
     void cleanup() override;
     bool readSysFsAsString(const std::string &relativeFilePath, std::string &readString);
     MOCKABLE_VIRTUAL std::string getSysFsPciPath();
+    std::unique_ptr<HwDeviceIdDrm> &getHwDeviceId() { return hwDeviceId; }
 
   protected:
     Drm(std::unique_ptr<HwDeviceIdDrm> &&hwDeviceIdIn, RootDeviceEnvironment &rootDeviceEnvironment);

shared/source/os_interface/linux/hw_device_id.h

@@ -19,13 +19,17 @@ class HwDeviceIdDrm : public HwDeviceId {
     HwDeviceIdDrm(int fileDescriptorIn, const char *pciPathIn)
         : HwDeviceId(DriverModelType::DRM),
           fileDescriptor(fileDescriptorIn), pciPath(pciPathIn) {}
+    HwDeviceIdDrm(int fileDescriptorIn, const char *pciPathIn, const char *devNodePathIn)
+        : HwDeviceId(DriverModelType::DRM),
+          fileDescriptor(fileDescriptorIn), pciPath(pciPathIn), devNodePath(devNodePathIn) {}
     ~HwDeviceIdDrm() override;
     int getFileDescriptor() const { return fileDescriptor; }
-    void closeFileDescriptor();
     const char *getPciPath() const { return pciPath.c_str(); }
+    const char *getDeviceNode() const { return devNodePath.c_str(); }
 
   protected:
     int fileDescriptor;
     const std::string pciPath;
+    const std::string devNodePath;
 };
 } // namespace NEO

shared/source/os_interface/linux/hw_device_id_linux.cpp

@@ -10,15 +10,8 @@
 
 namespace NEO {
 
-void HwDeviceIdDrm::closeFileDescriptor() {
-    if (fileDescriptor > 0) {
-        SysCalls::close(fileDescriptor);
-        fileDescriptor = -1;
-    }
-}
-
 HwDeviceIdDrm::~HwDeviceIdDrm() {
-    closeFileDescriptor();
+    SysCalls::close(fileDescriptor);
 }
 
 } // namespace NEO

shared/test/common/os_interface/linux/sys_calls_linux_ult.h

@@ -28,5 +28,6 @@ constexpr int fakeFileDescriptor = 123;
 extern int passedFileDescriptorFlagsToSet;
 extern int getFileDescriptorFlagsCalled;
 extern int setFileDescriptorFlagsCalled;
+extern uint32_t closeFuncCalled;
 } // namespace SysCalls
 } // namespace NEO