[Sysman] Add support to control lifetime of file descriptor

This patch adds support to open and close the drm fd,
so that more granular control of drm fd could be
achieved.

Related-To: LOCI-3986

Signed-off-by: Joshua Santosh Ranjan <joshua.santosh.ranjan@intel.com>

level_zero/sysman/source/CMakeLists.txt

@@ -16,6 +16,8 @@ set(L0_SRCS_SYSMAN
     ${CMAKE_CURRENT_SOURCE_DIR}/sysman_device_imp.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/sysman_device_imp.h
     ${CMAKE_CURRENT_SOURCE_DIR}/os_sysman.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/sysman_hw_device_id.h
+
 )
 
 target_sources(${L0_STATIC_LIB_NAME}

level_zero/sysman/source/linux/CMakeLists.txt

@@ -10,6 +10,8 @@ set(L0_SRCS_SYSMAN_LINUX
     ${CMAKE_CURRENT_SOURCE_DIR}/os_sysman_imp.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/fs_access.h
     ${CMAKE_CURRENT_SOURCE_DIR}/fs_access.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id_linux.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id_linux.h
 )
 
 if(UNIX)

level_zero/sysman/source/linux/hw_device_id_linux.cpp

@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/sysman/source/linux/hw_device_id_linux.h"
+
+#include "shared/source/os_interface/linux/sys_calls.h"
+
+#include "level_zero/sysman/source/sysman_hw_device_id.h"
+
+#include <fcntl.h>
+
+namespace L0 {
+namespace Sysman {
+
+int SysmanHwDeviceIdDrm::openFileDescriptor() {
+
+    std::unique_lock<std::mutex> lock(fdMutex);
+    if (fileDescriptor == -1) {
+        fileDescriptor = NEO::SysCalls::open(devNodePath.c_str(), O_RDWR);
+    }
+    ++fdRefCounter;
+    return fileDescriptor;
+}
+
+int SysmanHwDeviceIdDrm::closeFileDescriptor() {
+
+    int closeStatus = 0;
+    std::unique_lock<std::mutex> lock(fdMutex);
+
+    DEBUG_BREAK_IF(fdRefCounter == 0u);
+    if (fdRefCounter > 0u) {
+        --fdRefCounter;
+        if (fdRefCounter == 0u && fileDescriptor >= 0) {
+            closeStatus = NEO::SysCalls::close(fileDescriptor);
+            fileDescriptor = -1;
+        }
+    }
+    return closeStatus;
+}
+
+std::unique_ptr<NEO::HwDeviceId> createSysmanHwDeviceId(std::unique_ptr<NEO::HwDeviceId> &hwDeviceId) {
+
+    const auto hwDeviceIdDrm = static_cast<NEO::HwDeviceIdDrm *>(hwDeviceId.get());
+    return std::make_unique<SysmanHwDeviceIdDrm>(hwDeviceIdDrm->getFileDescriptor(), hwDeviceIdDrm->getPciPath(), hwDeviceIdDrm->getDeviceNode());
+}
+
+} // namespace Sysman
+} // namespace L0

level_zero/sysman/source/linux/hw_device_id_linux.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+#include "shared/source/os_interface/linux/hw_device_id.h"
+
+#include <mutex>
+
+namespace L0 {
+namespace Sysman {
+class SysmanHwDeviceIdDrm : public NEO::HwDeviceIdDrm {
+
+  public:
+    using NEO::HwDeviceIdDrm::HwDeviceIdDrm;
+    SysmanHwDeviceIdDrm() = delete;
+    int openFileDescriptor();
+    int closeFileDescriptor();
+
+  private:
+    std::mutex fdMutex{};
+    uint32_t fdRefCounter = 0;
+};
+
+} // namespace Sysman
+} // namespace L0

level_zero/sysman/source/linux/os_sysman_imp.cpp

@@ -33,8 +33,9 @@ ze_result_t LinuxSysmanImp::init() {
     if (osInterface.getDriverModel()->getDriverModelType() != NEO::DriverModelType::DRM) {
         return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
     }
-    auto pDrm = osInterface.getDriverModel()->as<NEO::Drm>();
-    int myDeviceFd = pDrm->getFileDescriptor();
+    auto sysmanHwDeviceId = getSysmanHwDeviceId();
+    sysmanHwDeviceId->openFileDescriptor();
+    int myDeviceFd = sysmanHwDeviceId->getFileDescriptor();
     std::string myDeviceName;
     result = pProcfsAccess->getFileName(pProcfsAccess->myProcessId(), myDeviceFd, myDeviceName);
     if (ZE_RESULT_SUCCESS != result) {
@@ -53,10 +54,19 @@ ze_result_t LinuxSysmanImp::init() {
 
     osInterface.getDriverModel()->as<NEO::Drm>()->cleanup();
     // Close Drm handles
-    pDrm->closeFileDescriptor();
+    sysmanHwDeviceId->closeFileDescriptor();
     return createPmtHandles();
 }
 
+SysmanHwDeviceIdDrm *LinuxSysmanImp::getSysmanHwDeviceId() {
+    return static_cast<SysmanHwDeviceIdDrm *>(getDrm()->getHwDeviceId().get());
+}
+
+NEO::Drm *LinuxSysmanImp::getDrm() {
+    const auto &osInterface = *pParentSysmanDeviceImp->getRootDeviceEnvironment().osInterface;
+    return osInterface.getDriverModel()->as<NEO::Drm>();
+}
+
 ze_result_t LinuxSysmanImp::createPmtHandles() {
     std::string gtDevicePCIPath;
     auto result = pSysfsAccess->getRealPath("device", gtDevicePCIPath);

level_zero/sysman/source/linux/os_sysman_imp.h

@@ -9,11 +9,16 @@
 #include "shared/source/execution_environment/execution_environment.h"
 #include "shared/source/helpers/non_copyable_or_moveable.h"
 
+#include "level_zero/sysman/source/linux/hw_device_id_linux.h"
 #include "level_zero/sysman/source/os_sysman.h"
 #include "level_zero/sysman/source/sysman_device_imp.h"
 
 #include <map>
 
+namespace NEO {
+class Drm;
+} // namespace NEO
+
 namespace L0 {
 namespace Sysman {
 class PlatformMonitoringTech;
@@ -36,6 +41,8 @@ class LinuxSysmanImp : public OsSysman, NEO::NonCopyableOrMovableClass {
     std::string getPciCardBusDirectoryPath(std::string realPciPath);
     PlatformMonitoringTech *getPlatformMonitoringTechAccess(uint32_t subDeviceId);
     PRODUCT_FAMILY getProductFamily() const { return pParentSysmanDeviceImp->getProductFamily(); }
+    SysmanHwDeviceIdDrm *getSysmanHwDeviceId();
+    NEO::Drm *getDrm();
     void releasePmtObject();
     ze_result_t createPmtHandles();
     std::string devicePciBdf = "";

level_zero/sysman/source/sysman_driver.cpp

@@ -14,6 +14,7 @@
 #include "level_zero/core/source/driver/driver.h"
 #include "level_zero/sysman/source/sysman_driver_handle_imp.h"
 #include "level_zero/sysman/source/sysman_driver_imp.h"
+#include "level_zero/sysman/source/sysman_hw_device_id.h"
 
 #include <cstring>
 #include <vector>
@@ -43,7 +44,12 @@ void SysmanDriverImp::initialize(ze_result_t *result) {
         executionEnvironment->prepareRootDeviceEnvironments(static_cast<uint32_t>(hwDeviceIds.size()));
         uint32_t rootDeviceIndex = 0u;
         for (auto &hwDeviceId : hwDeviceIds) {
-            if (!executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->initOsInterface(std::move(hwDeviceId), rootDeviceIndex)) {
+
+            auto sysmanHwDeviceId = createSysmanHwDeviceId(hwDeviceId);
+            auto initStatus = sysmanHwDeviceId != nullptr &&
+                              executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->initOsInterface(std::move(sysmanHwDeviceId), rootDeviceIndex);
+
+            if (!initStatus) {
                 NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr,
                                       "OsInterface initialization failed for device : %d\n", rootDeviceIndex);
                 *result = ZE_RESULT_ERROR_UNINITIALIZED;

level_zero/sysman/source/sysman_hw_device_id.h

@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+#include <memory>
+#include <vector>
+
+namespace NEO {
+class HwDeviceId;
+} // namespace NEO
+
+namespace L0 {
+namespace Sysman {
+std::unique_ptr<NEO::HwDeviceId> createSysmanHwDeviceId(std::unique_ptr<NEO::HwDeviceId> &hwDeviceId);
+} // namespace Sysman
+} // namespace L0

level_zero/sysman/source/windows/CMakeLists.txt

@@ -8,6 +8,7 @@ set(L0_SRCS_SYSMAN_WINDOWS
     ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
     ${CMAKE_CURRENT_SOURCE_DIR}/os_sysman_imp.h
     ${CMAKE_CURRENT_SOURCE_DIR}/os_sysman_imp.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id_windows.cpp
 )
 
 if(WIN32)

level_zero/sysman/source/windows/hw_device_id_windows.cpp

@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/os_interface/os_interface.h"
+
+#include "level_zero/sysman/source/sysman_hw_device_id.h"
+
+namespace L0 {
+namespace Sysman {
+
+std::unique_ptr<NEO::HwDeviceId> createSysmanHwDeviceId(std::unique_ptr<NEO::HwDeviceId> &hwDeviceId) {
+    return std::unique_ptr<NEO::HwDeviceId>{};
+}
+
+} // namespace Sysman
+} // namespace L0

level_zero/sysman/test/unit_tests/sources/linux/CMakeLists.txt

@@ -12,6 +12,7 @@ if(UNIX)
                  ${CMAKE_CURRENT_SOURCE_DIR}/mock_sysman_drm.h
                  ${CMAKE_CURRENT_SOURCE_DIR}/mock_sysman_driver.h
                  ${CMAKE_CURRENT_SOURCE_DIR}/test_sysman_driver.cpp
+                 ${CMAKE_CURRENT_SOURCE_DIR}/test_sysman_hw_device_id.cpp
   )
 endif()
 

level_zero/sysman/test/unit_tests/sources/linux/mock_sysman_drm.h

@@ -12,12 +12,14 @@
 #include "shared/source/os_interface/linux/drm_neo.h"
 #include "shared/source/os_interface/os_interface.h"
 
+#include "level_zero/sysman/source/linux/hw_device_id_linux.h"
+
 namespace L0 {
 namespace ult {
-constexpr int mockFd = 0;
+constexpr int mockFd = -1;
 class SysmanMockDrm : public NEO::Drm {
   public:
-    SysmanMockDrm(NEO::RootDeviceEnvironment &rootDeviceEnvironment) : Drm(std::make_unique<NEO::HwDeviceIdDrm>(mockFd, ""), rootDeviceEnvironment) {
+    SysmanMockDrm(NEO::RootDeviceEnvironment &rootDeviceEnvironment) : Drm(std::make_unique<L0::Sysman::SysmanHwDeviceIdDrm>(mockFd, ""), rootDeviceEnvironment) {
         setupIoctlHelper(rootDeviceEnvironment.getHardwareInfo()->platform.eProductFamily);
     }
 };

level_zero/sysman/test/unit_tests/sources/linux/mock_sysman_fixture.h

@@ -18,6 +18,7 @@
 #include "level_zero/sysman/source/linux/os_sysman_imp.h"
 #include "level_zero/sysman/source/sysman_device.h"
 #include "level_zero/sysman/source/sysman_driver_handle_imp.h"
+#include "level_zero/sysman/test/unit_tests/sources/linux/mock_sysman_drm.h"
 
 #include "gtest/gtest.h"
 
@@ -25,14 +26,6 @@ using namespace NEO;
 
 namespace L0 {
 namespace ult {
-constexpr int mockFd = 0;
-class SysmanMockDrm : public Drm {
-  public:
-    SysmanMockDrm(RootDeviceEnvironment &rootDeviceEnvironment) : Drm(std::make_unique<HwDeviceIdDrm>(mockFd, ""), rootDeviceEnvironment) {
-        setupIoctlHelper(rootDeviceEnvironment.getHardwareInfo()->platform.eProductFamily);
-    }
-};
-
 class PublicLinuxSysmanImp : public L0::Sysman::LinuxSysmanImp {
   public:
     using LinuxSysmanImp::mapOfSubDeviceIdToPmtObject;

level_zero/sysman/test/unit_tests/sources/linux/test_sysman_hw_device_id.cpp

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/test/common/helpers/variable_backup.h"
+#include "shared/test/common/os_interface/linux/sys_calls_linux_ult.h"
+
+#include "level_zero/sysman/source/linux/hw_device_id_linux.h"
+
+#include "gtest/gtest.h"
+
+#include <bitset>
+#include <cstring>
+
+namespace L0 {
+namespace ult {
+
+TEST(sysmanHwDeviceIdTest, whenOpenFileDescriptorIsCalledMultipleTimesThenOpenIsCalledOnlyOnce) {
+
+    L0::Sysman::SysmanHwDeviceIdDrm hwDeviceId(-1, "mockPciPath", "mockDevNodePath");
+    static uint32_t openCallCount = 0;
+
+    VariableBackup<decltype(NEO::SysCalls::sysCallsOpen)> mockOpen(&NEO::SysCalls::sysCallsOpen, [](const char *pathname, int flags) -> int {
+        openCallCount++;
+        return 1;
+    });
+
+    openCallCount = 0;
+    hwDeviceId.openFileDescriptor();
+    EXPECT_EQ(openCallCount, 1u);
+    hwDeviceId.openFileDescriptor();
+    EXPECT_EQ(openCallCount, 1u);
+}
+
+TEST(sysmanHwDeviceIdTest, whenOpenFileDescriptorIsCalledMultipleTimesThenCloseIsCalledOnlyAfterAllReferencesAreClosed) {
+
+    L0::Sysman::SysmanHwDeviceIdDrm hwDeviceId(-1, "mockPciPath", "mockDevNodePath");
+    VariableBackup<decltype(NEO::SysCalls::sysCallsOpen)> mockOpen(&NEO::SysCalls::sysCallsOpen, [](const char *pathname, int flags) -> int {
+        return 1;
+    });
+
+    const auto referenceCloseCount = NEO::SysCalls::closeFuncCalled;
+    hwDeviceId.openFileDescriptor();
+    hwDeviceId.openFileDescriptor();
+    hwDeviceId.openFileDescriptor();
+    hwDeviceId.closeFileDescriptor();
+    EXPECT_EQ(NEO::SysCalls::closeFuncCalled, referenceCloseCount);
+    hwDeviceId.closeFileDescriptor();
+    EXPECT_EQ(NEO::SysCalls::closeFuncCalled, referenceCloseCount);
+    hwDeviceId.closeFileDescriptor();
+    EXPECT_EQ(NEO::SysCalls::closeFuncCalled, referenceCloseCount + 1u);
+}
+
+} // namespace ult
+} // namespace L0