From 5edbca1aa2509246c6af74e4fd8c36d94687dcf5 Mon Sep 17 00:00:00 2001 From: Mayank Raghuwanshi Date: Mon, 14 Nov 2022 09:44:08 +0000 Subject: [PATCH] Use physical subdevice for sysman engine module Related-To: LOCI-3231 Signed-off-by: Mayank Raghuwanshi --- .../tools/source/sysman/engine/engine.cpp | 19 ++++++++++++----- .../tools/source/sysman/engine/engine.h | 5 +++-- .../tools/source/sysman/engine/engine_imp.cpp | 4 ++-- .../tools/source/sysman/engine/engine_imp.h | 4 ++-- .../sysman/engine/linux/os_engine_imp.cpp | 8 +++---- .../sysman/engine/linux/os_engine_imp.h | 4 ++-- .../engine/linux/os_engine_imp_prelim.cpp | 9 +++----- .../tools/source/sysman/engine/os_engine.h | 2 +- .../sysman/engine/windows/os_engine_imp.cpp | 2 +- .../source/sysman/linux/os_sysman_imp.cpp | 2 +- .../linux/mock_zes_sysman_diagnostics.h | 2 +- .../sysman/engine/linux/test_zes_engine.cpp | 21 ++++++++++++++++--- .../linux/mock_global_operations.h | 4 ++-- .../linux/test_zes_global_operations.cpp | 2 +- 14 files changed, 55 insertions(+), 33 deletions(-) diff --git a/level_zero/tools/source/sysman/engine/engine.cpp b/level_zero/tools/source/sysman/engine/engine.cpp index 8fb7c485bc..79a4d962bc 100644 --- a/level_zero/tools/source/sysman/engine/engine.cpp +++ b/level_zero/tools/source/sysman/engine/engine.cpp @@ -10,6 +10,8 @@ #include "shared/source/helpers/basic_math.h" #include "level_zero/tools/source/sysman/engine/engine_imp.h" +#include "level_zero/tools/source/sysman/os_sysman.h" +#include "level_zero/tools/source/sysman/sysman_imp.h" class OsEngine; namespace L0 { @@ -21,8 +23,8 @@ EngineHandleContext::~EngineHandleContext() { releaseEngines(); } -void EngineHandleContext::createHandle(zes_engine_group_t engineType, uint32_t engineInstance, uint32_t subDeviceId) { - Engine *pEngine = new EngineImp(pOsSysman, engineType, engineInstance, subDeviceId); +void EngineHandleContext::createHandle(zes_engine_group_t engineType, uint32_t engineInstance, uint32_t subDeviceId, ze_bool_t onSubdevice) { + Engine *pEngine = new EngineImp(pOsSysman, engineType, engineInstance, subDeviceId, onSubdevice); if (pEngine->initSuccess == true) { handleList.push_back(pEngine); } else { @@ -30,11 +32,18 @@ void EngineHandleContext::createHandle(zes_engine_group_t engineType, uint32_t e } } -void EngineHandleContext::init() { +void EngineHandleContext::init(std::vector &deviceHandles) { std::set> engineGroupInstance = {}; //set contains pair of engine group and struct containing engine instance and subdeviceId OsEngine::getNumEngineTypeAndInstances(engineGroupInstance, pOsSysman); for (auto itr = engineGroupInstance.begin(); itr != engineGroupInstance.end(); ++itr) { - createHandle(itr->first, itr->second.first, itr->second.second); + for (const auto &deviceHandle : deviceHandles) { + uint32_t subDeviceId = 0; + ze_bool_t onSubdevice = false; + SysmanDeviceImp::getSysmanDeviceInfo(deviceHandle, subDeviceId, onSubdevice); + if (subDeviceId == itr->second.second) { + createHandle(itr->first, itr->second.first, subDeviceId, onSubdevice); + } + } } } @@ -47,7 +56,7 @@ void EngineHandleContext::releaseEngines() { ze_result_t EngineHandleContext::engineGet(uint32_t *pCount, zes_engine_handle_t *phEngine) { std::call_once(initEngineOnce, [this]() { - this->init(); + this->init(pOsSysman->getDeviceHandles()); this->engineInitDone = true; }); uint32_t handleListSize = static_cast(handleList.size()); diff --git a/level_zero/tools/source/sysman/engine/engine.h b/level_zero/tools/source/sysman/engine/engine.h index 551902a4de..7572f666ba 100644 --- a/level_zero/tools/source/sysman/engine/engine.h +++ b/level_zero/tools/source/sysman/engine/engine.h @@ -6,6 +6,7 @@ */ #pragma once +#include "level_zero/core/source/device/device.h" #include #include @@ -36,7 +37,7 @@ struct EngineHandleContext { EngineHandleContext(OsSysman *pOsSysman); MOCKABLE_VIRTUAL ~EngineHandleContext(); - MOCKABLE_VIRTUAL void init(); + MOCKABLE_VIRTUAL void init(std::vector &deviceHandles); void releaseEngines(); ze_result_t engineGet(uint32_t *pCount, zes_engine_handle_t *phEngine); @@ -48,7 +49,7 @@ struct EngineHandleContext { } private: - void createHandle(zes_engine_group_t engineType, uint32_t engineInstance, uint32_t subDeviceId); + void createHandle(zes_engine_group_t engineType, uint32_t engineInstance, uint32_t subDeviceId, ze_bool_t onSubdevice); std::once_flag initEngineOnce; bool engineInitDone = false; }; diff --git a/level_zero/tools/source/sysman/engine/engine_imp.cpp b/level_zero/tools/source/sysman/engine/engine_imp.cpp index f2011e20c0..feb545be34 100644 --- a/level_zero/tools/source/sysman/engine/engine_imp.cpp +++ b/level_zero/tools/source/sysman/engine/engine_imp.cpp @@ -25,8 +25,8 @@ void EngineImp::init() { } } -EngineImp::EngineImp(OsSysman *pOsSysman, zes_engine_group_t engineType, uint32_t engineInstance, uint32_t subDeviceId) { - pOsEngine = OsEngine::create(pOsSysman, engineType, engineInstance, subDeviceId); +EngineImp::EngineImp(OsSysman *pOsSysman, zes_engine_group_t engineType, uint32_t engineInstance, uint32_t subDeviceId, ze_bool_t onSubdevice) { + pOsEngine = OsEngine::create(pOsSysman, engineType, engineInstance, subDeviceId, onSubdevice); init(); } diff --git a/level_zero/tools/source/sysman/engine/engine_imp.h b/level_zero/tools/source/sysman/engine/engine_imp.h index 4f786bbe19..45dac6e265 100644 --- a/level_zero/tools/source/sysman/engine/engine_imp.h +++ b/level_zero/tools/source/sysman/engine/engine_imp.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2021 Intel Corporation + * Copyright (C) 2020-2022 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -20,7 +20,7 @@ class EngineImp : public Engine, NEO::NonCopyableOrMovableClass { ze_result_t engineGetActivity(zes_engine_stats_t *pStats) override; EngineImp() = default; - EngineImp(OsSysman *pOsSysman, zes_engine_group_t engineType, uint32_t engineInstance, uint32_t subDeviceId); + EngineImp(OsSysman *pOsSysman, zes_engine_group_t engineType, uint32_t engineInstance, uint32_t subDeviceId, ze_bool_t onSubdevice); ~EngineImp() override; OsEngine *pOsEngine = nullptr; diff --git a/level_zero/tools/source/sysman/engine/linux/os_engine_imp.cpp b/level_zero/tools/source/sysman/engine/linux/os_engine_imp.cpp index d1bab04cf9..573f1caadf 100644 --- a/level_zero/tools/source/sysman/engine/linux/os_engine_imp.cpp +++ b/level_zero/tools/source/sysman/engine/linux/os_engine_imp.cpp @@ -63,7 +63,7 @@ ze_result_t LinuxEngineImp::getActivity(zes_engine_stats_t *pStats) { ze_result_t LinuxEngineImp::getProperties(zes_engine_properties_t &properties) { properties.type = engineGroup; - properties.onSubdevice = 0; + properties.onSubdevice = onSubDevice; properties.subdeviceId = subDeviceId; return ZE_RESULT_SUCCESS; } @@ -81,7 +81,7 @@ bool LinuxEngineImp::isEngineModuleSupported() { return true; } -LinuxEngineImp::LinuxEngineImp(OsSysman *pOsSysman, zes_engine_group_t type, uint32_t engineInstance, uint32_t subDeviceId) : engineGroup(type), engineInstance(engineInstance), subDeviceId(subDeviceId) { +LinuxEngineImp::LinuxEngineImp(OsSysman *pOsSysman, zes_engine_group_t type, uint32_t engineInstance, uint32_t subDeviceId, ze_bool_t onSubDevice) : engineGroup(type), engineInstance(engineInstance), subDeviceId(subDeviceId), onSubDevice(onSubDevice) { LinuxSysmanImp *pLinuxSysmanImp = static_cast(pOsSysman); pDrm = &pLinuxSysmanImp->getDrm(); pDevice = pLinuxSysmanImp->getDeviceHandle(); @@ -89,8 +89,8 @@ LinuxEngineImp::LinuxEngineImp(OsSysman *pOsSysman, zes_engine_group_t type, uin init(); } -OsEngine *OsEngine::create(OsSysman *pOsSysman, zes_engine_group_t type, uint32_t engineInstance, uint32_t subDeviceId) { - LinuxEngineImp *pLinuxEngineImp = new LinuxEngineImp(pOsSysman, type, engineInstance, subDeviceId); +OsEngine *OsEngine::create(OsSysman *pOsSysman, zes_engine_group_t type, uint32_t engineInstance, uint32_t subDeviceId, ze_bool_t onSubDevice) { + LinuxEngineImp *pLinuxEngineImp = new LinuxEngineImp(pOsSysman, type, engineInstance, subDeviceId, onSubDevice); return static_cast(pLinuxEngineImp); } diff --git a/level_zero/tools/source/sysman/engine/linux/os_engine_imp.h b/level_zero/tools/source/sysman/engine/linux/os_engine_imp.h index ccb6eb0a68..fbd3ba14ab 100644 --- a/level_zero/tools/source/sysman/engine/linux/os_engine_imp.h +++ b/level_zero/tools/source/sysman/engine/linux/os_engine_imp.h @@ -22,7 +22,7 @@ class LinuxEngineImp : public OsEngine, NEO::NonCopyableOrMovableClass { bool isEngineModuleSupported() override; static zes_engine_group_t getGroupFromEngineType(zes_engine_group_t type); LinuxEngineImp() = default; - LinuxEngineImp(OsSysman *pOsSysman, zes_engine_group_t type, uint32_t engineInstance, uint32_t subDeviceId); + LinuxEngineImp(OsSysman *pOsSysman, zes_engine_group_t type, uint32_t engineInstance, uint32_t subDeviceId, ze_bool_t onSubDevice); ~LinuxEngineImp() override { if (fd != -1) { close(static_cast(fd)); @@ -37,7 +37,7 @@ class LinuxEngineImp : public OsEngine, NEO::NonCopyableOrMovableClass { NEO::Drm *pDrm = nullptr; Device *pDevice = nullptr; uint32_t subDeviceId = 0; - uint32_t onSubDevice = 0; + ze_bool_t onSubDevice = false; private: void init(); diff --git a/level_zero/tools/source/sysman/engine/linux/os_engine_imp_prelim.cpp b/level_zero/tools/source/sysman/engine/linux/os_engine_imp_prelim.cpp index 3f664b8951..8b1801fb19 100644 --- a/level_zero/tools/source/sysman/engine/linux/os_engine_imp_prelim.cpp +++ b/level_zero/tools/source/sysman/engine/linux/os_engine_imp_prelim.cpp @@ -91,9 +91,6 @@ ze_result_t LinuxEngineImp::getProperties(zes_engine_properties_t &properties) { } void LinuxEngineImp::init() { - uint32_t subDeviceCount = 0; - pDevice->getSubDevices(&subDeviceCount, nullptr); - onSubDevice = (subDeviceCount == 0) ? 0 : 1; uint64_t config = UINT64_MAX; switch (engineGroup) { case ZES_ENGINE_GROUP_ALL: @@ -124,7 +121,7 @@ bool LinuxEngineImp::isEngineModuleSupported() { return true; } -LinuxEngineImp::LinuxEngineImp(OsSysman *pOsSysman, zes_engine_group_t type, uint32_t engineInstance, uint32_t subDeviceId) : engineGroup(type), engineInstance(engineInstance), subDeviceId(subDeviceId) { +LinuxEngineImp::LinuxEngineImp(OsSysman *pOsSysman, zes_engine_group_t type, uint32_t engineInstance, uint32_t subDeviceId, ze_bool_t onSubDevice) : engineGroup(type), engineInstance(engineInstance), subDeviceId(subDeviceId), onSubDevice(onSubDevice) { LinuxSysmanImp *pLinuxSysmanImp = static_cast(pOsSysman); pDrm = &pLinuxSysmanImp->getDrm(); pDevice = pLinuxSysmanImp->getDeviceHandle(); @@ -132,8 +129,8 @@ LinuxEngineImp::LinuxEngineImp(OsSysman *pOsSysman, zes_engine_group_t type, uin init(); } -OsEngine *OsEngine::create(OsSysman *pOsSysman, zes_engine_group_t type, uint32_t engineInstance, uint32_t subDeviceId) { - LinuxEngineImp *pLinuxEngineImp = new LinuxEngineImp(pOsSysman, type, engineInstance, subDeviceId); +OsEngine *OsEngine::create(OsSysman *pOsSysman, zes_engine_group_t type, uint32_t engineInstance, uint32_t subDeviceId, ze_bool_t onSubDevice) { + LinuxEngineImp *pLinuxEngineImp = new LinuxEngineImp(pOsSysman, type, engineInstance, subDeviceId, onSubDevice); return static_cast(pLinuxEngineImp); } diff --git a/level_zero/tools/source/sysman/engine/os_engine.h b/level_zero/tools/source/sysman/engine/os_engine.h index a60f826df2..67458f1275 100644 --- a/level_zero/tools/source/sysman/engine/os_engine.h +++ b/level_zero/tools/source/sysman/engine/os_engine.h @@ -22,7 +22,7 @@ class OsEngine { virtual ze_result_t getActivity(zes_engine_stats_t *pStats) = 0; virtual ze_result_t getProperties(zes_engine_properties_t &properties) = 0; virtual bool isEngineModuleSupported() = 0; - static OsEngine *create(OsSysman *pOsSysman, zes_engine_group_t engineType, uint32_t engineInstance, uint32_t subDeviceId); + static OsEngine *create(OsSysman *pOsSysman, zes_engine_group_t engineType, uint32_t engineInstance, uint32_t subDeviceId, ze_bool_t onSubdevice); static ze_result_t getNumEngineTypeAndInstances(std::set> &engineGroupInstance, OsSysman *pOsSysman); virtual ~OsEngine() = default; }; diff --git a/level_zero/tools/source/sysman/engine/windows/os_engine_imp.cpp b/level_zero/tools/source/sysman/engine/windows/os_engine_imp.cpp index 1f6a91a620..6edabe3b20 100644 --- a/level_zero/tools/source/sysman/engine/windows/os_engine_imp.cpp +++ b/level_zero/tools/source/sysman/engine/windows/os_engine_imp.cpp @@ -67,7 +67,7 @@ WddmEngineImp::WddmEngineImp(OsSysman *pOsSysman, zes_engine_group_t engineType, pKmdSysManager = &pWddmSysmanImp->getKmdSysManager(); } -OsEngine *OsEngine::create(OsSysman *pOsSysman, zes_engine_group_t engineType, uint32_t engineInstance, uint32_t subDeviceId) { +OsEngine *OsEngine::create(OsSysman *pOsSysman, zes_engine_group_t engineType, uint32_t engineInstance, uint32_t subDeviceId, ze_bool_t onSubDevice) { WddmEngineImp *pWddmEngineImp = new WddmEngineImp(pOsSysman, engineType, engineInstance, subDeviceId); return static_cast(pWddmEngineImp); } diff --git a/level_zero/tools/source/sysman/linux/os_sysman_imp.cpp b/level_zero/tools/source/sysman/linux/os_sysman_imp.cpp index ebf901293f..a4ebca4476 100644 --- a/level_zero/tools/source/sysman/linux/os_sysman_imp.cpp +++ b/level_zero/tools/source/sysman/linux/os_sysman_imp.cpp @@ -335,7 +335,7 @@ void LinuxSysmanImp::reInitSysmanDeviceResources() { getSysmanDeviceImp()->pRasHandleContext->init(getSysmanDeviceImp()->deviceHandles); } if (getSysmanDeviceImp()->pEngineHandleContext->isEngineInitDone()) { - getSysmanDeviceImp()->pEngineHandleContext->init(); + getSysmanDeviceImp()->pEngineHandleContext->init(getSysmanDeviceImp()->deviceHandles); } if (!diagnosticsReset) { if (getSysmanDeviceImp()->pDiagnosticsHandleContext->isDiagnosticsInitDone()) { diff --git a/level_zero/tools/test/unit_tests/sources/sysman/diagnostics/linux/mock_zes_sysman_diagnostics.h b/level_zero/tools/test/unit_tests/sources/sysman/diagnostics/linux/mock_zes_sysman_diagnostics.h index 14c5752458..f803238d44 100644 --- a/level_zero/tools/test/unit_tests/sources/sysman/diagnostics/linux/mock_zes_sysman_diagnostics.h +++ b/level_zero/tools/test/unit_tests/sources/sysman/diagnostics/linux/mock_zes_sysman_diagnostics.h @@ -69,7 +69,7 @@ struct Mock : public DiagnosticsFwInterface { }; struct MockGlobalOperationsEngineHandleContext : public EngineHandleContext { MockGlobalOperationsEngineHandleContext(OsSysman *pOsSysman) : EngineHandleContext(pOsSysman) {} - void init() override {} + void init(std::vector &deviceHandles) override {} }; class DiagFsAccess : public FsAccess {}; diff --git a/level_zero/tools/test/unit_tests/sources/sysman/engine/linux/test_zes_engine.cpp b/level_zero/tools/test/unit_tests/sources/sysman/engine/linux/test_zes_engine.cpp index 054ad507d9..00cd98fc3a 100644 --- a/level_zero/tools/test/unit_tests/sources/sysman/engine/linux/test_zes_engine.cpp +++ b/level_zero/tools/test/unit_tests/sources/sysman/engine/linux/test_zes_engine.cpp @@ -19,6 +19,7 @@ namespace ult { constexpr uint32_t handleComponentCount = 6u; class ZesEngineFixture : public SysmanDeviceFixture { protected: + std::vector deviceHandles; std::unique_ptr> pDrm; std::unique_ptr> pPmuInterface; Drm *pOriginalDrm = nullptr; @@ -68,6 +69,20 @@ class ZesEngineFixture : public SysmanDeviceFixture { ON_CALL(*pFsAccess.get(), read(_, _)) .WillByDefault(::testing::Invoke(pFsAccess.get(), &Mock::readValSuccess)); + for (auto handle : pSysmanDeviceImp->pEngineHandleContext->handleList) { + delete handle; + } + + pSysmanDeviceImp->pEngineHandleContext->handleList.clear(); + uint32_t subDeviceCount = 0; + // We received a device handle. Check for subdevices in this device + Device::fromHandle(device->toHandle())->getSubDevices(&subDeviceCount, nullptr); + if (subDeviceCount == 0) { + deviceHandles.resize(1, device->toHandle()); + } else { + deviceHandles.resize(subDeviceCount, nullptr); + Device::fromHandle(device->toHandle())->getSubDevices(&subDeviceCount, deviceHandles.data()); + } getEngineHandles(0); } @@ -169,7 +184,7 @@ TEST_F(ZesEngineFixture, GivenTestDiscreteDevicesAndValidEngineHandleWhenCalling ON_CALL(*pSysfsAccess.get(), readSymLink(_, _)) .WillByDefault(::testing::Invoke(pSysfsAccess.get(), &Mock::getValStringSymLinkFailure)); - auto pOsEngineTest1 = OsEngine::create(pOsSysman, ZES_ENGINE_GROUP_RENDER_SINGLE, 0u, 0u); + auto pOsEngineTest1 = OsEngine::create(pOsSysman, ZES_ENGINE_GROUP_RENDER_SINGLE, 0u, 0u, false); zes_engine_stats_t stats = {}; EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, pOsEngineTest1->getActivity(&stats)); @@ -179,7 +194,7 @@ TEST_F(ZesEngineFixture, GivenTestDiscreteDevicesAndValidEngineHandleWhenCalling ON_CALL(*pFsAccess.get(), read(_, _)) .WillByDefault(::testing::Invoke(pFsAccess.get(), &Mock::readValFailure)); - auto pOsEngineTest2 = OsEngine::create(pOsSysman, ZES_ENGINE_GROUP_RENDER_SINGLE, 0u, 0u); + auto pOsEngineTest2 = OsEngine::create(pOsSysman, ZES_ENGINE_GROUP_RENDER_SINGLE, 0u, 0u, false); EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, pOsEngineTest2->getActivity(&stats)); delete pOsEngineTest1; delete pOsEngineTest2; @@ -191,7 +206,7 @@ TEST_F(ZesEngineFixture, GivenTestIntegratedDevicesAndValidEngineHandleWhenCalli ON_CALL(*pFsAccess.get(), read(_, _)) .WillByDefault(::testing::Invoke(pFsAccess.get(), &Mock::readValFailure)); - auto pOsEngineTest1 = OsEngine::create(pOsSysman, ZES_ENGINE_GROUP_RENDER_SINGLE, 0u, 0u); + auto pOsEngineTest1 = OsEngine::create(pOsSysman, ZES_ENGINE_GROUP_RENDER_SINGLE, 0u, 0u, false); EXPECT_EQ(ZE_RESULT_ERROR_UNSUPPORTED_FEATURE, pOsEngineTest1->getActivity(&stats)); delete pOsEngineTest1; } diff --git a/level_zero/tools/test/unit_tests/sources/sysman/global_operations/linux/mock_global_operations.h b/level_zero/tools/test/unit_tests/sources/sysman/global_operations/linux/mock_global_operations.h index 0cbefb7d88..79280f23cf 100644 --- a/level_zero/tools/test/unit_tests/sources/sysman/global_operations/linux/mock_global_operations.h +++ b/level_zero/tools/test/unit_tests/sources/sysman/global_operations/linux/mock_global_operations.h @@ -56,9 +56,9 @@ struct GlobalOperationsEngineHandleContext : public EngineHandleContext { }; template <> struct Mock : public GlobalOperationsEngineHandleContext { - void initMock() {} + void initMock(std::vector &deviceHandles) {} Mock(OsSysman *pOsSysman) : GlobalOperationsEngineHandleContext(pOsSysman) {} - MOCK_METHOD(void, init, (), (override)); + MOCK_METHOD(void, init, (std::vector & deviceHandles), (override)); }; struct GlobalOperationsRasHandleContext : public RasHandleContext { diff --git a/level_zero/tools/test/unit_tests/sources/sysman/global_operations/linux/test_zes_global_operations.cpp b/level_zero/tools/test/unit_tests/sources/sysman/global_operations/linux/test_zes_global_operations.cpp index 1114af768f..408d3ce200 100644 --- a/level_zero/tools/test/unit_tests/sources/sysman/global_operations/linux/test_zes_global_operations.cpp +++ b/level_zero/tools/test/unit_tests/sources/sysman/global_operations/linux/test_zes_global_operations.cpp @@ -101,7 +101,7 @@ class SysmanGlobalOperationsFixture : public SysmanDeviceFixture { ON_CALL(*pRasHandleContext.get(), init(_)) .WillByDefault(::testing::Invoke(pRasHandleContext.get(), &Mock::initMock)); - ON_CALL(*pEngineHandleContext.get(), init()) + ON_CALL(*pEngineHandleContext.get(), init(_)) .WillByDefault(::testing::Invoke(pEngineHandleContext.get(), &Mock::initMock)); ON_CALL(*pSysfsAccess.get(), read(_, Matcher(_))) .WillByDefault(::testing::Invoke(pSysfsAccess.get(), &Mock::getValString));