diff --git a/level_zero/tools/source/sysman/frequency/frequency.cpp b/level_zero/tools/source/sysman/frequency/frequency.cpp index 5632e3a0f3..7b74111b5a 100644 --- a/level_zero/tools/source/sysman/frequency/frequency.cpp +++ b/level_zero/tools/source/sysman/frequency/frequency.cpp @@ -25,8 +25,8 @@ void FrequencyHandleContext::createHandle(ze_device_handle_t deviceHandle, zes_f handleList.push_back(pFrequency); } -ze_result_t FrequencyHandleContext::init(std::vector deviceHandles) { - for (auto deviceHandle : deviceHandles) { +ze_result_t FrequencyHandleContext::init(std::vector &deviceHandles) { + for (const auto &deviceHandle : deviceHandles) { auto totalDomains = OsFrequency::getNumberOfFreqDoainsSupported(pOsSysman); UNRECOVERABLE_IF(totalDomains > 2); for (uint32_t frequencyDomain = 0; frequencyDomain < totalDomains; frequencyDomain++) { diff --git a/level_zero/tools/source/sysman/frequency/frequency.h b/level_zero/tools/source/sysman/frequency/frequency.h index a0cd45d6c7..840e8dd58c 100644 --- a/level_zero/tools/source/sysman/frequency/frequency.h +++ b/level_zero/tools/source/sysman/frequency/frequency.h @@ -55,7 +55,7 @@ struct FrequencyHandleContext { FrequencyHandleContext(OsSysman *pOsSysman) : pOsSysman(pOsSysman){}; ~FrequencyHandleContext(); - ze_result_t init(std::vector deviceHandles); + ze_result_t init(std::vector &deviceHandles); ze_result_t frequencyGet(uint32_t *pCount, zes_freq_handle_t *phFrequency); diff --git a/level_zero/tools/source/sysman/scheduler/linux/os_scheduler_imp.cpp b/level_zero/tools/source/sysman/scheduler/linux/os_scheduler_imp.cpp index 55f6f35ebe..0320d34f4b 100644 --- a/level_zero/tools/source/sysman/scheduler/linux/os_scheduler_imp.cpp +++ b/level_zero/tools/source/sysman/scheduler/linux/os_scheduler_imp.cpp @@ -20,7 +20,8 @@ const std::string LinuxSchedulerImp::defaultHeartbeatIntervalMilliSecs(".default const std::string LinuxSchedulerImp::engineDir("engine"); ze_result_t LinuxSchedulerImp::getProperties(zes_sched_properties_t &schedProperties) { - schedProperties.onSubdevice = false; + schedProperties.onSubdevice = onSubdevice; + schedProperties.subdeviceId = subdeviceId; schedProperties.canControl = canControlScheduler(); schedProperties.engines = this->engineType; schedProperties.supportedModes = (1 << ZES_SCHED_MODE_TIMEOUT) | (1 << ZES_SCHED_MODE_TIMESLICE) | (1 << ZES_SCHED_MODE_EXCLUSIVE); @@ -175,9 +176,7 @@ static const std::multimap level0EngineType {ZES_ENGINE_TYPE_FLAG_MEDIA, "vcs"}, {ZES_ENGINE_TYPE_FLAG_OTHER, "vecs"}}; -ze_result_t OsScheduler::getNumEngineTypeAndInstances(std::map> &mapOfEngines, OsSysman *pOsSysman) { - LinuxSysmanImp *pLinuxSysmanImp = static_cast(pOsSysman); - auto pSysfsAccess = &pLinuxSysmanImp->getSysfsAccess(); +static ze_result_t getNumEngineTypeAndInstancesForDevice(std::map> &mapOfEngines, SysfsAccess *pSysfsAccess) { std::vector localListOfAllEngines = {}; auto result = pSysfsAccess->scanDirEntries(LinuxSchedulerImp::engineDir, localListOfAllEngines); if (ZE_RESULT_SUCCESS != result) { @@ -206,15 +205,24 @@ ze_result_t OsScheduler::getNumEngineTypeAndInstances(std::map &listOfEngines) { +ze_result_t OsScheduler::getNumEngineTypeAndInstances( + std::map> &mapOfEngines, OsSysman *pOsSysman, ze_device_handle_t subdeviceHandle) { + LinuxSysmanImp *pLinuxSysmanImp = static_cast(pOsSysman); + auto pSysfsAccess = &pLinuxSysmanImp->getSysfsAccess(); + return getNumEngineTypeAndInstancesForDevice(mapOfEngines, pSysfsAccess); +} + +LinuxSchedulerImp::LinuxSchedulerImp( + OsSysman *pOsSysman, zes_engine_type_flag_t type, std::vector &listOfEngines, ze_bool_t isSubdevice, + uint32_t subdeviceId) : engineType(type), onSubdevice(isSubdevice), subdeviceId(subdeviceId) { LinuxSysmanImp *pLinuxSysmanImp = static_cast(pOsSysman); pSysfsAccess = &pLinuxSysmanImp->getSysfsAccess(); - engineType = type; this->listOfEngines = listOfEngines; } -OsScheduler *OsScheduler::create(OsSysman *pOsSysman, zes_engine_type_flag_t type, std::vector &listOfEngines) { - LinuxSchedulerImp *pLinuxSchedulerImp = new LinuxSchedulerImp(pOsSysman, type, listOfEngines); +OsScheduler *OsScheduler::create( + OsSysman *pOsSysman, zes_engine_type_flag_t type, std::vector &listOfEngines, ze_bool_t isSubdevice, uint32_t subdeviceId) { + LinuxSchedulerImp *pLinuxSchedulerImp = new LinuxSchedulerImp(pOsSysman, type, listOfEngines, isSubdevice, subdeviceId); return static_cast(pLinuxSchedulerImp); } diff --git a/level_zero/tools/source/sysman/scheduler/linux/os_scheduler_imp.h b/level_zero/tools/source/sysman/scheduler/linux/os_scheduler_imp.h index 24c700c615..0e4b39ce39 100644 --- a/level_zero/tools/source/sysman/scheduler/linux/os_scheduler_imp.h +++ b/level_zero/tools/source/sysman/scheduler/linux/os_scheduler_imp.h @@ -6,12 +6,15 @@ */ #pragma once +#include "shared/source/os_interface/linux/drm_neo.h" + #include "sysman/scheduler/scheduler_imp.h" #include namespace L0 { class SysfsAccess; +struct Device; // Following below mappings of scheduler properties with sysfs nodes // zes_sched_timeslice_properties_t.interval = timeslice_duration_ms @@ -28,13 +31,17 @@ class LinuxSchedulerImp : public OsScheduler, NEO::NonCopyableOrMovableClass { ze_bool_t canControlScheduler() override; ze_result_t getProperties(zes_sched_properties_t &properties) override; LinuxSchedulerImp() = default; - LinuxSchedulerImp(OsSysman *pOsSysman, zes_engine_type_flag_t type, std::vector &listOfEngines); + LinuxSchedulerImp(OsSysman *pOsSysman, zes_engine_type_flag_t type, + std::vector &listOfEngines, ze_bool_t isSubdevice, uint32_t subdeviceId); ~LinuxSchedulerImp() override = default; static const std::string engineDir; protected: SysfsAccess *pSysfsAccess = nullptr; + Device *pDevice = nullptr; zes_engine_type_flag_t engineType = ZES_ENGINE_TYPE_FLAG_OTHER; + ze_bool_t onSubdevice = 0; + uint32_t subdeviceId = 0; private: static const std::string preemptTimeoutMilliSecs; diff --git a/level_zero/tools/source/sysman/scheduler/os_scheduler.h b/level_zero/tools/source/sysman/scheduler/os_scheduler.h index d1fc005f64..d2007e6713 100644 --- a/level_zero/tools/source/sysman/scheduler/os_scheduler.h +++ b/level_zero/tools/source/sysman/scheduler/os_scheduler.h @@ -26,8 +26,10 @@ class OsScheduler { virtual ze_result_t setHeartbeatInterval(uint64_t heartbeat) = 0; virtual ze_bool_t canControlScheduler() = 0; virtual ze_result_t getProperties(zes_sched_properties_t &properties) = 0; - static OsScheduler *create(OsSysman *pOsSysman, zes_engine_type_flag_t engineType, std::vector &listOfEngines); - static ze_result_t getNumEngineTypeAndInstances(std::map> &listOfEngines, OsSysman *pOsSysman); + static OsScheduler *create(OsSysman *pOsSysman, zes_engine_type_flag_t engineType, std::vector &listOfEngines, + ze_bool_t isSubdevice, uint32_t subdeviceId); + static ze_result_t getNumEngineTypeAndInstances(std::map> &listOfEngines, + OsSysman *pOsSysman, ze_device_handle_t subdeviceHandle); virtual ~OsScheduler() = default; }; diff --git a/level_zero/tools/source/sysman/scheduler/scheduler.cpp b/level_zero/tools/source/sysman/scheduler/scheduler.cpp index 8028e1ce8e..f6fdac3981 100644 --- a/level_zero/tools/source/sysman/scheduler/scheduler.cpp +++ b/level_zero/tools/source/sysman/scheduler/scheduler.cpp @@ -23,16 +23,18 @@ SchedulerHandleContext::~SchedulerHandleContext() { } handleList.clear(); } -void SchedulerHandleContext::createHandle(zes_engine_type_flag_t engineType, std::vector &listOfEngines) { - Scheduler *pScheduler = new SchedulerImp(pOsSysman, engineType, listOfEngines); +void SchedulerHandleContext::createHandle(zes_engine_type_flag_t engineType, std::vector &listOfEngines, ze_device_handle_t deviceHandle) { + Scheduler *pScheduler = new SchedulerImp(pOsSysman, engineType, listOfEngines, deviceHandle); handleList.push_back(pScheduler); } -void SchedulerHandleContext::init() { - std::map> engineTypeInstance = {}; - OsScheduler::getNumEngineTypeAndInstances(engineTypeInstance, pOsSysman); - for (auto itr = engineTypeInstance.begin(); itr != engineTypeInstance.end(); ++itr) { - createHandle(itr->first, itr->second); +void SchedulerHandleContext::init(std::vector &deviceHandles) { + for (const auto &deviceHandle : deviceHandles) { + std::map> engineTypeInstance = {}; + OsScheduler::getNumEngineTypeAndInstances(engineTypeInstance, pOsSysman, deviceHandle); + for (auto itr = engineTypeInstance.begin(); itr != engineTypeInstance.end(); ++itr) { + createHandle(itr->first, itr->second, deviceHandle); + } } } diff --git a/level_zero/tools/source/sysman/scheduler/scheduler.h b/level_zero/tools/source/sysman/scheduler/scheduler.h index 237131b728..623e7f4bcb 100644 --- a/level_zero/tools/source/sysman/scheduler/scheduler.h +++ b/level_zero/tools/source/sysman/scheduler/scheduler.h @@ -8,6 +8,7 @@ #pragma once #include "shared/source/helpers/non_copyable_or_moveable.h" +#include "level_zero/core/source/device/device.h" #include #include @@ -41,7 +42,7 @@ class Scheduler : _zes_sched_handle_t { struct SchedulerHandleContext : NEO::NonCopyableOrMovableClass { SchedulerHandleContext(OsSysman *pOsSysman); ~SchedulerHandleContext(); - void init(); + void init(std::vector &deviceHandles); ze_result_t schedulerGet(uint32_t *pCount, zes_sched_handle_t *phScheduler); OsSysman *pOsSysman = nullptr; @@ -49,7 +50,7 @@ struct SchedulerHandleContext : NEO::NonCopyableOrMovableClass { ze_device_handle_t hCoreDevice = nullptr; private: - void createHandle(zes_engine_type_flag_t engineType, std::vector &listOfEngines); + void createHandle(zes_engine_type_flag_t engineType, std::vector &listOfEngines, ze_device_handle_t deviceHandle); }; } // namespace L0 diff --git a/level_zero/tools/source/sysman/scheduler/scheduler_imp.cpp b/level_zero/tools/source/sysman/scheduler/scheduler_imp.cpp index 5e64c99e69..eb622154a6 100644 --- a/level_zero/tools/source/sysman/scheduler/scheduler_imp.cpp +++ b/level_zero/tools/source/sysman/scheduler/scheduler_imp.cpp @@ -143,8 +143,11 @@ void SchedulerImp::init() { pOsScheduler->getProperties(this->properties); } -SchedulerImp::SchedulerImp(OsSysman *pOsSysman, zes_engine_type_flag_t engineType, std::vector &listOfEngines) { - pOsScheduler = OsScheduler::create(pOsSysman, engineType, listOfEngines); +SchedulerImp::SchedulerImp(OsSysman *pOsSysman, zes_engine_type_flag_t engineType, std::vector &listOfEngines, ze_device_handle_t deviceHandle) { + ze_device_properties_t deviceProperties = {}; + Device::fromHandle(deviceHandle)->getProperties(&deviceProperties); + pOsScheduler = OsScheduler::create(pOsSysman, engineType, listOfEngines, + deviceProperties.flags & ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE, deviceProperties.subdeviceId); UNRECOVERABLE_IF(nullptr == pOsScheduler); init(); }; diff --git a/level_zero/tools/source/sysman/scheduler/scheduler_imp.h b/level_zero/tools/source/sysman/scheduler/scheduler_imp.h index d7d53ee8f8..2fd9e8f6a6 100644 --- a/level_zero/tools/source/sysman/scheduler/scheduler_imp.h +++ b/level_zero/tools/source/sysman/scheduler/scheduler_imp.h @@ -29,7 +29,7 @@ class SchedulerImp : public Scheduler, NEO::NonCopyableOrMovableClass { SchedulerImp() = default; OsScheduler *pOsScheduler = nullptr; - SchedulerImp(OsSysman *pOsSysman, zes_engine_type_flag_t type, std::vector &listOfEngines); + SchedulerImp(OsSysman *pOsSysman, zes_engine_type_flag_t type, std::vector &listOfEngines, ze_device_handle_t deviceHandle); ~SchedulerImp() override; zes_engine_type_flag_t engineType; diff --git a/level_zero/tools/source/sysman/scheduler/windows/os_scheduler_imp.cpp b/level_zero/tools/source/sysman/scheduler/windows/os_scheduler_imp.cpp index a82090f804..9a89f681db 100644 --- a/level_zero/tools/source/sysman/scheduler/windows/os_scheduler_imp.cpp +++ b/level_zero/tools/source/sysman/scheduler/windows/os_scheduler_imp.cpp @@ -43,11 +43,12 @@ ze_result_t WddmSchedulerImp::getProperties(zes_sched_properties_t &properties) return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ze_result_t OsScheduler::getNumEngineTypeAndInstances(std::map> &listOfEngines, OsSysman *pOsSysman) { +ze_result_t OsScheduler::getNumEngineTypeAndInstances(std::map> &listOfEngines, OsSysman *pOsSysman, ze_device_handle_t subdeviceHandle) { return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; } -OsScheduler *OsScheduler::create(OsSysman *pOsSysman, zes_engine_type_flag_t type, std::vector &listOfEngines) { +OsScheduler *OsScheduler::create( + OsSysman *pOsSysman, zes_engine_type_flag_t type, std::vector &listOfEngines, ze_bool_t isSubdevice, uint32_t subdeviceId) { WddmSchedulerImp *pWddmSchedulerImp = new WddmSchedulerImp(); return static_cast(pWddmSchedulerImp); } diff --git a/level_zero/tools/source/sysman/sysman_imp.cpp b/level_zero/tools/source/sysman/sysman_imp.cpp index b9b7a71e2d..be9f9d8612 100644 --- a/level_zero/tools/source/sysman/sysman_imp.cpp +++ b/level_zero/tools/source/sysman/sysman_imp.cpp @@ -88,7 +88,7 @@ void SysmanDeviceImp::init() { pEngineHandleContext->init(); } if (pSchedulerHandleContext) { - pSchedulerHandleContext->init(); + pSchedulerHandleContext->init(deviceHandles); } if (pRasHandleContext) { pRasHandleContext->init(); diff --git a/level_zero/tools/test/black_box_tests/zello_sysman.cpp b/level_zero/tools/test/black_box_tests/zello_sysman.cpp index 5f63e21c8a..0879369732 100644 --- a/level_zero/tools/test/black_box_tests/zello_sysman.cpp +++ b/level_zero/tools/test/black_box_tests/zello_sysman.cpp @@ -366,6 +366,9 @@ void testSysmanScheduler(ze_device_handle_t &device) { std::cout << "Could not retrieve scheduler domains" << std::endl; return; } + if (verbose) { + std::cout << "Number of scheduler domains = " << count << std::endl; + } std::vector handles(count, nullptr); VALIDATECALL(zesDeviceEnumSchedulers(device, &count, handles.data())); @@ -375,6 +378,19 @@ void testSysmanScheduler(ze_device_handle_t &device) { if (verbose) { std::cout << "Current Mode = " << getSchedulerModeName(currentMode) << std::endl; } + + zes_sched_properties_t schedProperties = {}; + VALIDATECALL(zesSchedulerGetProperties(handle, &schedProperties)); + if (verbose) { + std::cout << "Scheduler properties: onSubdevice = " << schedProperties.onSubdevice << std::endl; + if (schedProperties.onSubdevice) { + std::cout << "Scheduler properties: subdeviceId = " << schedProperties.subdeviceId << std::endl; + } + std::cout << "Scheduler properties: canControl = " << schedProperties.canControl << std::endl; + std::cout << "Scheduler properties: engines = " << schedProperties.engines << std::endl; + std::cout << "Scheduler properties: supportedModes = " << schedProperties.supportedModes << std::endl; + } + zes_sched_timeout_properties_t timeoutProperties = {}; zes_sched_timeslice_properties_t timesliceProperties = {}; diff --git a/level_zero/tools/test/unit_tests/sources/sysman/frequency/linux/test_zes_frequency.cpp b/level_zero/tools/test/unit_tests/sources/sysman/frequency/linux/test_zes_frequency.cpp index 3cf5e3ec68..956147cc81 100644 --- a/level_zero/tools/test/unit_tests/sources/sysman/frequency/linux/test_zes_frequency.cpp +++ b/level_zero/tools/test/unit_tests/sources/sysman/frequency/linux/test_zes_frequency.cpp @@ -537,5 +537,17 @@ TEST_F(SysmanDeviceFrequencyFixture, GivenValidFrequencyHandleWhenCallingzesFreq } } +TEST_F(SysmanMultiDeviceFixture, GivenValidDevicePointerWhenGettingFrequencyPropertiesThenValidSchedPropertiesRetrieved) { + zes_freq_properties_t properties = {}; + ze_device_properties_t deviceProperties = {}; + Device::fromHandle(device)->getProperties(&deviceProperties); + LinuxFrequencyImp *pLinuxFrequencyImp = new LinuxFrequencyImp(pOsSysman, deviceProperties.flags & ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE, + deviceProperties.subdeviceId, ZES_FREQ_DOMAIN_GPU); + EXPECT_EQ(ZE_RESULT_SUCCESS, pLinuxFrequencyImp->osFrequencyGetProperties(properties)); + EXPECT_EQ(properties.subdeviceId, deviceProperties.subdeviceId); + EXPECT_EQ(properties.onSubdevice, deviceProperties.flags & ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE); + delete pLinuxFrequencyImp; +} + } // namespace ult } // namespace L0 diff --git a/level_zero/tools/test/unit_tests/sources/sysman/scheduler/linux/CMakeLists.txt b/level_zero/tools/test/unit_tests/sources/sysman/scheduler/linux/CMakeLists.txt index 7da661b94a..c0f51d1744 100644 --- a/level_zero/tools/test/unit_tests/sources/sysman/scheduler/linux/CMakeLists.txt +++ b/level_zero/tools/test/unit_tests/sources/sysman/scheduler/linux/CMakeLists.txt @@ -8,7 +8,7 @@ if(UNIX) target_sources(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt - ${CMAKE_CURRENT_SOURCE_DIR}/test_zes_scheduler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/mock_sysfs_scheduler.h + ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/test_zes_scheduler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/mock_sysfs_scheduler.h ) endif() diff --git a/level_zero/tools/test/unit_tests/sources/sysman/scheduler/linux/test_zes_scheduler.cpp b/level_zero/tools/test/unit_tests/sources/sysman/scheduler/linux/test_zes_scheduler.cpp index 40bd32dfee..6141be6ad7 100644 --- a/level_zero/tools/test/unit_tests/sources/sysman/scheduler/linux/test_zes_scheduler.cpp +++ b/level_zero/tools/test/unit_tests/sources/sysman/scheduler/linux/test_zes_scheduler.cpp @@ -34,6 +34,7 @@ class SysmanDeviceSchedulerFixture : public SysmanDeviceFixture { protected: std::unique_ptr> pSysfsAccess; SysfsAccess *pSysfsAccessOld = nullptr; + std::vector deviceHandles; void SetUp() override { SysmanDeviceFixture::SetUp(); @@ -56,7 +57,22 @@ class SysmanDeviceSchedulerFixture : public SysmanDeviceFixture { ON_CALL(*pSysfsAccess.get(), scanDirEntries(_, _)) .WillByDefault(::testing::Invoke(pSysfsAccess.get(), &Mock::getscanDirEntries)); - pSysmanDeviceImp->pSchedulerHandleContext->init(); + // delete handles created in initial SysmanDeviceHandleContext::init() call + for (auto handle : pSysmanDeviceImp->pSchedulerHandleContext->handleList) { + delete handle; + } + pSysmanDeviceImp->pSchedulerHandleContext->handleList.clear(); + uint32_t subDeviceCount = 0; + + // We received a device handle. Check for subdevices in this device + Device::fromHandle(device->toHandle())->getSubDevices(&subDeviceCount, nullptr); + if (subDeviceCount == 0) { + deviceHandles.resize(1, device->toHandle()); + } else { + deviceHandles.resize(subDeviceCount, nullptr); + Device::fromHandle(device->toHandle())->getSubDevices(&subDeviceCount, deviceHandles.data()); + } + pSysmanDeviceImp->pSchedulerHandleContext->init(deviceHandles); } void TearDown() override { @@ -98,7 +114,7 @@ TEST_F(SysmanDeviceSchedulerFixture, GivenComponentCountZeroWhenCallingzesDevice .WillByDefault(::testing::Invoke(pSysfsAccess.get(), &Mock::getscanDirEntriesStatusReturnError)); auto pSchedulerHandleContextTest = std::make_unique(pOsSysman); - pSchedulerHandleContextTest->init(); + pSchedulerHandleContextTest->init(deviceHandles); EXPECT_EQ(0u, static_cast(pSchedulerHandleContextTest->handleList.size())); } @@ -431,5 +447,18 @@ TEST_F(SysmanDeviceSchedulerFixture, GivenValidDeviceHandleWhenCallingzesSchedul } } +TEST_F(SysmanMultiDeviceFixture, GivenValidDevicePointerWhenGettingSchedPropertiesThenValidSchedPropertiesRetrieved) { + zes_sched_properties_t properties = {}; + std::vector listOfEngines; + ze_device_properties_t deviceProperties = {}; + Device::fromHandle(device)->getProperties(&deviceProperties); + LinuxSchedulerImp *pLinuxSchedulerImp = new LinuxSchedulerImp(pOsSysman, ZES_ENGINE_TYPE_FLAG_COMPUTE, listOfEngines, + deviceProperties.flags & ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE, deviceProperties.subdeviceId); + EXPECT_EQ(ZE_RESULT_SUCCESS, pLinuxSchedulerImp->getProperties(properties)); + EXPECT_EQ(properties.subdeviceId, deviceProperties.subdeviceId); + EXPECT_EQ(properties.onSubdevice, deviceProperties.flags & ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE); + delete pLinuxSchedulerImp; +} + } // namespace ult } // namespace L0 \ No newline at end of file