From c6dc6bf1b907249d1882dce334890dbb2b7734a6 Mon Sep 17 00:00:00 2001 From: Robert Krzemien Date: Fri, 9 Jul 2021 14:08:58 +0000 Subject: [PATCH] Metrics Api: Initialize metrics devices for all sub devices Signed-off-by: Robert Krzemien --- .../source/metrics/metric_enumeration_imp.cpp | 84 +++++++++-- .../sources/metrics/mock_metric.cpp | 88 ++++++++++++ .../unit_tests/sources/metrics/mock_metric.h | 26 ++++ .../metrics/test_metric_enumeration.cpp | 136 +++++++++++++++++- 4 files changed, 321 insertions(+), 13 deletions(-) diff --git a/level_zero/tools/source/metrics/metric_enumeration_imp.cpp b/level_zero/tools/source/metrics/metric_enumeration_imp.cpp index 35fa834abf..d2582a38da 100644 --- a/level_zero/tools/source/metrics/metric_enumeration_imp.cpp +++ b/level_zero/tools/source/metrics/metric_enumeration_imp.cpp @@ -11,6 +11,8 @@ #include "shared/source/helpers/string.h" #include "shared/source/os_interface/os_library.h" +#include "level_zero/core/source/device/device_imp.h" + #include namespace L0 { @@ -120,16 +122,38 @@ ze_result_t MetricEnumeration::openMetricsDiscovery() { return ZE_RESULT_ERROR_NOT_AVAILABLE; } - if (subDeviceIndex == 0) { - pAdapter->OpenMetricsDevice(&pMetricsDevice); - } else { - pAdapter->OpenMetricsSubDevice(subDeviceIndex, &pMetricsDevice); - } + auto &device = metricContext.getDevice(); + if (device.isMultiDeviceCapable()) { - if (pMetricsDevice == nullptr) { - NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, "unable to open metrics device %u\n", subDeviceIndex); - cleanupMetricsDiscovery(); - return ZE_RESULT_ERROR_NOT_AVAILABLE; + // Open metrics device for each sub device. + const auto &deviceImp = *static_cast(&device); + + for (uint32_t i = 0; i < deviceImp.subDevices.size(); i++) { + + auto &metricsDevice = deviceImp.subDevices[i]->getMetricContext().getMetricEnumeration().pMetricsDevice; + + pAdapter->OpenMetricsSubDevice(i, &metricsDevice); + + if (metricsDevice == nullptr) { + NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, "unable to open metrics device %u\n", i); + cleanupMetricsDiscovery(); + return ZE_RESULT_ERROR_NOT_AVAILABLE; + } + } + } else { + if (subDeviceIndex == 0) { + // Open metrics device for root device or sub device with index 0. + pAdapter->OpenMetricsDevice(&pMetricsDevice); + } else { + // Open metrics device for a given sub device index. + pAdapter->OpenMetricsSubDevice(subDeviceIndex, &pMetricsDevice); + } + + if (pMetricsDevice == nullptr) { + NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, "unable to open metrics device %u\n", subDeviceIndex); + cleanupMetricsDiscovery(); + return ZE_RESULT_ERROR_NOT_AVAILABLE; + } } return ZE_RESULT_SUCCESS; @@ -142,9 +166,29 @@ ze_result_t MetricEnumeration::cleanupMetricsDiscovery() { metricGroups.clear(); - if (pAdapter && pMetricsDevice) { - pAdapter->CloseMetricsDevice(pMetricsDevice); - pMetricsDevice = nullptr; + if (pAdapter) { + + auto &device = metricContext.getDevice(); + if (device.isMultiDeviceCapable()) { + + // Close metrics device for each sub device. + const auto &deviceImp = *static_cast(&device); + + for (uint32_t i = 0; i < deviceImp.subDevices.size(); i++) { + + auto &metricsDevice = deviceImp.subDevices[i]->getMetricContext().getMetricEnumeration().pMetricsDevice; + + if (metricsDevice) { + pAdapter->CloseMetricsDevice(metricsDevice); + metricsDevice = nullptr; + } + } + } else if (pMetricsDevice) { + + // Close metrics device for one sub device or root device. + pAdapter->CloseMetricsDevice(pMetricsDevice); + pMetricsDevice = nullptr; + } } if (hMetricsDiscovery != nullptr) { @@ -156,6 +200,22 @@ ze_result_t MetricEnumeration::cleanupMetricsDiscovery() { } ze_result_t MetricEnumeration::cacheMetricInformation() { + + MetricsDiscovery::IMetricsDevice_1_5 *pMetricsDevice = nullptr; + + auto &device = metricContext.getDevice(); + + if (device.isMultiDeviceCapable()) { + + // Get metric information from sub device 0. + const auto &deviceImp = *static_cast(&device); + pMetricsDevice = deviceImp.subDevices[0]->getMetricContext().getMetricEnumeration().pMetricsDevice; + } else { + + // Get metric information from root or sub device. + pMetricsDevice = this->pMetricsDevice; + } + DEBUG_BREAK_IF(pMetricsDevice == nullptr); MetricsDiscovery::TMetricsDeviceParams_1_2 *pMetricsDeviceParams = pMetricsDevice->GetParams(); diff --git a/level_zero/tools/test/unit_tests/sources/metrics/mock_metric.cpp b/level_zero/tools/test/unit_tests/sources/metrics/mock_metric.cpp index b0d01cb5af..36770e4143 100644 --- a/level_zero/tools/test/unit_tests/sources/metrics/mock_metric.cpp +++ b/level_zero/tools/test/unit_tests/sources/metrics/mock_metric.cpp @@ -7,6 +7,7 @@ #include "level_zero/tools/test/unit_tests/sources/metrics/mock_metric.h" +#include "shared/source/command_container/implicit_scaling.h" #include "shared/test/common/mocks/mock_os_library.h" using namespace MetricsLibraryApi; @@ -99,6 +100,93 @@ void MetricContextFixture::openMetricsAdapterGroup() { .WillOnce(Return(TCompletionCode::CC_OK)); } +void MetricMultiDeviceFixture::SetUp() { + + NEO::ImplicitScaling::apiSupport = true; + + MultiDeviceFixture::SetUp(); + + devices.resize(driverHandle->devices.size()); + + for (uint32_t i = 0; i < driverHandle->devices.size(); i++) { + devices[i] = driverHandle->devices[i]; + } + + auto &metricContext = devices[0]->getMetricContext(); + metricContext.setInitializationState(ZE_RESULT_SUCCESS); + + mockMetricsLibrary = std::unique_ptr>(new (std::nothrow) Mock(metricContext)); + mockMetricsLibrary->setMockedApi(&mockMetricsLibraryApi); + mockMetricsLibrary->handle = new MockOsLibrary(); + + mockMetricEnumeration = std::unique_ptr>(new (std::nothrow) Mock(metricContext)); + mockMetricEnumeration->setMockedApi(&mockMetricsDiscoveryApi); + mockMetricEnumeration->hMetricsDiscovery = std::make_unique(); + + // Metrics Discovery device common settings. + metricsDeviceParams.Version.MajorNumber = MetricEnumeration::requiredMetricsDiscoveryMajorVersion; + metricsDeviceParams.Version.MinorNumber = MetricEnumeration::requiredMetricsDiscoveryMinorVersion; +} + +void MetricMultiDeviceFixture::TearDown() { + + // Restore original metrics library + delete mockMetricsLibrary->handle; + mockMetricsLibrary->setMockedApi(nullptr); + mockMetricsLibrary.reset(); + + // Restore original metric enumeration. + mockMetricEnumeration->setMockedApi(nullptr); + mockMetricEnumeration.reset(); + + MultiDeviceFixture::TearDown(); + + NEO::ImplicitScaling::apiSupport = false; +} + +void MetricMultiDeviceFixture::openMetricsAdapter() { + + EXPECT_CALL(*mockMetricEnumeration, loadMetricsDiscovery()) + .Times(0); + + EXPECT_CALL(*mockMetricEnumeration->g_mockApi, MockOpenAdapterGroup(_)) + .Times(1) + .WillOnce(DoAll(::testing::SetArgPointee<0>(&adapterGroup), Return(TCompletionCode::CC_OK))); + + EXPECT_CALL(adapter, OpenMetricsDevice(_)) + .Times(1) + .WillOnce(DoAll(::testing::SetArgPointee<0>(&metricsDevice), Return(TCompletionCode::CC_OK))); + + EXPECT_CALL(adapter, CloseMetricsDevice(_)) + .Times(1) + .WillOnce(Return(TCompletionCode::CC_OK)); + + EXPECT_CALL(adapterGroup, GetAdapter(_)) + .Times(0); + + EXPECT_CALL(*mockMetricEnumeration, getMetricsAdapter()) + .Times(1) + .WillOnce(Return(&adapter)); +} + +void MetricMultiDeviceFixture::openMetricsAdapterGroup() { + + EXPECT_CALL(*mockMetricEnumeration, loadMetricsDiscovery()) + .Times(0); + + EXPECT_CALL(*mockMetricEnumeration->g_mockApi, MockOpenAdapterGroup(_)) + .Times(1) + .WillOnce(DoAll(::testing::SetArgPointee<0>(&adapterGroup), Return(TCompletionCode::CC_OK))); + + EXPECT_CALL(adapter, OpenMetricsDevice(_)) + .Times(1) + .WillOnce(DoAll(::testing::SetArgPointee<0>(&metricsDevice), Return(TCompletionCode::CC_OK))); + + EXPECT_CALL(adapter, CloseMetricsDevice(_)) + .Times(1) + .WillOnce(Return(TCompletionCode::CC_OK)); +} + Mock::Mock(::L0::MetricContext &metricContext) : MetricsLibrary(metricContext) { } diff --git a/level_zero/tools/test/unit_tests/sources/metrics/mock_metric.h b/level_zero/tools/test/unit_tests/sources/metrics/mock_metric.h index 4dd3da95ad..5344bedc62 100644 --- a/level_zero/tools/test/unit_tests/sources/metrics/mock_metric.h +++ b/level_zero/tools/test/unit_tests/sources/metrics/mock_metric.h @@ -166,5 +166,31 @@ class MetricContextFixture : public ContextFixture { MetricsDiscovery::TMetricsDeviceParams_1_2 metricsDeviceParams = {}; }; +class MetricMultiDeviceFixture : public MultiDeviceFixture { + + protected: + void SetUp() override; + void TearDown() override; + void openMetricsAdapter(); + void openMetricsAdapterGroup(); + + public: + std::vector devices; + + // Mocked objects. + std::unique_ptr> mockMetricEnumeration = nullptr; + std::unique_ptr> mockMetricsLibrary = nullptr; + + // Mocked metrics library/discovery APIs. + MockMetricsLibraryApi mockMetricsLibraryApi = {}; + MockMetricsDiscoveryApi mockMetricsDiscoveryApi = {}; + + // Metrics discovery device + Mock adapterGroup; + Mock adapter; + Mock metricsDevice; + MetricsDiscovery::TMetricsDeviceParams_1_2 metricsDeviceParams = {}; +}; + } // namespace ult } // namespace L0 diff --git a/level_zero/tools/test/unit_tests/sources/metrics/test_metric_enumeration.cpp b/level_zero/tools/test/unit_tests/sources/metrics/test_metric_enumeration.cpp index 3c61393471..067f513f01 100644 --- a/level_zero/tools/test/unit_tests/sources/metrics/test_metric_enumeration.cpp +++ b/level_zero/tools/test/unit_tests/sources/metrics/test_metric_enumeration.cpp @@ -2073,7 +2073,7 @@ TEST_F(MetricEnumerationTest, givenSubDeviceWhenLoadDependenciesIsCalledThenOpen .Times(1) .WillOnce(Return(&metricsDeviceParams)); - // Use first sub device. + // Use second sub device. metricContext.setSubDeviceIndex(1); mockMetricsLibrary->initializationState = ZE_RESULT_SUCCESS; @@ -2083,6 +2083,48 @@ TEST_F(MetricEnumerationTest, givenSubDeviceWhenLoadDependenciesIsCalledThenOpen EXPECT_EQ(mockMetricEnumeration->cleanupMetricsDiscovery(), ZE_RESULT_SUCCESS); } +TEST_F(MetricEnumerationTest, givenSubDeviceWhenLoadDependenciesIsCalledThenOpenMetricsSubDeviceWillBeCalledWithoutSuccess) { + + auto &metricContext = device->getMetricContext(); + Mock mockAdapterGroup; + Mock mockAdapter; + Mock mockDevice; + + EXPECT_CALL(*mockMetricsLibrary, load()) + .Times(1) + .WillOnce(Return(true)); + + EXPECT_CALL(*mockMetricEnumeration, loadMetricsDiscovery()) + .Times(1) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + + EXPECT_CALL(*Mock::g_mockApi, MockOpenAdapterGroup(_)) + .Times(1) + .WillOnce(DoAll(::testing::SetArgPointee<0>(&mockAdapterGroup), Return(TCompletionCode::CC_OK))); + + EXPECT_CALL(*mockMetricEnumeration, getMetricsAdapter()) + .Times(1) + .WillOnce(Return(&mockAdapter)); + + EXPECT_CALL(mockAdapter, OpenMetricsSubDevice(_, _)) + .Times(1) + .WillOnce(Return(TCompletionCode::CC_OK)); + + EXPECT_CALL(mockAdapter, CloseMetricsDevice(_)) + .Times(0); + + EXPECT_CALL(mockDevice, GetParams()) + .Times(0); + + // Use second sub device. + metricContext.setSubDeviceIndex(1); + mockMetricsLibrary->initializationState = ZE_RESULT_SUCCESS; + + EXPECT_EQ(metricContext.loadDependencies(), true); + EXPECT_EQ(metricContext.isInitialized(), true); + EXPECT_EQ(mockMetricEnumeration->baseIsInitialized(), false); +} + TEST_F(MetricEnumerationTest, givenMetricContextWhenEnablingOrDisablingCollectionThenExpectProperFlagSet) { auto &metricContext = device->getMetricContext(); @@ -2096,6 +2138,98 @@ TEST_F(MetricEnumerationTest, givenMetricContextWhenEnablingOrDisablingCollectio EXPECT_EQ(metricGroupCount, 0u); } +using MetricEnumerationMultiDeviceTest = Test; + +TEST_F(MetricEnumerationMultiDeviceTest, givenRootDeviceWhenLoadDependenciesIsCalledThenOpenMetricsSubDeviceWillBeCalled) { + + // Use first root device. + auto &metricContext = devices[0]->getMetricContext(); + const uint32_t subDeviceCount = static_cast(devices.size()); + Mock mockAdapterGroup; + Mock mockAdapter; + Mock mockDevice; + + EXPECT_CALL(*mockMetricsLibrary, load()) + .Times(1) + .WillOnce(Return(true)); + + EXPECT_CALL(*mockMetricEnumeration, loadMetricsDiscovery()) + .Times(1) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + + EXPECT_CALL(*Mock::g_mockApi, MockOpenAdapterGroup(_)) + .Times(1) + .WillOnce(DoAll(::testing::SetArgPointee<0>(&mockAdapterGroup), Return(TCompletionCode::CC_OK))); + + EXPECT_CALL(*mockMetricEnumeration, getMetricsAdapter()) + .Times(1) + .WillOnce(Return(&mockAdapter)); + + EXPECT_CALL(mockAdapter, OpenMetricsSubDevice(_, _)) + .Times(subDeviceCount) + .WillRepeatedly(DoAll(::testing::SetArgPointee<1>(&mockDevice), Return(TCompletionCode::CC_OK))); + + EXPECT_CALL(mockAdapter, CloseMetricsDevice(_)) + .Times(subDeviceCount) + .WillRepeatedly(Return(TCompletionCode::CC_OK)); + + EXPECT_CALL(mockDevice, GetParams()) + .Times(1) + .WillOnce(Return(&metricsDeviceParams)); + + // Use root device. + metricContext.setSubDeviceIndex(0); + mockMetricsLibrary->initializationState = ZE_RESULT_SUCCESS; + + EXPECT_EQ(metricContext.loadDependencies(), true); + EXPECT_EQ(metricContext.isInitialized(), true); + EXPECT_EQ(mockMetricEnumeration->baseIsInitialized(), true); + EXPECT_EQ(mockMetricEnumeration->cleanupMetricsDiscovery(), ZE_RESULT_SUCCESS); +} + +TEST_F(MetricEnumerationMultiDeviceTest, givenRootDeviceWhenLoadDependenciesIsCalledThenOpenMetricsSubDeviceWillBeCalledWithoutSuccess) { + + // Use first root device. + auto &metricContext = devices[0]->getMetricContext(); + Mock mockAdapterGroup; + Mock mockAdapter; + Mock mockDevice; + + EXPECT_CALL(*mockMetricsLibrary, load()) + .Times(1) + .WillOnce(Return(true)); + + EXPECT_CALL(*mockMetricEnumeration, loadMetricsDiscovery()) + .Times(1) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + + EXPECT_CALL(*Mock::g_mockApi, MockOpenAdapterGroup(_)) + .Times(1) + .WillOnce(DoAll(::testing::SetArgPointee<0>(&mockAdapterGroup), Return(TCompletionCode::CC_OK))); + + EXPECT_CALL(*mockMetricEnumeration, getMetricsAdapter()) + .Times(1) + .WillOnce(Return(&mockAdapter)); + + EXPECT_CALL(mockAdapter, OpenMetricsSubDevice(_, _)) + .Times(1) + .WillOnce(Return(TCompletionCode::CC_ERROR_GENERAL)); + + EXPECT_CALL(mockAdapter, CloseMetricsDevice(_)) + .Times(0); + + EXPECT_CALL(mockDevice, GetParams()) + .Times(0); + + // Use root device. + metricContext.setSubDeviceIndex(0); + mockMetricsLibrary->initializationState = ZE_RESULT_SUCCESS; + + EXPECT_EQ(metricContext.loadDependencies(), true); + EXPECT_EQ(metricContext.isInitialized(), true); + EXPECT_EQ(mockMetricEnumeration->baseIsInitialized(), false); +} + class MetricEnumerationTestMetricTypes : public MetricEnumerationTest, public ::testing::WithParamInterface { public: