diff --git a/opencl/test/unit_test/mt_tests/direct_submission/direct_submission_controller_tests_mt.cpp b/opencl/test/unit_test/mt_tests/direct_submission/direct_submission_controller_tests_mt.cpp index 08eefdcf43..449ca42599 100644 --- a/opencl/test/unit_test/mt_tests/direct_submission/direct_submission_controller_tests_mt.cpp +++ b/opencl/test/unit_test/mt_tests/direct_submission/direct_submission_controller_tests_mt.cpp @@ -34,7 +34,7 @@ TEST(DirectSubmissionControllerTestsMt, givenDirectSubmissionControllerWhenTimeo DirectSubmissionControllerMock controller; executionEnvironment.directSubmissionController.reset(&controller); - controller.timeoutElapsedReturnValue.store(true); + controller.timeoutElapsedReturnValue.store(TimeoutElapsedMode::fullyElapsed); controller.startThread(); csr.startControllingDirectSubmissions(); controller.registerDirectSubmission(&csr); diff --git a/shared/source/debug_settings/debug_variables_base.inl b/shared/source/debug_settings/debug_variables_base.inl index d40bf84382..b542913964 100644 --- a/shared/source/debug_settings/debug_variables_base.inl +++ b/shared/source/debug_settings/debug_variables_base.inl @@ -428,6 +428,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionDetectGpuHang, -1, "-1: default, DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionFlatRingBuffer, -1, "-1: default, 0: disable, 1: enable, Copies task command buffer directly into ring, implemented for immediate command lists only") DECLARE_DEBUG_VARIABLE(int32_t, EnableDirectSubmissionController, -1, "Enable direct submission terminating after given timeout, -1: default, 0: disabled, 1: enabled") DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionControllerTimeout, -1, "Set direct submission controller timeout, -1: default 5000 us, >=0: timeout in us") +DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionControllerBcsTimeoutDivisor, -1, "If >=1, divide controller timeout to stop BCS only engines faster than others") DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionControllerMaxTimeout, -1, "Set direct submission controller max timeout - timeout will increase up to given value, -1: default 5000 us, >=0: max timeout in us") DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionControllerDivisor, -1, "Set direct submission controller timeout divider, -1: default 1, >0: divider value") DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionControllerAdjustOnThrottleAndAcLineStatus, -1, "Adjust controller timeout settings based on queue throttle and ac line status, -1: default, 0: disabled, 1: enabled") diff --git a/shared/source/direct_submission/direct_submission_controller.cpp b/shared/source/direct_submission/direct_submission_controller.cpp index 22546e59c7..cb2ab6e122 100644 --- a/shared/source/direct_submission/direct_submission_controller.cpp +++ b/shared/source/direct_submission/direct_submission_controller.cpp @@ -28,6 +28,10 @@ DirectSubmissionController::DirectSubmissionController() { if (debugManager.flags.DirectSubmissionControllerDivisor.get() != -1) { timeoutDivisor = debugManager.flags.DirectSubmissionControllerDivisor.get(); } + if (debugManager.flags.DirectSubmissionControllerBcsTimeoutDivisor.get() != -1) { + bcsTimeoutDivisor = debugManager.flags.DirectSubmissionControllerBcsTimeoutDivisor.get(); + } + if (debugManager.flags.DirectSubmissionControllerMaxTimeout.get() != -1) { maxTimeout = std::chrono::microseconds{debugManager.flags.DirectSubmissionControllerMaxTimeout.get()}; } @@ -138,15 +142,21 @@ void *DirectSubmissionController::controlDirectSubmissionsState(void *self) { } void DirectSubmissionController::checkNewSubmissions() { - if (!timeoutElapsed()) { + auto timeoutMode = timeoutElapsed(); + if (timeoutMode == TimeoutElapsedMode::notElapsed) { return; } + std::lock_guard lock(this->directSubmissionsMutex); bool shouldRecalculateTimeout = false; for (auto &directSubmission : this->directSubmissions) { auto csr = directSubmission.first; auto &state = directSubmission.second; + if (timeoutMode == TimeoutElapsedMode::bcsOnly && !EngineHelpers::isBcs(csr->getOsContext().getEngineType())) { + continue; + } + auto taskCount = csr->peekTaskCount(); if (taskCount == state.taskCount) { if (state.isStopped) { @@ -172,11 +182,14 @@ void DirectSubmissionController::checkNewSubmissions() { if (shouldRecalculateTimeout) { this->recalculateTimeout(); } - this->timeSinceLastCheck = getCpuTimestamp(); + + if (timeoutMode != TimeoutElapsedMode::bcsOnly) { + this->timeSinceLastCheck = getCpuTimestamp(); + } } bool DirectSubmissionController::sleep(std::unique_lock &lock) { - return NEO::waitOnConditionWithPredicate(condVar, lock, std::chrono::microseconds(this->timeout), [&] { return !pagingFenceRequests.empty(); }); + return NEO::waitOnConditionWithPredicate(condVar, lock, getSleepValue(), [&] { return !pagingFenceRequests.empty(); }); } bool DirectSubmissionController::isDirectSubmissionIdle(CommandStreamReceiver *csr, std::unique_lock &csrLock) { @@ -266,9 +279,14 @@ void DirectSubmissionController::handlePagingFenceRequests(std::unique_lock(getCpuTimestamp() - this->timeSinceLastCheck); - return diff >= this->timeout; -} + if (diff >= this->timeout) { + return TimeoutElapsedMode::fullyElapsed; + } else if (this->bcsTimeoutDivisor > 1 && diff >= (this->timeout / this->bcsTimeoutDivisor)) { + return TimeoutElapsedMode::bcsOnly; + } + return TimeoutElapsedMode::notElapsed; +} } // namespace NEO diff --git a/shared/source/direct_submission/direct_submission_controller.h b/shared/source/direct_submission/direct_submission_controller.h index 83805baa42..15be336b4b 100644 --- a/shared/source/direct_submission/direct_submission_controller.h +++ b/shared/source/direct_submission/direct_submission_controller.h @@ -31,7 +31,7 @@ using SteadyClock = std::chrono::steady_clock; struct TimeoutParams { std::chrono::microseconds maxTimeout; std::chrono::microseconds timeout; - int timeoutDivisor; + int32_t timeoutDivisor; bool directSubmissionEnabled; }; @@ -40,6 +40,12 @@ struct WaitForPagingFenceRequest { uint64_t pagingFenceValue; }; +enum class TimeoutElapsedMode { + notElapsed, + bcsOnly, + fullyElapsed +}; + class DirectSubmissionController { public: static constexpr size_t defaultTimeout = 5'000; @@ -98,7 +104,8 @@ class DirectSubmissionController { size_t getTimeoutParamsMapKey(QueueThrottle throttle, bool acLineStatus); void handlePagingFenceRequests(std::unique_lock &lock, bool checkForNewSubmissions); - MOCKABLE_VIRTUAL bool timeoutElapsed(); + MOCKABLE_VIRTUAL TimeoutElapsedMode timeoutElapsed(); + std::chrono::microseconds getSleepValue() const { return std::chrono::microseconds(this->timeout / this->bcsTimeoutDivisor); } uint32_t maxCcsCount = 1u; std::array ccsCount = {}; @@ -113,7 +120,8 @@ class DirectSubmissionController { SteadyClock::time_point lastTerminateCpuTimestamp{}; std::chrono::microseconds maxTimeout{defaultTimeout}; std::chrono::microseconds timeout{defaultTimeout}; - int timeoutDivisor = 1; + int32_t timeoutDivisor = 1; + int32_t bcsTimeoutDivisor = 1; std::unordered_map timeoutParamsMap; QueueThrottle lowestThrottleSubmitted = QueueThrottle::HIGH; bool adjustTimeoutOnThrottleAndAcLineStatus = false; diff --git a/shared/test/common/test_files/igdrcl.config b/shared/test/common/test_files/igdrcl.config index 8b820d9481..ab0eba7007 100644 --- a/shared/test/common/test_files/igdrcl.config +++ b/shared/test/common/test_files/igdrcl.config @@ -636,4 +636,5 @@ ForceWddmHugeChunkSizeMB = -1 DirectSubmissionControllerIdleDetection = -1 DebugUmdInterruptTimeout = -1 DebugUmdMaxReadWriteRetry = -1 +DirectSubmissionControllerBcsTimeoutDivisor = -1 # Please don't edit below this line diff --git a/shared/test/unit_test/direct_submission/direct_submission_controller_mock.h b/shared/test/unit_test/direct_submission/direct_submission_controller_mock.h index 3dd96c6ac8..5409b1484f 100644 --- a/shared/test/unit_test/direct_submission/direct_submission_controller_mock.h +++ b/shared/test/unit_test/direct_submission/direct_submission_controller_mock.h @@ -12,11 +12,13 @@ namespace NEO { struct DirectSubmissionControllerMock : public DirectSubmissionController { using DirectSubmissionController::adjustTimeoutOnThrottleAndAcLineStatus; + using DirectSubmissionController::bcsTimeoutDivisor; using DirectSubmissionController::checkNewSubmissions; using DirectSubmissionController::condVarMutex; using DirectSubmissionController::directSubmissionControllingThread; using DirectSubmissionController::directSubmissions; using DirectSubmissionController::directSubmissionsMutex; + using DirectSubmissionController::getSleepValue; using DirectSubmissionController::getTimeoutParamsMapKey; using DirectSubmissionController::handlePagingFenceRequests; using DirectSubmissionController::keepControlling; @@ -40,7 +42,7 @@ struct DirectSubmissionControllerMock : public DirectSubmissionController { return cpuTimestamp; } - bool timeoutElapsed() override { + TimeoutElapsedMode timeoutElapsed() override { if (timeoutElapsedCallBase) { return DirectSubmissionController::timeoutElapsed(); } @@ -51,7 +53,7 @@ struct DirectSubmissionControllerMock : public DirectSubmissionController { SteadyClock::time_point cpuTimestamp{}; std::atomic sleepCalled{false}; std::atomic sleepReturnValue{false}; - std::atomic timeoutElapsedReturnValue{false}; + std::atomic timeoutElapsedReturnValue{TimeoutElapsedMode::notElapsed}; std::atomic timeoutElapsedCallBase{false}; }; } // namespace NEO \ No newline at end of file diff --git a/shared/test/unit_test/direct_submission/direct_submission_controller_tests.cpp b/shared/test/unit_test/direct_submission/direct_submission_controller_tests.cpp index b69427c073..c8a78c0adf 100644 --- a/shared/test/unit_test/direct_submission/direct_submission_controller_tests.cpp +++ b/shared/test/unit_test/direct_submission/direct_submission_controller_tests.cpp @@ -51,7 +51,7 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWhenRegiste csr.taskCount.store(5u); DirectSubmissionControllerMock controller; - controller.timeoutElapsedReturnValue.store(true); + controller.timeoutElapsedReturnValue.store(TimeoutElapsedMode::fullyElapsed); controller.registerDirectSubmission(&csr); controller.checkNewSubmissions(); EXPECT_FALSE(controller.directSubmissions[&csr].isStopped); @@ -78,6 +78,94 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWhenRegiste controller.unregisterDirectSubmission(&csr); } +TEST(DirectSubmissionControllerTests, givenDebugFlagSetWhenCheckingIfTimeoutElapsedThenReturnCorrectValue) { + DebugManagerStateRestore restorer; + DirectSubmissionControllerMock defaultController; + defaultController.timeoutElapsedCallBase.store(true); + EXPECT_EQ(1, defaultController.bcsTimeoutDivisor); + EXPECT_EQ(std::chrono::microseconds{defaultController.timeout}, defaultController.getSleepValue()); + + debugManager.flags.DirectSubmissionControllerBcsTimeoutDivisor.set(2); + + DirectSubmissionControllerMock controller; + controller.timeoutElapsedCallBase.store(true); + EXPECT_EQ(2, controller.bcsTimeoutDivisor); + EXPECT_EQ(std::chrono::microseconds{controller.timeout / controller.bcsTimeoutDivisor}, controller.getSleepValue()); + + auto now = SteadyClock::now(); + controller.timeSinceLastCheck = now; + controller.cpuTimestamp = now; + + defaultController.timeSinceLastCheck = now; + defaultController.cpuTimestamp = now; + + EXPECT_EQ(TimeoutElapsedMode::notElapsed, controller.timeoutElapsed()); + EXPECT_EQ(TimeoutElapsedMode::notElapsed, defaultController.timeoutElapsed()); + + controller.cpuTimestamp = now + std::chrono::microseconds{controller.timeout / controller.bcsTimeoutDivisor}; + defaultController.cpuTimestamp = now + std::chrono::microseconds{defaultController.timeout - std::chrono::microseconds{1}}; + + EXPECT_EQ(TimeoutElapsedMode::bcsOnly, controller.timeoutElapsed()); + EXPECT_EQ(TimeoutElapsedMode::notElapsed, defaultController.timeoutElapsed()); + + controller.cpuTimestamp = now + std::chrono::microseconds{controller.timeout}; + defaultController.cpuTimestamp = now + std::chrono::microseconds{defaultController.timeout}; + EXPECT_EQ(TimeoutElapsedMode::fullyElapsed, controller.timeoutElapsed()); + EXPECT_EQ(TimeoutElapsedMode::fullyElapsed, defaultController.timeoutElapsed()); +} + +TEST(DirectSubmissionControllerTests, givenDebugFlagSetWhenCheckingNewSubmissionThenStopOnlyBcsEngines) { + DebugManagerStateRestore restorer; + debugManager.flags.DirectSubmissionControllerBcsTimeoutDivisor.set(2); + + MockExecutionEnvironment executionEnvironment; + executionEnvironment.prepareRootDeviceEnvironments(1); + executionEnvironment.initializeMemoryManager(); + executionEnvironment.rootDeviceEnvironments[0]->initOsTime(); + + DeviceBitfield deviceBitfield(1); + MockCommandStreamReceiver bcsCsr(executionEnvironment, 0, deviceBitfield); + MockCommandStreamReceiver ccsCsr(executionEnvironment, 0, deviceBitfield); + std::unique_ptr bcsOsContext(OsContext::create(nullptr, 0, 0, EngineDescriptorHelper::getDefaultDescriptor({aub_stream::ENGINE_BCS, EngineUsage::regular}, PreemptionMode::ThreadGroup, deviceBitfield))); + std::unique_ptr ccsOsContext(OsContext::create(nullptr, 0, 0, EngineDescriptorHelper::getDefaultDescriptor({aub_stream::ENGINE_CCS, EngineUsage::regular}, PreemptionMode::ThreadGroup, deviceBitfield))); + bcsCsr.setupContext(*bcsOsContext.get()); + ccsCsr.setupContext(*ccsOsContext.get()); + bcsCsr.taskCount.store(5u); + ccsCsr.taskCount.store(5u); + + DirectSubmissionControllerMock controller; + controller.timeoutElapsedReturnValue.store(TimeoutElapsedMode::fullyElapsed); + controller.registerDirectSubmission(&bcsCsr); + controller.registerDirectSubmission(&ccsCsr); + controller.checkNewSubmissions(); + + EXPECT_FALSE(controller.directSubmissions[&bcsCsr].isStopped); + EXPECT_EQ(controller.directSubmissions[&bcsCsr].taskCount, 5u); + EXPECT_FALSE(controller.directSubmissions[&ccsCsr].isStopped); + EXPECT_EQ(controller.directSubmissions[&ccsCsr].taskCount, 5u); + + controller.timeoutElapsedReturnValue.store(TimeoutElapsedMode::bcsOnly); + + auto timeSinceLastCheck = controller.timeSinceLastCheck; + + bcsCsr.taskCount.store(6u); + ccsCsr.taskCount.store(6u); + controller.checkNewSubmissions(); + EXPECT_FALSE(controller.directSubmissions[&bcsCsr].isStopped); + EXPECT_EQ(controller.directSubmissions[&bcsCsr].taskCount, 6u); + EXPECT_FALSE(controller.directSubmissions[&ccsCsr].isStopped); + EXPECT_EQ(controller.directSubmissions[&ccsCsr].taskCount, 5u); + + controller.checkNewSubmissions(); + EXPECT_TRUE(controller.directSubmissions[&bcsCsr].isStopped); + EXPECT_FALSE(controller.directSubmissions[&ccsCsr].isStopped); + + EXPECT_EQ(timeSinceLastCheck, controller.timeSinceLastCheck); + + controller.unregisterDirectSubmission(&bcsCsr); + controller.unregisterDirectSubmission(&ccsCsr); +} + TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerAndDivisorDisabledWhenIncreaseTimeoutEnabledThenTimeoutIsIncreased) { DebugManagerStateRestore restorer; debugManager.flags.DirectSubmissionControllerMaxTimeout.set(200'000); @@ -96,7 +184,7 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerAndDivisorD csr.setupContext(*osContext.get()); DirectSubmissionControllerMock controller; - controller.timeoutElapsedReturnValue.store(true); + controller.timeoutElapsedReturnValue.store(TimeoutElapsedMode::fullyElapsed); controller.registerDirectSubmission(&csr); { csr.taskCount.store(1u); @@ -216,7 +304,7 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerAndAdjustOn PreemptionMode::ThreadGroup, deviceBitfield))); csr.setupContext(*osContext.get()); DirectSubmissionControllerMock controller; - controller.timeoutElapsedReturnValue.store(true); + controller.timeoutElapsedReturnValue.store(TimeoutElapsedMode::fullyElapsed); controller.setTimeoutParamsForPlatform(csr.getProductHelper()); controller.registerDirectSubmission(&csr); EXPECT_TRUE(controller.adjustTimeoutOnThrottleAndAcLineStatus); @@ -638,7 +726,7 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWhenEnqueue std::mutex mtx; std::unique_lock lock(mtx); - controller.timeoutElapsedReturnValue.store(true); + controller.timeoutElapsedReturnValue.store(TimeoutElapsedMode::fullyElapsed); controller.handlePagingFenceRequests(lock, true); EXPECT_EQ(10u, csr.pagingFenceValueToUnblock); EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 5u); @@ -650,10 +738,10 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWhenCheckTi controller.timeoutElapsedCallBase.store(true); controller.timeSinceLastCheck = controller.getCpuTimestamp() - std::chrono::seconds(10); - EXPECT_TRUE(controller.timeoutElapsed()); + EXPECT_EQ(TimeoutElapsedMode::fullyElapsed, controller.timeoutElapsed()); controller.timeSinceLastCheck = controller.getCpuTimestamp() - std::chrono::seconds(1); - EXPECT_FALSE(controller.timeoutElapsed()); + EXPECT_EQ(TimeoutElapsedMode::notElapsed, controller.timeoutElapsed()); } struct TagUpdateMockCommandStreamReceiver : public MockCommandStreamReceiver { @@ -688,7 +776,7 @@ struct DirectSubmissionIdleDetectionTests : public ::testing::Test { PreemptionMode::ThreadGroup, deviceBitfield))); csr->setupContext(*osContext); - controller->timeoutElapsedReturnValue.store(true); + controller->timeoutElapsedReturnValue.store(TimeoutElapsedMode::fullyElapsed); controller->registerDirectSubmission(csr.get()); csr->taskCount.store(10u); controller->checkNewSubmissions(); @@ -751,7 +839,7 @@ TEST_F(DirectSubmissionIdleDetectionTests, givenDebugFlagSetWhenTaskCountNotUpda debugManager.flags.DirectSubmissionControllerIdleDetection.set(false); controller->unregisterDirectSubmission(csr.get()); controller = std::make_unique(); - controller->timeoutElapsedReturnValue.store(true); + controller->timeoutElapsedReturnValue.store(TimeoutElapsedMode::fullyElapsed); controller->registerDirectSubmission(csr.get()); csr->taskCount.store(10u);