diff --git a/shared/source/debug_settings/debug_variables_base.inl b/shared/source/debug_settings/debug_variables_base.inl index 0eb184ec4b..cb65b13133 100644 --- a/shared/source/debug_settings/debug_variables_base.inl +++ b/shared/source/debug_settings/debug_variables_base.inl @@ -339,6 +339,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionDisableMonitorFence, -1, "Disabl DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionFlatRingBuffer, -1, "-1: default, 0: disable, 1: enable, Copies task command buffer directly into ring, implemented for immediate command lists only") DECLARE_DEBUG_VARIABLE(int32_t, EnableDirectSubmissionController, -1, "Enable direct submission terminating after given timeout, -1: default, 0: disabled, 1: enabled") DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionControllerTimeout, -1, "Set direct submission controller timeout, -1: default 5000 us, >=0: timeout in us") +DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionControllerMaxTimeout, -1, "Set direct submission controller max timeout - timeout will increase up to given value, -1: default 5000 us, >=0: max timeout in us") DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionControllerDivisor, -1, "Set direct submission controller timeout divider, -1: default 1, >0: divider value") DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionForceLocalMemoryStorageMode, -1, "Force local memory storage for command/ring/semaphore buffer, -1: default - for all engines, 0: disabled, 1: for multiOsContextCapable engine, 2: for all engines") DECLARE_DEBUG_VARIABLE(int32_t, EnableRingSwitchTagUpdateWa, -1, "-1: default, 0 - disable, 1 - enable. If enabled, completionFences wont be updated if ring is not running.") diff --git a/shared/source/direct_submission/direct_submission_controller.cpp b/shared/source/direct_submission/direct_submission_controller.cpp index 5fe94d8a81..6b6cae687d 100644 --- a/shared/source/direct_submission/direct_submission_controller.cpp +++ b/shared/source/direct_submission/direct_submission_controller.cpp @@ -19,11 +19,14 @@ namespace NEO { DirectSubmissionController::DirectSubmissionController() { if (DebugManager.flags.DirectSubmissionControllerTimeout.get() != -1) { - timeout = DebugManager.flags.DirectSubmissionControllerTimeout.get(); + timeout = std::chrono::microseconds{DebugManager.flags.DirectSubmissionControllerTimeout.get()}; } if (DebugManager.flags.DirectSubmissionControllerDivisor.get() != -1) { timeoutDivisor = DebugManager.flags.DirectSubmissionControllerDivisor.get(); } + if (DebugManager.flags.DirectSubmissionControllerMaxTimeout.get() != -1) { + maxTimeout = std::chrono::microseconds{DebugManager.flags.DirectSubmissionControllerMaxTimeout.get()}; + } directSubmissionControllingThread = Thread::create(controlDirectSubmissionsState, reinterpret_cast(this)); }; @@ -74,7 +77,7 @@ void *DirectSubmissionController::controlDirectSubmissionsState(void *self) { void DirectSubmissionController::checkNewSubmissions() { std::lock_guard lock(this->directSubmissionsMutex); - + bool shouldRecalculateTimeout = false; for (auto &directSubmission : this->directSubmissions) { auto csr = directSubmission.first; auto &state = directSubmission.second; @@ -87,18 +90,26 @@ void DirectSubmissionController::checkNewSubmissions() { auto lock = csr->obtainUniqueOwnership(); csr->stopDirectSubmission(); state.isStopped = true; + shouldRecalculateTimeout = true; } } else { state.isStopped = false; state.taskCount = taskCount; } } + if (shouldRecalculateTimeout) { + this->recalculateTimeout(); + } } void DirectSubmissionController::sleep() { std::this_thread::sleep_for(std::chrono::microseconds(this->timeout)); } +SteadyClock::time_point DirectSubmissionController::getCpuTimestamp() { + return SteadyClock::now(); +} + void DirectSubmissionController::adjustTimeout(CommandStreamReceiver *csr) { if (EngineHelpers::isCcs(csr->getOsContext().getEngineType())) { for (size_t subDeviceIndex = 0u; subDeviceIndex < csr->getOsContext().getDeviceBitfield().size(); ++subDeviceIndex) { @@ -114,4 +125,16 @@ void DirectSubmissionController::adjustTimeout(CommandStreamReceiver *csr) { } } +void DirectSubmissionController::recalculateTimeout() { + const auto now = this->getCpuTimestamp(); + const auto timeSinceLastTerminate = std::chrono::duration_cast(now - this->lastTerminateCpuTimestamp); + DEBUG_BREAK_IF(timeSinceLastTerminate.count() < 0); + if (timeSinceLastTerminate.count() > this->timeout.count() && + timeSinceLastTerminate.count() <= this->maxTimeout.count()) { + const auto newTimeout = std::chrono::duration_cast(timeSinceLastTerminate * 1.5); + this->timeout = newTimeout.count() < this->maxTimeout.count() ? newTimeout : this->maxTimeout; + } + this->lastTerminateCpuTimestamp = now; +} + } // namespace NEO \ No newline at end of file diff --git a/shared/source/direct_submission/direct_submission_controller.h b/shared/source/direct_submission/direct_submission_controller.h index 26668245b2..4d4c669db2 100644 --- a/shared/source/direct_submission/direct_submission_controller.h +++ b/shared/source/direct_submission/direct_submission_controller.h @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -21,8 +22,11 @@ class MemoryManager; class CommandStreamReceiver; class Thread; +using SteadyClock = std::chrono::steady_clock; + class DirectSubmissionController { public: + static constexpr size_t defaultTimeout = 5'000; DirectSubmissionController(); virtual ~DirectSubmissionController(); @@ -42,8 +46,10 @@ class DirectSubmissionController { static void *controlDirectSubmissionsState(void *self); void checkNewSubmissions(); MOCKABLE_VIRTUAL void sleep(); + MOCKABLE_VIRTUAL SteadyClock::time_point getCpuTimestamp(); void adjustTimeout(CommandStreamReceiver *csr); + void recalculateTimeout(); uint32_t maxCcsCount = 1u; std::array ccsCount = {}; @@ -54,7 +60,9 @@ class DirectSubmissionController { std::atomic_bool keepControlling = true; std::atomic_bool runControlling = false; - int timeout = 5000; + SteadyClock::time_point lastTerminateCpuTimestamp{}; + std::chrono::microseconds maxTimeout{defaultTimeout}; + std::chrono::microseconds timeout{defaultTimeout}; int timeoutDivisor = 1; }; } // namespace NEO \ No newline at end of file diff --git a/shared/test/common/test_files/igdrcl.config b/shared/test/common/test_files/igdrcl.config index e83c61b39b..dbab8e57a4 100644 --- a/shared/test/common/test_files/igdrcl.config +++ b/shared/test/common/test_files/igdrcl.config @@ -511,4 +511,5 @@ ExperimentalCopyThroughLockWaitlistSizeThreshold= -1 ForceDummyBlitWa = -1 DetectIndirectAccessInKernel = -1 OptimizeIoqBarriersHandling = -1 -AllocateSharedAllocationsInHeapExtended = 0 \ No newline at end of file +AllocateSharedAllocationsInHeapExtended = 0 +DirectSubmissionControllerMaxTimeout = -1 diff --git a/shared/test/unit_test/direct_submission/direct_submission_controller_mock.h b/shared/test/unit_test/direct_submission/direct_submission_controller_mock.h index d440f48a80..63c49cc11e 100644 --- a/shared/test/unit_test/direct_submission/direct_submission_controller_mock.h +++ b/shared/test/unit_test/direct_submission/direct_submission_controller_mock.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2022 Intel Corporation + * Copyright (C) 2019-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -16,6 +16,8 @@ struct DirectSubmissionControllerMock : public DirectSubmissionController { using DirectSubmissionController::directSubmissions; using DirectSubmissionController::directSubmissionsMutex; using DirectSubmissionController::keepControlling; + using DirectSubmissionController::lastTerminateCpuTimestamp; + using DirectSubmissionController::maxTimeout; using DirectSubmissionController::timeout; using DirectSubmissionController::timeoutDivisor; @@ -24,6 +26,11 @@ struct DirectSubmissionControllerMock : public DirectSubmissionController { this->sleepCalled = true; } + SteadyClock::time_point getCpuTimestamp() override { + return cpuTimestamp; + } + + SteadyClock::time_point cpuTimestamp{}; bool sleepCalled = false; }; } // namespace NEO \ No newline at end of file diff --git a/shared/test/unit_test/direct_submission/direct_submission_controller_tests.cpp b/shared/test/unit_test/direct_submission/direct_submission_controller_tests.cpp index 05b52c686c..c9f48eea23 100644 --- a/shared/test/unit_test/direct_submission/direct_submission_controller_tests.cpp +++ b/shared/test/unit_test/direct_submission/direct_submission_controller_tests.cpp @@ -22,7 +22,7 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerTimeoutWhen DirectSubmissionControllerMock controller; - EXPECT_EQ(controller.timeout, 14); + EXPECT_EQ(controller.timeout.count(), 14); } TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllertimeoutDivisorWhenCreateObjectThentimeoutDivisorIsEqualWithDebugFlag) { @@ -124,6 +124,87 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWithStarted controller.directSubmissionControllingThread.reset(); } +TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerAndDivisorDisabledWhenIncreaseTimeoutEnabledThenTimeoutIsIncreased) { + DebugManagerStateRestore restorer; + DebugManager.flags.DirectSubmissionControllerMaxTimeout.set(200'000); + DebugManager.flags.DirectSubmissionControllerDivisor.set(1); + MockExecutionEnvironment executionEnvironment; + executionEnvironment.prepareRootDeviceEnvironments(1); + executionEnvironment.initializeMemoryManager(); + + DeviceBitfield deviceBitfield(1); + MockCommandStreamReceiver csr(executionEnvironment, 0, deviceBitfield); + std::unique_ptr osContext(OsContext::create(nullptr, 0, 0, + EngineDescriptorHelper::getDefaultDescriptor({aub_stream::ENGINE_CCS, EngineUsage::Regular}, + PreemptionMode::ThreadGroup, deviceBitfield))); + csr.setupContext(*osContext.get()); + + DirectSubmissionControllerMock controller; + controller.keepControlling.store(false); + controller.directSubmissionControllingThread->join(); + controller.directSubmissionControllingThread.reset(); + controller.registerDirectSubmission(&csr); + { + csr.taskCount.store(1u); + controller.checkNewSubmissions(); + EXPECT_FALSE(controller.directSubmissions[&csr].isStopped); + EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 1u); + + auto previousTimestamp = controller.lastTerminateCpuTimestamp; + controller.cpuTimestamp += std::chrono::microseconds(5'000); + controller.checkNewSubmissions(); + EXPECT_EQ(std::chrono::duration_cast(controller.lastTerminateCpuTimestamp - previousTimestamp).count(), 5'000); + EXPECT_TRUE(controller.directSubmissions[&csr].isStopped); + EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 1u); + EXPECT_EQ(controller.timeout.count(), 5'000); + EXPECT_EQ(controller.maxTimeout.count(), 200'000); + } + { + csr.taskCount.store(2u); + controller.checkNewSubmissions(); + EXPECT_FALSE(controller.directSubmissions[&csr].isStopped); + EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 2u); + + auto previousTimestamp = controller.lastTerminateCpuTimestamp; + controller.cpuTimestamp += std::chrono::microseconds(5'500); + controller.checkNewSubmissions(); + EXPECT_EQ(std::chrono::duration_cast(controller.lastTerminateCpuTimestamp - previousTimestamp).count(), 5'500); + EXPECT_TRUE(controller.directSubmissions[&csr].isStopped); + EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 2u); + EXPECT_EQ(controller.timeout.count(), 8'250); + } + { + csr.taskCount.store(3u); + controller.checkNewSubmissions(); + EXPECT_FALSE(controller.directSubmissions[&csr].isStopped); + EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 3u); + + auto previousTimestamp = controller.lastTerminateCpuTimestamp; + controller.cpuTimestamp += controller.maxTimeout; + controller.checkNewSubmissions(); + EXPECT_EQ(std::chrono::duration_cast(controller.lastTerminateCpuTimestamp - previousTimestamp).count(), controller.maxTimeout.count()); + EXPECT_TRUE(controller.directSubmissions[&csr].isStopped); + EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 3u); + EXPECT_EQ(controller.timeout.count(), controller.maxTimeout.count()); + } + { + controller.timeout = std::chrono::microseconds(5'000); + csr.taskCount.store(4u); + controller.checkNewSubmissions(); + EXPECT_FALSE(controller.directSubmissions[&csr].isStopped); + EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 4u); + + auto previousTimestamp = controller.lastTerminateCpuTimestamp; + controller.cpuTimestamp += controller.maxTimeout * 2; + controller.checkNewSubmissions(); + EXPECT_EQ(std::chrono::duration_cast(controller.lastTerminateCpuTimestamp - previousTimestamp).count(), controller.maxTimeout.count() * 2); + EXPECT_TRUE(controller.directSubmissions[&csr].isStopped); + EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 4u); + EXPECT_EQ(controller.timeout.count(), 5'000); + } + controller.unregisterDirectSubmission(&csr); +} + TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWithNotStartedControllingWhenShuttingDownThenNoHang) { DirectSubmissionControllerMock controller; EXPECT_NE(controller.directSubmissionControllingThread.get(), nullptr); @@ -176,22 +257,22 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWhenRegiste controller.directSubmissionControllingThread->join(); controller.directSubmissionControllingThread.reset(); - EXPECT_EQ(controller.timeout, 5000); + EXPECT_EQ(controller.timeout.count(), 5'000); controller.registerDirectSubmission(&csr); - EXPECT_EQ(controller.timeout, 5000); + EXPECT_EQ(controller.timeout.count(), 5'000); controller.registerDirectSubmission(&csr3); - EXPECT_EQ(controller.timeout, 5000); + EXPECT_EQ(controller.timeout.count(), 5'000); controller.registerDirectSubmission(&csr1); - EXPECT_EQ(controller.timeout, 5000); + EXPECT_EQ(controller.timeout.count(), 5'000); controller.registerDirectSubmission(&csr2); - EXPECT_EQ(controller.timeout, 5000); + EXPECT_EQ(controller.timeout.count(), 5'000); controller.registerDirectSubmission(&csr4); - EXPECT_EQ(controller.timeout, 5000); + EXPECT_EQ(controller.timeout.count(), 5'000); controller.unregisterDirectSubmission(&csr); controller.unregisterDirectSubmission(&csr1); @@ -280,37 +361,37 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWhenRegiste controller.directSubmissionControllingThread->join(); controller.directSubmissionControllingThread.reset(); - EXPECT_EQ(controller.timeout, 5000); + EXPECT_EQ(controller.timeout.count(), 5'000); controller.registerDirectSubmission(&csr); - EXPECT_EQ(controller.timeout, 5000); + EXPECT_EQ(controller.timeout.count(), 5'000); controller.registerDirectSubmission(&csr5); - EXPECT_EQ(controller.timeout, 5000); + EXPECT_EQ(controller.timeout.count(), 5'000); controller.registerDirectSubmission(&csr1); - EXPECT_EQ(controller.timeout, 1250); + EXPECT_EQ(controller.timeout.count(), 1'250); controller.registerDirectSubmission(&csr2); - EXPECT_EQ(controller.timeout, 312); + EXPECT_EQ(controller.timeout.count(), 312); controller.registerDirectSubmission(&csr4); - EXPECT_EQ(controller.timeout, 312); + EXPECT_EQ(controller.timeout.count(), 312); controller.registerDirectSubmission(&csr6); - EXPECT_EQ(controller.timeout, 312); + EXPECT_EQ(controller.timeout.count(), 312); controller.registerDirectSubmission(&csr7); - EXPECT_EQ(controller.timeout, 312); + EXPECT_EQ(controller.timeout.count(), 312); controller.registerDirectSubmission(&csr9); - EXPECT_EQ(controller.timeout, 312); + EXPECT_EQ(controller.timeout.count(), 312); controller.registerDirectSubmission(&csr8); - EXPECT_EQ(controller.timeout, 78); + EXPECT_EQ(controller.timeout.count(), 78); controller.registerDirectSubmission(&csr10); - EXPECT_EQ(controller.timeout, 78); + EXPECT_EQ(controller.timeout.count(), 78); controller.unregisterDirectSubmission(&csr); controller.unregisterDirectSubmission(&csr1); @@ -363,22 +444,22 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerDirectSubmi controller.directSubmissionControllingThread->join(); controller.directSubmissionControllingThread.reset(); - EXPECT_EQ(controller.timeout, 5000); + EXPECT_EQ(controller.timeout.count(), 5'000); controller.registerDirectSubmission(&csr); - EXPECT_EQ(controller.timeout, 5000); + EXPECT_EQ(controller.timeout.count(), 5'000); controller.registerDirectSubmission(&csr3); - EXPECT_EQ(controller.timeout, 5000); + EXPECT_EQ(controller.timeout.count(), 5'000); controller.registerDirectSubmission(&csr1); - EXPECT_EQ(controller.timeout, 1000); + EXPECT_EQ(controller.timeout.count(), 1'000); controller.registerDirectSubmission(&csr2); - EXPECT_EQ(controller.timeout, 200); + EXPECT_EQ(controller.timeout.count(), 200); controller.registerDirectSubmission(&csr4); - EXPECT_EQ(controller.timeout, 200); + EXPECT_EQ(controller.timeout.count(), 200); controller.unregisterDirectSubmission(&csr); controller.unregisterDirectSubmission(&csr1);