feat: direct submission variable timeout

Add mechanism to increase direct submission timeout up to a maximum
value when no new submissions were made since last sleep.

This should help in workloads that have delays between iterations larger
than current direct submission controller timeout.

Related-To: NEO-7878

Signed-off-by: Dominik Dabek <dominik.dabek@intel.com>

shared/source/debug_settings/debug_variables_base.inl

@@ -339,6 +339,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionDisableMonitorFence, -1, "Disabl
 DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionFlatRingBuffer, -1, "-1: default, 0: disable, 1: enable, Copies task command buffer directly into ring, implemented for immediate command lists only")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableDirectSubmissionController, -1, "Enable direct submission terminating after given timeout, -1: default, 0: disabled, 1: enabled")
 DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionControllerTimeout, -1, "Set direct submission controller timeout, -1: default 5000 us, >=0: timeout in us")
+DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionControllerMaxTimeout, -1, "Set direct submission controller max timeout - timeout will increase up to given value, -1: default 5000 us, >=0: max timeout in us")
 DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionControllerDivisor, -1, "Set direct submission controller timeout divider, -1: default 1, >0: divider value")
 DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionForceLocalMemoryStorageMode, -1, "Force local memory storage for command/ring/semaphore buffer, -1: default - for all engines, 0: disabled, 1: for multiOsContextCapable engine, 2: for all engines")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableRingSwitchTagUpdateWa, -1, "-1: default, 0 - disable, 1 - enable. If enabled, completionFences wont be updated if ring is not running.")

shared/source/direct_submission/direct_submission_controller.cpp

@@ -19,11 +19,14 @@ namespace NEO {
 
 DirectSubmissionController::DirectSubmissionController() {
     if (DebugManager.flags.DirectSubmissionControllerTimeout.get() != -1) {
-        timeout = DebugManager.flags.DirectSubmissionControllerTimeout.get();
+        timeout = std::chrono::microseconds{DebugManager.flags.DirectSubmissionControllerTimeout.get()};
     }
     if (DebugManager.flags.DirectSubmissionControllerDivisor.get() != -1) {
         timeoutDivisor = DebugManager.flags.DirectSubmissionControllerDivisor.get();
     }
+    if (DebugManager.flags.DirectSubmissionControllerMaxTimeout.get() != -1) {
+        maxTimeout = std::chrono::microseconds{DebugManager.flags.DirectSubmissionControllerMaxTimeout.get()};
+    }
 
     directSubmissionControllingThread = Thread::create(controlDirectSubmissionsState, reinterpret_cast<void *>(this));
 };
@@ -74,7 +77,7 @@ void *DirectSubmissionController::controlDirectSubmissionsState(void *self) {
 
 void DirectSubmissionController::checkNewSubmissions() {
     std::lock_guard<std::mutex> lock(this->directSubmissionsMutex);
-
+    bool shouldRecalculateTimeout = false;
     for (auto &directSubmission : this->directSubmissions) {
         auto csr = directSubmission.first;
         auto &state = directSubmission.second;
@@ -87,18 +90,26 @@ void DirectSubmissionController::checkNewSubmissions() {
                 auto lock = csr->obtainUniqueOwnership();
                 csr->stopDirectSubmission();
                 state.isStopped = true;
+                shouldRecalculateTimeout = true;
             }
         } else {
             state.isStopped = false;
             state.taskCount = taskCount;
         }
     }
+    if (shouldRecalculateTimeout) {
+        this->recalculateTimeout();
+    }
 }
 
 void DirectSubmissionController::sleep() {
     std::this_thread::sleep_for(std::chrono::microseconds(this->timeout));
 }
 
+SteadyClock::time_point DirectSubmissionController::getCpuTimestamp() {
+    return SteadyClock::now();
+}
+
 void DirectSubmissionController::adjustTimeout(CommandStreamReceiver *csr) {
     if (EngineHelpers::isCcs(csr->getOsContext().getEngineType())) {
         for (size_t subDeviceIndex = 0u; subDeviceIndex < csr->getOsContext().getDeviceBitfield().size(); ++subDeviceIndex) {
@@ -114,4 +125,16 @@ void DirectSubmissionController::adjustTimeout(CommandStreamReceiver *csr) {
     }
 }
 
+void DirectSubmissionController::recalculateTimeout() {
+    const auto now = this->getCpuTimestamp();
+    const auto timeSinceLastTerminate = std::chrono::duration_cast<std::chrono::microseconds>(now - this->lastTerminateCpuTimestamp);
+    DEBUG_BREAK_IF(timeSinceLastTerminate.count() < 0);
+    if (timeSinceLastTerminate.count() > this->timeout.count() &&
+        timeSinceLastTerminate.count() <= this->maxTimeout.count()) {
+        const auto newTimeout = std::chrono::duration_cast<std::chrono::microseconds>(timeSinceLastTerminate * 1.5);
+        this->timeout = newTimeout.count() < this->maxTimeout.count() ? newTimeout : this->maxTimeout;
+    }
+    this->lastTerminateCpuTimestamp = now;
+}
+
 } // namespace NEO

shared/source/direct_submission/direct_submission_controller.h

@@ -12,6 +12,7 @@
 
 #include <array>
 #include <atomic>
+#include <chrono>
 #include <memory>
 #include <mutex>
 #include <unordered_map>
@@ -21,8 +22,11 @@ class MemoryManager;
 class CommandStreamReceiver;
 class Thread;
 
+using SteadyClock = std::chrono::steady_clock;
+
 class DirectSubmissionController {
   public:
+    static constexpr size_t defaultTimeout = 5'000;
     DirectSubmissionController();
     virtual ~DirectSubmissionController();
 
@@ -42,8 +46,10 @@ class DirectSubmissionController {
     static void *controlDirectSubmissionsState(void *self);
     void checkNewSubmissions();
     MOCKABLE_VIRTUAL void sleep();
+    MOCKABLE_VIRTUAL SteadyClock::time_point getCpuTimestamp();
 
     void adjustTimeout(CommandStreamReceiver *csr);
+    void recalculateTimeout();
 
     uint32_t maxCcsCount = 1u;
     std::array<uint32_t, DeviceBitfield().size()> ccsCount = {};
@@ -54,7 +60,9 @@ class DirectSubmissionController {
     std::atomic_bool keepControlling = true;
     std::atomic_bool runControlling = false;
 
-    int timeout = 5000;
+    SteadyClock::time_point lastTerminateCpuTimestamp{};
+    std::chrono::microseconds maxTimeout{defaultTimeout};
+    std::chrono::microseconds timeout{defaultTimeout};
     int timeoutDivisor = 1;
 };
 } // namespace NEO

shared/test/common/test_files/igdrcl.config

@@ -512,3 +512,4 @@ ForceDummyBlitWa = -1
 DetectIndirectAccessInKernel = -1
 OptimizeIoqBarriersHandling = -1
 AllocateSharedAllocationsInHeapExtended = 0
+DirectSubmissionControllerMaxTimeout = -1

shared/test/unit_test/direct_submission/direct_submission_controller_mock.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2022 Intel Corporation
+ * Copyright (C) 2019-2023 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -16,6 +16,8 @@ struct DirectSubmissionControllerMock : public DirectSubmissionController {
     using DirectSubmissionController::directSubmissions;
     using DirectSubmissionController::directSubmissionsMutex;
     using DirectSubmissionController::keepControlling;
+    using DirectSubmissionController::lastTerminateCpuTimestamp;
+    using DirectSubmissionController::maxTimeout;
     using DirectSubmissionController::timeout;
     using DirectSubmissionController::timeoutDivisor;
 
@@ -24,6 +26,11 @@ struct DirectSubmissionControllerMock : public DirectSubmissionController {
         this->sleepCalled = true;
     }
 
+    SteadyClock::time_point getCpuTimestamp() override {
+        return cpuTimestamp;
+    }
+
+    SteadyClock::time_point cpuTimestamp{};
     bool sleepCalled = false;
 };
 } // namespace NEO

shared/test/unit_test/direct_submission/direct_submission_controller_tests.cpp

@@ -22,7 +22,7 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerTimeoutWhen
 
     DirectSubmissionControllerMock controller;
 
-    EXPECT_EQ(controller.timeout, 14);
+    EXPECT_EQ(controller.timeout.count(), 14);
 }
 
 TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllertimeoutDivisorWhenCreateObjectThentimeoutDivisorIsEqualWithDebugFlag) {
@@ -124,6 +124,87 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWithStarted
     controller.directSubmissionControllingThread.reset();
 }
 
+TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerAndDivisorDisabledWhenIncreaseTimeoutEnabledThenTimeoutIsIncreased) {
+    DebugManagerStateRestore restorer;
+    DebugManager.flags.DirectSubmissionControllerMaxTimeout.set(200'000);
+    DebugManager.flags.DirectSubmissionControllerDivisor.set(1);
+    MockExecutionEnvironment executionEnvironment;
+    executionEnvironment.prepareRootDeviceEnvironments(1);
+    executionEnvironment.initializeMemoryManager();
+
+    DeviceBitfield deviceBitfield(1);
+    MockCommandStreamReceiver csr(executionEnvironment, 0, deviceBitfield);
+    std::unique_ptr<OsContext> osContext(OsContext::create(nullptr, 0, 0,
+                                                           EngineDescriptorHelper::getDefaultDescriptor({aub_stream::ENGINE_CCS, EngineUsage::Regular},
+                                                                                                        PreemptionMode::ThreadGroup, deviceBitfield)));
+    csr.setupContext(*osContext.get());
+
+    DirectSubmissionControllerMock controller;
+    controller.keepControlling.store(false);
+    controller.directSubmissionControllingThread->join();
+    controller.directSubmissionControllingThread.reset();
+    controller.registerDirectSubmission(&csr);
+    {
+        csr.taskCount.store(1u);
+        controller.checkNewSubmissions();
+        EXPECT_FALSE(controller.directSubmissions[&csr].isStopped);
+        EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 1u);
+
+        auto previousTimestamp = controller.lastTerminateCpuTimestamp;
+        controller.cpuTimestamp += std::chrono::microseconds(5'000);
+        controller.checkNewSubmissions();
+        EXPECT_EQ(std::chrono::duration_cast<std::chrono::microseconds>(controller.lastTerminateCpuTimestamp - previousTimestamp).count(), 5'000);
+        EXPECT_TRUE(controller.directSubmissions[&csr].isStopped);
+        EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 1u);
+        EXPECT_EQ(controller.timeout.count(), 5'000);
+        EXPECT_EQ(controller.maxTimeout.count(), 200'000);
+    }
+    {
+        csr.taskCount.store(2u);
+        controller.checkNewSubmissions();
+        EXPECT_FALSE(controller.directSubmissions[&csr].isStopped);
+        EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 2u);
+
+        auto previousTimestamp = controller.lastTerminateCpuTimestamp;
+        controller.cpuTimestamp += std::chrono::microseconds(5'500);
+        controller.checkNewSubmissions();
+        EXPECT_EQ(std::chrono::duration_cast<std::chrono::microseconds>(controller.lastTerminateCpuTimestamp - previousTimestamp).count(), 5'500);
+        EXPECT_TRUE(controller.directSubmissions[&csr].isStopped);
+        EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 2u);
+        EXPECT_EQ(controller.timeout.count(), 8'250);
+    }
+    {
+        csr.taskCount.store(3u);
+        controller.checkNewSubmissions();
+        EXPECT_FALSE(controller.directSubmissions[&csr].isStopped);
+        EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 3u);
+
+        auto previousTimestamp = controller.lastTerminateCpuTimestamp;
+        controller.cpuTimestamp += controller.maxTimeout;
+        controller.checkNewSubmissions();
+        EXPECT_EQ(std::chrono::duration_cast<std::chrono::microseconds>(controller.lastTerminateCpuTimestamp - previousTimestamp).count(), controller.maxTimeout.count());
+        EXPECT_TRUE(controller.directSubmissions[&csr].isStopped);
+        EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 3u);
+        EXPECT_EQ(controller.timeout.count(), controller.maxTimeout.count());
+    }
+    {
+        controller.timeout = std::chrono::microseconds(5'000);
+        csr.taskCount.store(4u);
+        controller.checkNewSubmissions();
+        EXPECT_FALSE(controller.directSubmissions[&csr].isStopped);
+        EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 4u);
+
+        auto previousTimestamp = controller.lastTerminateCpuTimestamp;
+        controller.cpuTimestamp += controller.maxTimeout * 2;
+        controller.checkNewSubmissions();
+        EXPECT_EQ(std::chrono::duration_cast<std::chrono::microseconds>(controller.lastTerminateCpuTimestamp - previousTimestamp).count(), controller.maxTimeout.count() * 2);
+        EXPECT_TRUE(controller.directSubmissions[&csr].isStopped);
+        EXPECT_EQ(controller.directSubmissions[&csr].taskCount, 4u);
+        EXPECT_EQ(controller.timeout.count(), 5'000);
+    }
+    controller.unregisterDirectSubmission(&csr);
+}
+
 TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWithNotStartedControllingWhenShuttingDownThenNoHang) {
     DirectSubmissionControllerMock controller;
     EXPECT_NE(controller.directSubmissionControllingThread.get(), nullptr);
@@ -176,22 +257,22 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWhenRegiste
     controller.directSubmissionControllingThread->join();
     controller.directSubmissionControllingThread.reset();
 
-    EXPECT_EQ(controller.timeout, 5000);
+    EXPECT_EQ(controller.timeout.count(), 5'000);
 
     controller.registerDirectSubmission(&csr);
-    EXPECT_EQ(controller.timeout, 5000);
+    EXPECT_EQ(controller.timeout.count(), 5'000);
 
     controller.registerDirectSubmission(&csr3);
-    EXPECT_EQ(controller.timeout, 5000);
+    EXPECT_EQ(controller.timeout.count(), 5'000);
 
     controller.registerDirectSubmission(&csr1);
-    EXPECT_EQ(controller.timeout, 5000);
+    EXPECT_EQ(controller.timeout.count(), 5'000);
 
     controller.registerDirectSubmission(&csr2);
-    EXPECT_EQ(controller.timeout, 5000);
+    EXPECT_EQ(controller.timeout.count(), 5'000);
 
     controller.registerDirectSubmission(&csr4);
-    EXPECT_EQ(controller.timeout, 5000);
+    EXPECT_EQ(controller.timeout.count(), 5'000);
 
     controller.unregisterDirectSubmission(&csr);
     controller.unregisterDirectSubmission(&csr1);
@@ -280,37 +361,37 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWhenRegiste
     controller.directSubmissionControllingThread->join();
     controller.directSubmissionControllingThread.reset();
 
-    EXPECT_EQ(controller.timeout, 5000);
+    EXPECT_EQ(controller.timeout.count(), 5'000);
 
     controller.registerDirectSubmission(&csr);
-    EXPECT_EQ(controller.timeout, 5000);
+    EXPECT_EQ(controller.timeout.count(), 5'000);
 
     controller.registerDirectSubmission(&csr5);
-    EXPECT_EQ(controller.timeout, 5000);
+    EXPECT_EQ(controller.timeout.count(), 5'000);
 
     controller.registerDirectSubmission(&csr1);
-    EXPECT_EQ(controller.timeout, 1250);
+    EXPECT_EQ(controller.timeout.count(), 1'250);
 
     controller.registerDirectSubmission(&csr2);
-    EXPECT_EQ(controller.timeout, 312);
+    EXPECT_EQ(controller.timeout.count(), 312);
 
     controller.registerDirectSubmission(&csr4);
-    EXPECT_EQ(controller.timeout, 312);
+    EXPECT_EQ(controller.timeout.count(), 312);
 
     controller.registerDirectSubmission(&csr6);
-    EXPECT_EQ(controller.timeout, 312);
+    EXPECT_EQ(controller.timeout.count(), 312);
 
     controller.registerDirectSubmission(&csr7);
-    EXPECT_EQ(controller.timeout, 312);
+    EXPECT_EQ(controller.timeout.count(), 312);
 
     controller.registerDirectSubmission(&csr9);
-    EXPECT_EQ(controller.timeout, 312);
+    EXPECT_EQ(controller.timeout.count(), 312);
 
     controller.registerDirectSubmission(&csr8);
-    EXPECT_EQ(controller.timeout, 78);
+    EXPECT_EQ(controller.timeout.count(), 78);
 
     controller.registerDirectSubmission(&csr10);
-    EXPECT_EQ(controller.timeout, 78);
+    EXPECT_EQ(controller.timeout.count(), 78);
 
     controller.unregisterDirectSubmission(&csr);
     controller.unregisterDirectSubmission(&csr1);
@@ -363,22 +444,22 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerDirectSubmi
     controller.directSubmissionControllingThread->join();
     controller.directSubmissionControllingThread.reset();
 
-    EXPECT_EQ(controller.timeout, 5000);
+    EXPECT_EQ(controller.timeout.count(), 5'000);
 
     controller.registerDirectSubmission(&csr);
-    EXPECT_EQ(controller.timeout, 5000);
+    EXPECT_EQ(controller.timeout.count(), 5'000);
 
     controller.registerDirectSubmission(&csr3);
-    EXPECT_EQ(controller.timeout, 5000);
+    EXPECT_EQ(controller.timeout.count(), 5'000);
 
     controller.registerDirectSubmission(&csr1);
-    EXPECT_EQ(controller.timeout, 1000);
+    EXPECT_EQ(controller.timeout.count(), 1'000);
 
     controller.registerDirectSubmission(&csr2);
-    EXPECT_EQ(controller.timeout, 200);
+    EXPECT_EQ(controller.timeout.count(), 200);
 
     controller.registerDirectSubmission(&csr4);
-    EXPECT_EQ(controller.timeout, 200);
+    EXPECT_EQ(controller.timeout.count(), 200);
 
     controller.unregisterDirectSubmission(&csr);
     controller.unregisterDirectSubmission(&csr1);