fix: drain paging fence queue before waiting for resources

Related-To: NEO-12197

If ULLS controller waits for CSR lock, and driver must
wait for resources due to OOM, then draing paging fence queue
directly

Signed-off-by: Szymon Morek <szymon.morek@intel.com>

shared/source/command_stream/command_stream_receiver.cpp

@@ -659,6 +659,13 @@ bool CommandStreamReceiver::enqueueWaitForPagingFence(uint64_t pagingFenceValue)
     return false;
 }
 
+void CommandStreamReceiver::drainPagingFenceQueue() {
+    auto controller = this->executionEnvironment.directSubmissionController.get();
+    if (this->isAnyDirectSubmissionEnabled() && controller) {
+        controller->drainPagingFenceQueue();
+    }
+}
+
 GraphicsAllocation *CommandStreamReceiver::allocateDebugSurface(size_t size) {
     UNRECOVERABLE_IF(debugSurface != nullptr);
     if (primaryCsr) {

shared/source/command_stream/command_stream_receiver.h

@@ -547,6 +547,7 @@ class CommandStreamReceiver {
 
     bool enqueueWaitForPagingFence(uint64_t pagingFenceValue);
     virtual void unblockPagingFenceSemaphore(uint64_t pagingFenceValue) {}
+    MOCKABLE_VIRTUAL void drainPagingFenceQueue();
 
   protected:
     void cleanupResources();

shared/source/direct_submission/direct_submission_controller.cpp

@@ -208,6 +208,16 @@ void DirectSubmissionController::enqueueWaitForPagingFence(CommandStreamReceiver
     condVar.notify_one();
 }
 
+void DirectSubmissionController::drainPagingFenceQueue() {
+    std::lock_guard lock(this->condVarMutex);
+
+    while (!pagingFenceRequests.empty()) {
+        auto request = pagingFenceRequests.front();
+        pagingFenceRequests.pop();
+        request.csr->unblockPagingFenceSemaphore(request.pagingFenceValue);
+    }
+}
+
 void DirectSubmissionController::handlePagingFenceRequests(std::unique_lock<std::mutex> &lock, bool checkForNewSubmissions) {
     UNRECOVERABLE_IF(!lock.owns_lock())
     while (!pagingFenceRequests.empty()) {

shared/source/direct_submission/direct_submission_controller.h

@@ -57,6 +57,7 @@ class DirectSubmissionController {
     static bool isSupported();
 
     void enqueueWaitForPagingFence(CommandStreamReceiver *csr, uint64_t pagingFenceValue);
+    void drainPagingFenceQueue();
 
   protected:
     struct DirectSubmissionState {

shared/source/os_interface/windows/trim_callback.cpp

@@ -120,6 +120,7 @@ void WddmResidencyController::trimResidency(const D3DDDI_TRIMRESIDENCYSET_FLAGS
 }
 
 bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes, std::unique_lock<std::mutex> &lock) {
+    this->csr->drainPagingFenceQueue();
     uint64_t sizeToTrim = 0;
     uint64_t numberOfBytesToTrim = bytes;
     WddmAllocation *wddmAllocation = nullptr;

shared/test/common/libult/ult_command_stream_receiver.h

@@ -524,6 +524,11 @@ class UltCommandStreamReceiver : public CommandStreamReceiverHw<GfxFamily>, publ
         BaseClass::unblockPagingFenceSemaphore(pagingFenceValue);
     }
 
+    void drainPagingFenceQueue() override {
+        drainPagingFenceQueueCalled++;
+        BaseClass::drainPagingFenceQueue();
+    }
+
     std::vector<std::string> aubCommentMessages;
 
     BatchBuffer latestFlushedBatchBuffer = {};
@@ -549,6 +554,7 @@ class UltCommandStreamReceiver : public CommandStreamReceiverHw<GfxFamily>, publ
     uint32_t fillReusableAllocationsListCalled = 0;
     uint32_t pollForCompletionCalled = 0;
     uint32_t initializeDeviceWithFirstSubmissionCalled = 0;
+    uint32_t drainPagingFenceQueueCalled = 0;
     mutable uint32_t checkGpuHangDetectedCalled = 0;
     int ensureCommandBufferAllocationCalled = 0;
     DispatchFlags recordedDispatchFlags;

shared/test/unit_test/command_stream/command_stream_receiver_tests.cpp

@@ -43,6 +43,7 @@
 #include "shared/test/common/mocks/mock_bindless_heaps_helper.h"
 #include "shared/test/common/mocks/mock_csr.h"
 #include "shared/test/common/mocks/mock_device.h"
+#include "shared/test/common/mocks/mock_direct_submission_hw.h"
 #include "shared/test/common/mocks/mock_driver_model.h"
 #include "shared/test/common/mocks/mock_execution_environment.h"
 #include "shared/test/common/mocks/mock_internal_allocation_storage.h"
@@ -5861,3 +5862,36 @@ HWTEST_F(CommandStreamReceiverTest, givenCommandStreamReceiverWhenEnqueueWaitFor
     controller->handlePagingFenceRequests(lock, false);
     EXPECT_EQ(10u, csr.pagingFenceValueToUnblock);
 }
+
+HWTEST_F(CommandStreamReceiverTest, givenCommandStreamReceiverWhenDrainPagingFenceQueueThenQueueDrained) {
+    DebugManagerStateRestore restorer;
+    debugManager.flags.EnableDirectSubmissionController.set(1);
+    auto &csr = pDevice->getUltCommandStreamReceiver<FamilyType>();
+    auto directSubmission = new MockDirectSubmissionHw<FamilyType, RenderDispatcher<FamilyType>>(csr);
+    csr.directSubmission.reset(directSubmission);
+
+    auto executionEnvironment = pDevice->getExecutionEnvironment();
+    auto pagingFenceValue = 10u;
+    EXPECT_FALSE(csr.enqueueWaitForPagingFence(pagingFenceValue));
+
+    VariableBackup<decltype(NEO::Thread::createFunc)> funcBackup{&NEO::Thread::createFunc, [](void *(*func)(void *), void *arg) -> std::unique_ptr<Thread> { return nullptr; }};
+
+    csr.drainPagingFenceQueue();
+    EXPECT_EQ(0u, csr.pagingFenceValueToUnblock);
+
+    auto controller = static_cast<DirectSubmissionControllerMock *>(executionEnvironment->initializeDirectSubmissionController());
+    controller->stopThread();
+    csr.directSubmissionAvailable = true;
+    EXPECT_TRUE(csr.enqueueWaitForPagingFence(pagingFenceValue));
+    EXPECT_EQ(0u, csr.pagingFenceValueToUnblock);
+
+    std::mutex mtx;
+    std::unique_lock<std::mutex> lock(mtx);
+    csr.directSubmissionAvailable = false;
+    csr.drainPagingFenceQueue();
+    EXPECT_EQ(0u, csr.pagingFenceValueToUnblock);
+
+    csr.directSubmissionAvailable = true;
+    csr.drainPagingFenceQueue();
+    EXPECT_EQ(10u, csr.pagingFenceValueToUnblock);
+}

shared/test/unit_test/direct_submission/direct_submission_controller_tests.cpp

@@ -587,6 +587,27 @@ TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWhenEnqueue
     EXPECT_EQ(0u, csr.pagingFenceValueToUnblock);
 }
 
+TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWhenDrainPagingFenceQueueThenPagingFenceHandled) {
+    MockExecutionEnvironment executionEnvironment;
+    executionEnvironment.prepareRootDeviceEnvironments(1);
+    executionEnvironment.initializeMemoryManager();
+    DeviceBitfield deviceBitfield(1);
+
+    MockCommandStreamReceiver csr(executionEnvironment, 0, deviceBitfield);
+
+    DirectSubmissionControllerMock controller;
+    EXPECT_TRUE(controller.pagingFenceRequests.empty());
+    controller.enqueueWaitForPagingFence(&csr, 10u);
+    EXPECT_FALSE(controller.pagingFenceRequests.empty());
+
+    auto request = controller.pagingFenceRequests.front();
+    EXPECT_EQ(request.csr, &csr);
+    EXPECT_EQ(request.pagingFenceValue, 10u);
+
+    controller.drainPagingFenceQueue();
+    EXPECT_EQ(10u, csr.pagingFenceValueToUnblock);
+}
+
 TEST(DirectSubmissionControllerTests, givenDirectSubmissionControllerWhenEnqueueWaitForPagingFenceWithCheckSubmissionsThenCheckSubmissions) {
     MockExecutionEnvironment executionEnvironment;
     executionEnvironment.prepareRootDeviceEnvironments(1);

shared/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp

@@ -21,6 +21,7 @@
 #include "shared/test/common/helpers/debug_manager_state_restore.h"
 #include "shared/test/common/helpers/engine_descriptor_helper.h"
 #include "shared/test/common/libult/create_command_stream.h"
+#include "shared/test/common/libult/ult_command_stream_receiver.h"
 #include "shared/test/common/mocks/mock_allocation_properties.h"
 #include "shared/test/common/mocks/mock_execution_environment.h"
 #include "shared/test/common/mocks/mock_io_functions.h"
@@ -417,11 +418,14 @@ TEST_F(WddmResidencyControllerWithGdiTest, givenRestartPeriodicTrimWhenTrimCallb
     EXPECT_EQ(20u, residencyController->lastTrimFenceValue);
 }
 
-TEST_F(WddmResidencyControllerWithGdiTest, GivenZeroWhenTrimmingToBudgetThenTrueIsReturned) {
+HWTEST_F(WddmResidencyControllerWithGdiTest, GivenZeroWhenTrimmingToBudgetThenTrueIsReturnedAndDrainPagingFenceQueueCalled) {
+    auto ultCsr = static_cast<UltCommandStreamReceiver<FamilyType> *>(csr.get());
+    EXPECT_EQ(0u, ultCsr->drainPagingFenceQueueCalled);
     std::mutex mtx;
     std::unique_lock<std::mutex> lock(mtx);
     bool status = residencyController->trimResidencyToBudget(0, lock);
     EXPECT_TRUE(status);
+    EXPECT_EQ(1u, ultCsr->drainPagingFenceQueueCalled);
 }
 
 TEST_F(WddmResidencyControllerWithGdiTest, WhenTrimmingToBudgetThenAllDoneAllocationsAreTrimmed) {