Recycle old command buffers of immediate command lists

Related-To: NEO-6242

Signed-off-by: Zbigniew Zdanowicz <zbigniew.zdanowicz@intel.com>

shared/source/command_container/cmdcontainer.cpp

@@ -229,13 +229,17 @@ IndirectHeap *CommandContainer::getHeapWithRequiredSizeAndAlignment(HeapType hea
 
 void CommandContainer::handleCmdBufferAllocations(size_t startIndex) {
     for (size_t i = startIndex; i < cmdBufferAllocations.size(); i++) {
-        if (this->reusableAllocationList) {
+        handleCmdBufferAllocation(cmdBufferAllocations[i]);
-            this->device->getMemoryManager()->handleFenceCompletion(cmdBufferAllocations[i]);
-            reusableAllocationList->pushFrontOne(*cmdBufferAllocations[i]);
-        } else {
-            this->device->getMemoryManager()->freeGraphicsMemory(cmdBufferAllocations[i]);
     }
 }
+
+void CommandContainer::handleCmdBufferAllocation(GraphicsAllocation *allocation) {
+    if (this->reusableAllocationList) {
+        this->device->getMemoryManager()->handleFenceCompletion(allocation);
+        reusableAllocationList->pushFrontOne(*allocation);
+    } else {
+        this->device->getMemoryManager()->freeGraphicsMemory(allocation);
+    }
 }
 
 GraphicsAllocation *CommandContainer::obtainNextCommandBufferAllocation() {
@@ -264,6 +268,11 @@ void CommandContainer::allocateNextCommandBuffer() {
     auto cmdBufferAllocation = this->obtainNextCommandBufferAllocation();
     UNRECOVERABLE_IF(!cmdBufferAllocation);
 
+    if (getFlushTaskUsedForImmediate()) {
+        NEO::GraphicsAllocation *oldCmdBuffer = cmdBufferAllocations.back();
+        handleCmdBufferAllocation(oldCmdBuffer);
+        cmdBufferAllocations.pop_back();
+    }
     cmdBufferAllocations.push_back(cmdBufferAllocation);
 
     size_t alignedSize = alignUp<size_t>(totalCmdBufferSize, MemoryConstants::pageSize64k);

shared/source/command_container/cmdcontainer.h

@@ -88,6 +88,7 @@ class CommandContainer : public NonCopyableOrMovableClass {
     void closeAndAllocateNextCommandBuffer();
 
     void handleCmdBufferAllocations(size_t startIndex);
+    void handleCmdBufferAllocation(GraphicsAllocation *allocation);
     GraphicsAllocation *obtainNextCommandBufferAllocation();
 
     void reset();

shared/test/common/fixtures/command_container_fixture.h

@@ -9,20 +9,16 @@
 #include "shared/source/command_container/command_encoder.h"
 #include "shared/source/kernel/kernel_descriptor.h"
 #include "shared/test/common/fixtures/device_fixture.h"
+#include "shared/test/common/mocks/mock_command_container.h"
 #include "shared/test/common/test_macros/test.h"
 
 namespace NEO {
 
 class CommandEncodeStatesFixture : public DeviceFixture {
   public:
-    class MyMockCommandContainer : public CommandContainer {
-      public:
-        using CommandContainer::dirtyHeaps;
-    };
-
     void SetUp() {
         DeviceFixture::SetUp();
-        cmdContainer = std::make_unique<MyMockCommandContainer>();
+        cmdContainer = std::make_unique<MockCommandContainer>();
         cmdContainer->initialize(pDevice, nullptr);
         cmdContainer->setDirtyStateForAllHeaps(false);
     }
@@ -30,7 +26,7 @@ class CommandEncodeStatesFixture : public DeviceFixture {
         cmdContainer.reset();
         DeviceFixture::TearDown();
     }
-    std::unique_ptr<MyMockCommandContainer> cmdContainer;
+    std::unique_ptr<MockCommandContainer> cmdContainer;
     KernelDescriptor descriptor;
 
     EncodeDispatchKernelArgs createDefaultDispatchKernelArgs(Device *device,

shared/test/common/mocks/CMakeLists.txt

@@ -27,6 +27,7 @@ set(NEO_CORE_tests_mocks
     ${CMAKE_CURRENT_SOURCE_DIR}/mock_builtinslib.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock_cif.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/mock_cif.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/mock_command_container.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock_command_stream_receiver.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/mock_command_stream_receiver.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock_compiler_interface_spirv.cpp

shared/test/common/mocks/mock_command_container.h

@@ -0,0 +1,16 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+#include "shared/source/command_container/cmdcontainer.h"
+
+struct MockCommandContainer : public NEO::CommandContainer {
+    using CommandContainer::cmdBufferAllocations;
+    using CommandContainer::dirtyHeaps;
+    using CommandContainer::isFlushTaskUsedForImmediate;
+    using CommandContainer::reusableAllocationList;
+};

shared/test/unit_test/command_container/command_container_tests.cpp

@@ -10,6 +10,7 @@
 #include "shared/source/memory_manager/allocations_list.h"
 #include "shared/test/common/fixtures/device_fixture.h"
 #include "shared/test/common/helpers/debug_manager_state_restore.h"
+#include "shared/test/common/mocks/mock_command_container.h"
 #include "shared/test/common/mocks/mock_graphics_allocation.h"
 #include "shared/test/common/mocks/mock_memory_manager.h"
 #include "shared/test/common/test_macros/test.h"
@@ -27,18 +28,17 @@ class CommandContainerTest : public DeviceFixture,
         DeviceFixture::SetUp();
     }
     void TearDown() override {
+        allocList.freeAllGraphicsAllocations(pDevice);
+
         DeviceFixture::TearDown();
         ::testing::Test::TearDown();
     }
+
+    AllocationsList allocList;
 };
 
 struct CommandContainerHeapStateTests : public ::testing::Test {
-    class MyMockCommandContainer : public CommandContainer {
+    MockCommandContainer myCommandContainer;
-      public:
-        using CommandContainer::dirtyHeaps;
-    };
-
-    MyMockCommandContainer myCommandContainer;
 };
 
 TEST_F(CommandContainerHeapStateTests, givenDirtyHeapsWhenSettingStateForAllThenValuesAreCorrect) {
@@ -173,7 +173,6 @@ TEST_F(CommandContainerTest, givenCommandContainerDuringInitWhenAllocateGfxMemor
 }
 
 TEST_F(CommandContainerTest, givenCmdContainerWithAllocsListWhenAllocateAndResetThenCmdBufferAllocIsReused) {
-    AllocationsList allocList;
     auto cmdContainer = std::make_unique<CommandContainer>();
     cmdContainer->initialize(pDevice, &allocList);
     auto &cmdBufferAllocs = cmdContainer->getCmdBufferAllocations();
@@ -203,7 +202,6 @@ TEST_F(CommandContainerTest, givenCmdContainerWithAllocsListWhenAllocateAndReset
     cmdContainer.reset();
     EXPECT_EQ(memoryManager->handleFenceCompletionCalled, 3u);
     EXPECT_FALSE(allocList.peekIsEmpty());
-    allocList.freeAllGraphicsAllocations(pDevice);
 }
 
 TEST_F(CommandContainerTest, givenCommandContainerDuringInitWhenAllocateHeapMemoryFailsThenErrorIsReturned) {
@@ -729,3 +727,48 @@ TEST_F(CommandContainerTest, givenCmdContainerWhenCloseAndAllocateNextCommandBuf
     cmdContainer.closeAndAllocateNextCommandBuffer();
     EXPECT_EQ(cmdContainer.getCmdBufferAllocations().size(), 2u);
 }
+
+TEST_F(CommandContainerTest, givenFlushTaskCmdContainerWithAllocationListWhenNewCmdBufferAllocatedThenOldCmdBufferIsStored) {
+    auto cmdContainer = std::make_unique<MockCommandContainer>();
+    cmdContainer->isFlushTaskUsedForImmediate = true;
+    cmdContainer->initialize(pDevice, &allocList);
+
+    auto &cmdBufferAllocs = cmdContainer->cmdBufferAllocations;
+    auto memoryManager = static_cast<MockMemoryManager *>(pDevice->getMemoryManager());
+    EXPECT_EQ(0u, memoryManager->handleFenceCompletionCalled);
+    EXPECT_EQ(1u, cmdBufferAllocs.size());
+    EXPECT_TRUE(allocList.peekIsEmpty());
+    GraphicsAllocation *oldAllocation = cmdContainer->getCommandStream()->getGraphicsAllocation();
+
+    cmdContainer->allocateNextCommandBuffer();
+    EXPECT_EQ(1u, cmdBufferAllocs.size());
+
+    auto cmdBuffer0 = cmdBufferAllocs[0];
+    EXPECT_EQ(cmdBuffer0, cmdContainer->getCommandStream()->getGraphicsAllocation());
+    EXPECT_NE(cmdBuffer0, oldAllocation);
+
+    EXPECT_EQ(0u, memoryManager->freeGraphicsMemoryCalled);
+    EXPECT_EQ(1u, memoryManager->handleFenceCompletionCalled);
+    EXPECT_FALSE(allocList.peekIsEmpty());
+}
+
+TEST_F(CommandContainerTest, givenFlushTaskCmdContainerWithoutAllocationListWhenNewCmdBufferAllocatedThenOldCmdBufferIsFreed) {
+    auto cmdContainer = std::make_unique<MockCommandContainer>();
+    cmdContainer->isFlushTaskUsedForImmediate = true;
+    cmdContainer->initialize(pDevice, nullptr);
+
+    auto &cmdBufferAllocs = cmdContainer->cmdBufferAllocations;
+    auto memoryManager = static_cast<MockMemoryManager *>(pDevice->getMemoryManager());
+    EXPECT_EQ(0u, memoryManager->handleFenceCompletionCalled);
+    EXPECT_EQ(1u, cmdBufferAllocs.size());
+    GraphicsAllocation *oldAllocation = cmdContainer->getCommandStream()->getGraphicsAllocation();
+
+    cmdContainer->allocateNextCommandBuffer();
+    EXPECT_EQ(1u, cmdBufferAllocs.size());
+
+    auto cmdBuffer0 = cmdBufferAllocs[0];
+    EXPECT_EQ(cmdBuffer0, cmdContainer->getCommandStream()->getGraphicsAllocation());
+    EXPECT_NE(cmdBuffer0, oldAllocation);
+
+    EXPECT_EQ(1u, memoryManager->freeGraphicsMemoryCalled);
+}

shared/test/unit_test/encoders/test_encode_dispatch_kernel.cpp

@@ -15,6 +15,7 @@
 #include "shared/test/common/fixtures/command_container_fixture.h"
 #include "shared/test/common/fixtures/front_window_fixture.h"
 #include "shared/test/common/helpers/debug_manager_state_restore.h"
+#include "shared/test/common/mocks/mock_command_container.h"
 #include "shared/test/common/mocks/mock_device.h"
 #include "shared/test/common/mocks/mock_dispatch_kernel_encoder_interface.h"
 #include "shared/test/common/test_macros/matchers.h"
@@ -461,7 +462,7 @@ HWCMDTEST_F(IGFX_GEN8_CORE, CommandEncodeStatesTest, givenForceBtpPrefetchModeDe
 
     {
         DebugManager.flags.ForceBtpPrefetchMode.set(-1);
-        cmdContainer.reset(new MyMockCommandContainer());
+        cmdContainer.reset(new MockCommandContainer());
         cmdContainer->initialize(pDevice, nullptr);
         bool requiresUncachedMocs = false;
         EncodeDispatchKernelArgs dispatchArgs = createDefaultDispatchKernelArgs(pDevice, dispatchInterface.get(), dims, requiresUncachedMocs);
@@ -492,7 +493,7 @@ HWCMDTEST_F(IGFX_GEN8_CORE, CommandEncodeStatesTest, givenForceBtpPrefetchModeDe
 
     {
         DebugManager.flags.ForceBtpPrefetchMode.set(0);
-        cmdContainer.reset(new MyMockCommandContainer());
+        cmdContainer.reset(new MockCommandContainer());
         cmdContainer->initialize(pDevice, nullptr);
 
         bool requiresUncachedMocs = false;
@@ -519,7 +520,7 @@ HWCMDTEST_F(IGFX_GEN8_CORE, CommandEncodeStatesTest, givenForceBtpPrefetchModeDe
 
     {
         DebugManager.flags.ForceBtpPrefetchMode.set(1);
-        cmdContainer.reset(new MyMockCommandContainer());
+        cmdContainer.reset(new MockCommandContainer());
         cmdContainer->initialize(pDevice, nullptr);
 
         bool requiresUncachedMocs = false;

shared/test/unit_test/encoders/test_encode_dispatch_kernel_xehp_and_later.cpp

@@ -18,6 +18,7 @@
 #include "shared/test/common/fixtures/command_container_fixture.h"
 #include "shared/test/common/helpers/debug_manager_state_restore.h"
 #include "shared/test/common/helpers/variable_backup.h"
+#include "shared/test/common/mocks/mock_command_container.h"
 #include "shared/test/common/mocks/mock_device.h"
 #include "shared/test/common/mocks/mock_dispatch_kernel_encoder_interface.h"
 #include "shared/test/common/test_macros/test.h"
@@ -363,7 +364,7 @@ HWCMDTEST_F(IGFX_XE_HP_CORE, CommandEncodeStatesTest, givenForceBtpPrefetchModeD
 
     {
         DebugManager.flags.ForceBtpPrefetchMode.set(-1);
-        cmdContainer.reset(new MyMockCommandContainer());
+        cmdContainer.reset(new MockCommandContainer());
         cmdContainer->initialize(pDevice, nullptr);
 
         bool requiresUncachedMocs = false;
@@ -394,7 +395,7 @@ HWCMDTEST_F(IGFX_XE_HP_CORE, CommandEncodeStatesTest, givenForceBtpPrefetchModeD
 
     {
         DebugManager.flags.ForceBtpPrefetchMode.set(0);
-        cmdContainer.reset(new MyMockCommandContainer());
+        cmdContainer.reset(new MockCommandContainer());
         cmdContainer->initialize(pDevice, nullptr);
 
         bool requiresUncachedMocs = false;
@@ -416,7 +417,7 @@ HWCMDTEST_F(IGFX_XE_HP_CORE, CommandEncodeStatesTest, givenForceBtpPrefetchModeD
 
     {
         DebugManager.flags.ForceBtpPrefetchMode.set(1);
-        cmdContainer.reset(new MyMockCommandContainer());
+        cmdContainer.reset(new MockCommandContainer());
         cmdContainer->initialize(pDevice, nullptr);
 
         bool requiresUncachedMocs = false;