Move allocation lists to internal allocation storage

Change-Id: I543f1551c8fb161cf99c5870de44afec390415b2
Signed-off-by: Mateusz Jablonski <mateusz.jablonski@intel.com>

runtime/command_stream/command_stream_receiver.cpp

@@ -50,12 +50,8 @@ CommandStreamReceiver::~CommandStreamReceiver() {
     }
     cleanupResources();
 
-    if (!allocationsForReuse.peekIsEmpty()) {
-        internalAllocationStorage->freeAllocationsList(-1, allocationsForReuse);
-    }
-    if (!temporaryAllocations.peekIsEmpty()) {
-        internalAllocationStorage->freeAllocationsList(-1, temporaryAllocations);
-    }
+    internalAllocationStorage->cleanAllocationList(-1, REUSABLE_ALLOCATION);
+    internalAllocationStorage->cleanAllocationList(-1, TEMPORARY_ALLOCATION);
 }
 
 void CommandStreamReceiver::makeResident(GraphicsAllocation &gfxAllocation) {
@@ -110,12 +106,7 @@ void CommandStreamReceiver::waitForTaskCountAndCleanAllocationList(uint32_t requ
         while (*address < requiredTaskCount)
             ;
     }
-
-    auto &allocationList = (allocationType == TEMPORARY_ALLOCATION) ? temporaryAllocations : allocationsForReuse;
-    if (allocationList.peekIsEmpty()) {
-        return;
-    }
-    internalAllocationStorage->freeAllocationsList(requiredTaskCount, allocationList);
+    internalAllocationStorage->cleanAllocationList(requiredTaskCount, allocationType);
 }
 
 MemoryManager *CommandStreamReceiver::getMemoryManager() const {
@@ -347,5 +338,7 @@ bool CommandStreamReceiver::initializeTagAllocation() {
 std::unique_lock<CommandStreamReceiver::MutexType> CommandStreamReceiver::obtainUniqueOwnership() {
     return std::unique_lock<CommandStreamReceiver::MutexType>(this->ownershipMutex);
 }
+AllocationsList &CommandStreamReceiver::getTemporaryAllocations() { return internalAllocationStorage->getTemporaryAllocations(); }
+AllocationsList &CommandStreamReceiver::getAllocationsForReuse() { return internalAllocationStorage->getAllocationsForReuse(); }
 
 } // namespace OCLRT

runtime/command_stream/command_stream_receiver.h

@@ -17,11 +17,11 @@
 #include "runtime/helpers/options.h"
 #include "runtime/indirect_heap/indirect_heap.h"
 #include "runtime/kernel/grf_config.h"
-#include "runtime/memory_manager/allocations_list.h"
 #include <cstddef>
 #include <cstdint>
 
 namespace OCLRT {
+class AllocationsList;
 class Device;
 class EventBuilder;
 class ExecutionEnvironment;
@@ -148,8 +148,8 @@ class CommandStreamReceiver {
     size_t defaultSshSize;
 
     void setDeviceIndex(uint32_t deviceIndex) { this->deviceIndex = deviceIndex; }
-    AllocationsList &getTemporaryAllocations() { return temporaryAllocations; }
-    AllocationsList &getAllocationsForReuse() { return allocationsForReuse; }
+    AllocationsList &getTemporaryAllocations();
+    AllocationsList &getAllocationsForReuse();
     InternalAllocationStorage *getInternalAllocationStorage() const { return internalAllocationStorage.get(); }
 
   protected:
@@ -214,9 +214,6 @@ class CommandStreamReceiver {
     ExecutionEnvironment &executionEnvironment;
     uint32_t deviceIndex = 0u;
     std::unique_ptr<InternalAllocationStorage> internalAllocationStorage;
-
-    AllocationsList temporaryAllocations;
-    AllocationsList allocationsForReuse;
 };
 
 typedef CommandStreamReceiver *(*CommandStreamReceiverCreateFunc)(const HardwareInfo &hwInfoIn, bool withAubDump, ExecutionEnvironment &executionEnvironment);

runtime/memory_manager/internal_allocation_storage.cpp

@@ -30,14 +30,14 @@ void InternalAllocationStorage::storeAllocationWithTaskCount(std::unique_ptr<Gra
             return;
         }
     }
-    auto &allocationsList = (allocationUsage == TEMPORARY_ALLOCATION) ? commandStreamReceiver.getTemporaryAllocations() : commandStreamReceiver.getAllocationsForReuse();
+    auto &allocationsList = (allocationUsage == TEMPORARY_ALLOCATION) ? temporaryAllocations : allocationsForReuse;
     gfxAllocation->taskCount = taskCount;
     allocationsList.pushTailOne(*gfxAllocation.release());
 }
 
 void InternalAllocationStorage::cleanAllocationList(uint32_t waitTaskCount, uint32_t allocationUsage) {
     std::lock_guard<decltype(mutex)> lock(mutex);
-    freeAllocationsList(waitTaskCount, (allocationUsage == TEMPORARY_ALLOCATION) ? commandStreamReceiver.getTemporaryAllocations() : commandStreamReceiver.getAllocationsForReuse());
+    freeAllocationsList(waitTaskCount, (allocationUsage == TEMPORARY_ALLOCATION) ? temporaryAllocations : allocationsForReuse);
 }
 
 void InternalAllocationStorage::freeAllocationsList(uint32_t waitTaskCount, AllocationsList &allocationsList) {
@@ -63,7 +63,7 @@ void InternalAllocationStorage::freeAllocationsList(uint32_t waitTaskCount, Allo
 
 std::unique_ptr<GraphicsAllocation> InternalAllocationStorage::obtainReusableAllocation(size_t requiredSize, bool internalAllocation) {
     std::lock_guard<decltype(mutex)> lock(mutex);
-    auto allocation = commandStreamReceiver.getAllocationsForReuse().detachAllocation(requiredSize, commandStreamReceiver.getTagAddress(), internalAllocation);
+    auto allocation = allocationsForReuse.detachAllocation(requiredSize, commandStreamReceiver.getTagAddress(), internalAllocation);
     return allocation;
 }
 

runtime/memory_manager/internal_allocation_storage.h

@@ -6,12 +6,12 @@
  */
 
 #pragma once
+#include "runtime/memory_manager/allocations_list.h"
 #include <cstdint>
 #include <memory>
 #include <mutex>
 
 namespace OCLRT {
-class AllocationsList;
 class CommandStreamReceiver;
 class GraphicsAllocation;
 
@@ -20,13 +20,18 @@ class InternalAllocationStorage {
     MOCKABLE_VIRTUAL ~InternalAllocationStorage() = default;
     InternalAllocationStorage(CommandStreamReceiver &commandStreamReceiver);
     MOCKABLE_VIRTUAL void cleanAllocationList(uint32_t waitTaskCount, uint32_t allocationUsage);
-    void freeAllocationsList(uint32_t waitTaskCount, AllocationsList &allocationsList);
     void storeAllocation(std::unique_ptr<GraphicsAllocation> gfxAllocation, uint32_t allocationUsage);
     void storeAllocationWithTaskCount(std::unique_ptr<GraphicsAllocation> gfxAllocation, uint32_t allocationUsage, uint32_t taskCount);
     std::unique_ptr<GraphicsAllocation> obtainReusableAllocation(size_t requiredSize, bool isInternalAllocationRequired);
+    AllocationsList &getTemporaryAllocations() { return temporaryAllocations; }
+    AllocationsList &getAllocationsForReuse() { return allocationsForReuse; }
 
   protected:
+    void freeAllocationsList(uint32_t waitTaskCount, AllocationsList &allocationsList);
     std::recursive_mutex mutex;
     CommandStreamReceiver &commandStreamReceiver;
+
+    AllocationsList temporaryAllocations;
+    AllocationsList allocationsForReuse;
 };
 } // namespace OCLRT

unit_tests/aub_tests/command_queue/enqueue_write_copy_read_buffer_aub_tests.cpp

@@ -5,6 +5,7 @@
  *
  */
 
+#include "runtime/memory_manager/allocations_list.h"
 #include "unit_tests/aub_tests/command_queue/enqueue_write_copy_read_buffer_aub_tests.h"
 
 #include "test.h"

unit_tests/command_queue/enqueue_fill_buffer_tests.cpp

@@ -13,6 +13,7 @@
 #include "runtime/helpers/ptr_math.h"
 #include "runtime/helpers/aligned_memory.h"
 #include "runtime/helpers/dispatch_info.h"
+#include "runtime/memory_manager/allocations_list.h"
 #include "unit_tests/command_queue/enqueue_fixture.h"
 #include "unit_tests/command_queue/enqueue_fill_buffer_fixture.h"
 #include "unit_tests/gen_common/gen_commands_common_validation.h"

unit_tests/command_queue/enqueue_kernel_tests.cpp

@@ -10,6 +10,7 @@
 #include "runtime/command_queue/command_queue_hw.h"
 #include "reg_configs_common.h"
 #include "runtime/helpers/preamble.h"
+#include "runtime/memory_manager/allocations_list.h"
 #include "runtime/memory_manager/graphics_allocation.h"
 #include "runtime/memory_manager/memory_constants.h"
 #include "runtime/utilities/tag_allocator.h"

unit_tests/command_queue/enqueue_read_image_tests.cpp

@@ -6,6 +6,7 @@
  */
 
 #include "runtime/built_ins/builtins_dispatch_builder.h"
+#include "runtime/memory_manager/allocations_list.h"
 #include "reg_configs_common.h"
 #include "unit_tests/command_queue/enqueue_read_image_fixture.h"
 #include "unit_tests/gen_common/gen_commands_common_validation.h"

unit_tests/command_queue/enqueue_svm_tests.cpp

@@ -10,6 +10,7 @@
 #include "unit_tests/fixtures/device_fixture.h"
 #include "unit_tests/fixtures/buffer_fixture.h"
 #include "unit_tests/helpers/debug_manager_state_restore.h"
+#include "runtime/memory_manager/allocations_list.h"
 #include "runtime/memory_manager/svm_memory_manager.h"
 #include "unit_tests/mocks/mock_context.h"
 #include "unit_tests/mocks/mock_kernel.h"

unit_tests/command_queue/enqueue_write_image_tests.cpp

@@ -7,6 +7,7 @@
 
 #include "runtime/built_ins/builtins_dispatch_builder.h"
 #include "reg_configs_common.h"
+#include "runtime/memory_manager/allocations_list.h"
 #include "runtime/memory_manager/memory_manager.h"
 #include "unit_tests/command_queue/enqueue_write_image_fixture.h"
 #include "unit_tests/gen_common/gen_commands_common_validation.h"

unit_tests/kernel/kernel_tests.cpp

@@ -11,6 +11,7 @@
 #include "runtime/helpers/surface_formats.h"
 #include "runtime/kernel/kernel.h"
 #include "runtime/mem_obj/image.h"
+#include "runtime/memory_manager/allocations_list.h"
 #include "runtime/memory_manager/os_agnostic_memory_manager.h"
 #include "runtime/os_interface/debug_settings_manager.h"
 #include "unit_tests/fixtures/device_fixture.h"

unit_tests/mem_obj/mem_obj_destruction_tests.cpp

@@ -6,6 +6,7 @@
  */
 
 #include "runtime/mem_obj/mem_obj.h"
+#include "runtime/memory_manager/allocations_list.h"
 #include "unit_tests/mocks/mock_context.h"
 #include "unit_tests/mocks/mock_device.h"
 #include "unit_tests/mocks/mock_memory_manager.h"
@@ -103,13 +104,14 @@ TEST_P(MemObjAsyncDestructionTest, givenMemObjWithDestructableAllocationWhenAsyn
     } else {
         makeMemObjNotReady();
     }
-    EXPECT_TRUE(memoryManager->isAllocationListEmpty());
+    auto &allocationList = memoryManager->getCommandStreamReceiver(0)->getTemporaryAllocations();
+    EXPECT_TRUE(allocationList.peekIsEmpty());
 
     delete memObj;
 
-    EXPECT_EQ(!expectedDeferration, memoryManager->isAllocationListEmpty());
+    EXPECT_EQ(!expectedDeferration, allocationList.peekIsEmpty());
     if (expectedDeferration) {
-        EXPECT_EQ(allocation, memoryManager->peekAllocationListHead());
+        EXPECT_EQ(allocation, allocationList.peekHead());
     }
 }
 
@@ -262,7 +264,8 @@ HWTEST_P(MemObjSyncDestructionTest, givenMemObjWithDestructableAllocationWhenAsy
         .WillByDefault(::testing::Invoke(waitForCompletionWithTimeoutMock));
 
     delete memObj;
-    EXPECT_TRUE(memoryManager->isAllocationListEmpty());
+    auto &allocationList = memoryManager->getCommandStreamReceiver(0)->getTemporaryAllocations();
+    EXPECT_TRUE(allocationList.peekIsEmpty());
 }
 
 INSTANTIATE_TEST_CASE_P(

unit_tests/mem_obj/mem_obj_tests.cpp

@@ -10,6 +10,7 @@
 #include "runtime/device/device.h"
 #include "runtime/gmm_helper/gmm.h"
 #include "runtime/helpers/properties_helper.h"
+#include "runtime/memory_manager/allocations_list.h"
 #include "unit_tests/mocks/mock_context.h"
 #include "unit_tests/mocks/mock_deferred_deleter.h"
 #include "unit_tests/mocks/mock_graphics_allocation.h"
@@ -165,11 +166,11 @@ TEST(MemObj, givenNotReadyGraphicsAllocationWhenMemObjDestroysAllocationAsyncThe
     *(memoryManager.getCommandStreamReceiver(0)->getTagAddress()) = 1;
     MemObj memObj(&context, CL_MEM_OBJECT_BUFFER, CL_MEM_COPY_HOST_PTR,
                   MemoryConstants::pageSize, nullptr, nullptr, nullptr, true, false, false);
-
-    EXPECT_TRUE(memoryManager.isAllocationListEmpty());
+    auto &allocationList = memoryManager.getCommandStreamReceiver(0)->getTemporaryAllocations();
+    EXPECT_TRUE(allocationList.peekIsEmpty());
     memObj.destroyGraphicsAllocation(allocation, true);
 
-    EXPECT_FALSE(memoryManager.isAllocationListEmpty());
+    EXPECT_FALSE(allocationList.peekIsEmpty());
 }
 
 TEST(MemObj, givenReadyGraphicsAllocationWhenMemObjDestroysAllocationAsyncThenAllocationIsNotAddedToMemoryManagerAllocationList) {
@@ -184,10 +185,11 @@ TEST(MemObj, givenReadyGraphicsAllocationWhenMemObjDestroysAllocationAsyncThenAl
     MemObj memObj(&context, CL_MEM_OBJECT_BUFFER, CL_MEM_COPY_HOST_PTR,
                   MemoryConstants::pageSize, nullptr, nullptr, nullptr, true, false, false);
 
-    EXPECT_TRUE(memoryManager.isAllocationListEmpty());
+    auto &allocationList = memoryManager.getCommandStreamReceiver(0)->getTemporaryAllocations();
+    EXPECT_TRUE(allocationList.peekIsEmpty());
     memObj.destroyGraphicsAllocation(allocation, true);
 
-    EXPECT_TRUE(memoryManager.isAllocationListEmpty());
+    EXPECT_TRUE(allocationList.peekIsEmpty());
 }
 
 TEST(MemObj, givenNotUsedGraphicsAllocationWhenMemObjDestroysAllocationAsyncThenAllocationIsNotAddedToMemoryManagerAllocationList) {
@@ -201,10 +203,11 @@ TEST(MemObj, givenNotUsedGraphicsAllocationWhenMemObjDestroysAllocationAsyncThen
     MemObj memObj(&context, CL_MEM_OBJECT_BUFFER, CL_MEM_COPY_HOST_PTR,
                   MemoryConstants::pageSize, nullptr, nullptr, nullptr, true, false, false);
 
-    EXPECT_TRUE(memoryManager.isAllocationListEmpty());
+    auto &allocationList = memoryManager.getCommandStreamReceiver(0)->getTemporaryAllocations();
+    EXPECT_TRUE(allocationList.peekIsEmpty());
     memObj.destroyGraphicsAllocation(allocation, true);
 
-    EXPECT_TRUE(memoryManager.isAllocationListEmpty());
+    EXPECT_TRUE(allocationList.peekIsEmpty());
 }
 
 TEST(MemObj, givenMemoryManagerWithoutDeviceWhenMemObjDestroysAllocationAsyncThenAllocationIsNotAddedToMemoryManagerAllocationList) {
@@ -218,10 +221,11 @@ TEST(MemObj, givenMemoryManagerWithoutDeviceWhenMemObjDestroysAllocationAsyncThe
     MemObj memObj(&context, CL_MEM_OBJECT_BUFFER, CL_MEM_COPY_HOST_PTR,
                   MemoryConstants::pageSize, nullptr, nullptr, nullptr, true, false, false);
 
-    EXPECT_TRUE(memoryManager.isAllocationListEmpty());
+    auto &allocationList = memoryManager.getCommandStreamReceiver(0)->getTemporaryAllocations();
+    EXPECT_TRUE(allocationList.peekIsEmpty());
     memObj.destroyGraphicsAllocation(allocation, true);
 
-    EXPECT_TRUE(memoryManager.isAllocationListEmpty());
+    EXPECT_TRUE(allocationList.peekIsEmpty());
 }
 
 TEST(MemObj, givenMemObjWhenItDoesntHaveGraphicsAllocationThenWaitForCsrCompletionDoesntCrash) {

unit_tests/mocks/mock_memory_manager.cpp

@@ -34,14 +34,6 @@ GraphicsAllocation *MockMemoryManager::allocateGraphicsMemoryForImage(ImageInfo
     return allocation;
 }
 
-bool MockMemoryManager::isAllocationListEmpty() {
-    return getCommandStreamReceiver(0)->getTemporaryAllocations().peekIsEmpty();
-}
-
-GraphicsAllocation *MockMemoryManager::peekAllocationListHead() {
-    return getCommandStreamReceiver(0)->getTemporaryAllocations().peekHead();
-}
-
 GraphicsAllocation *MockMemoryManager::allocateGraphicsMemory64kb(size_t size, size_t alignment, bool forcePin, bool preferRenderCompressed) {
     allocation64kbPageCreated = true;
     preferRenderCompressedFlagPassed = preferRenderCompressed;

unit_tests/mocks/mock_memory_manager.h

@@ -36,8 +36,6 @@ class MockMemoryManager : public OsAgnosticMemoryManager {
     void overrideAsyncDeleterFlag(bool newValue);
     GraphicsAllocation *allocateGraphicsMemoryForImage(ImageInfo &imgInfo, Gmm *gmm) override;
     int redundancyRatio = 1;
-    bool isAllocationListEmpty();
-    GraphicsAllocation *peekAllocationListHead();
 
     GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override;
     GraphicsAllocation *allocateGraphicsMemory(size_t size, size_t alignment, bool forcePin, bool uncacheable) override;

unit_tests/program/program_data_tests.cpp

@@ -5,9 +5,8 @@
  *
  */
 
-#include "../mocks/mock_program.h"
-#include "../mocks/mock_csr.h"
 #include "runtime/helpers/string.h"
+#include "runtime/memory_manager/allocations_list.h"
 #include "runtime/memory_manager/graphics_allocation.h"
 #include "runtime/platform/platform.h"
 #include "runtime/program/program.h"
@@ -15,6 +14,8 @@
 #include "unit_tests/gen_common/test.h"
 #include "unit_tests/program/program_with_source.h"
 #include "unit_tests/mocks/mock_buffer.h"
+#include "unit_tests/mocks/mock_program.h"
+#include "unit_tests/mocks/mock_csr.h"
 
 using namespace OCLRT;
 

unit_tests/program/program_tests.cpp

@@ -18,6 +18,7 @@
 #include "runtime/helpers/string.h"
 #include "runtime/indirect_heap/indirect_heap.h"
 #include "runtime/kernel/kernel.h"
+#include "runtime/memory_manager/allocations_list.h"
 #include "runtime/memory_manager/graphics_allocation.h"
 #include "runtime/memory_manager/surface.h"
 #include "runtime/program/create.inl"
@@ -2926,4 +2927,4 @@ TEST_F(ProgramTests, givenProgramWhenUnknownInternalOptionsArePassedThenTheyAreN
     pProgram.extractInternalOptionsForward(buildOptions);
     EXPECT_EQ(0u, pProgram.getInternalOptions().length());
     EXPECT_TRUE(buildOptions == internalOption);
-}
+}