diff --git a/runtime/command_queue/command_queue.cpp b/runtime/command_queue/command_queue.cpp index 7f62785a97..9c861a49de 100644 --- a/runtime/command_queue/command_queue.cpp +++ b/runtime/command_queue/command_queue.cpp @@ -26,7 +26,7 @@ #include "runtime/mem_obj/buffer.h" #include "runtime/mem_obj/image.h" #include "runtime/helpers/surface_formats.h" -#include "runtime/memory_manager/memory_manager.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/helpers/string.h" #include "CL/cl_ext.h" #include "runtime/utilities/api_intercept.h" @@ -91,11 +91,10 @@ CommandQueue::~CommandQueue() { } if (device) { - auto memoryManager = device->getMemoryManager(); - DEBUG_BREAK_IF(nullptr == memoryManager); + auto storageForAllocation = device->getCommandStreamReceiver().getInternalAllocationStorage(); if (commandStream && commandStream->getGraphicsAllocation()) { - memoryManager->storeAllocation(std::unique_ptr(commandStream->getGraphicsAllocation()), REUSABLE_ALLOCATION); + storageForAllocation->storeAllocation(std::unique_ptr(commandStream->getGraphicsAllocation()), REUSABLE_ALLOCATION); commandStream->replaceGraphicsAllocation(nullptr); } delete commandStream; @@ -198,6 +197,7 @@ uint32_t CommandQueue::getTaskLevelFromWaitList(uint32_t taskLevel, LinearStream &CommandQueue::getCS(size_t minRequiredSize) { DEBUG_BREAK_IF(nullptr == device); auto &commandStreamReceiver = device->getCommandStreamReceiver(); + auto storageForAllocation = commandStreamReceiver.getInternalAllocationStorage(); auto memoryManager = commandStreamReceiver.getMemoryManager(); DEBUG_BREAK_IF(nullptr == memoryManager); @@ -226,7 +226,7 @@ LinearStream &CommandQueue::getCS(size_t minRequiredSize) { auto oldAllocation = commandStream->getGraphicsAllocation(); if (oldAllocation) { - memoryManager->storeAllocation(std::unique_ptr(oldAllocation), REUSABLE_ALLOCATION); + storageForAllocation->storeAllocation(std::unique_ptr(oldAllocation), REUSABLE_ALLOCATION); } commandStream->replaceBuffer(allocation->getUnderlyingBuffer(), minRequiredSize - CSRequirements::minCommandQueueCommandStreamSize); commandStream->replaceGraphicsAllocation(allocation); diff --git a/runtime/command_queue/enqueue_common.h b/runtime/command_queue/enqueue_common.h index 5fcf099ff5..e194786e18 100644 --- a/runtime/command_queue/enqueue_common.h +++ b/runtime/command_queue/enqueue_common.h @@ -23,6 +23,7 @@ #include "runtime/helpers/task_information.h" #include "runtime/mem_obj/buffer.h" #include "runtime/mem_obj/image.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/memory_manager.h" #include "runtime/memory_manager/surface.h" #include "runtime/program/printf_handler.h" @@ -740,7 +741,8 @@ bool CommandQueueHw::createAllocationForHostSurface(HostPtrSurface &s } allocation->taskCount = Event::eventNotReady; surface.setAllocation(allocation); - memoryManager->storeAllocation(std::unique_ptr(allocation), TEMPORARY_ALLOCATION); + auto storageForAllocation = device->getCommandStreamReceiver().getInternalAllocationStorage(); + storageForAllocation->storeAllocation(std::unique_ptr(allocation), TEMPORARY_ALLOCATION); return true; } diff --git a/runtime/command_queue/enqueue_fill_buffer.h b/runtime/command_queue/enqueue_fill_buffer.h index baba4c318d..8678609c2d 100644 --- a/runtime/command_queue/enqueue_fill_buffer.h +++ b/runtime/command_queue/enqueue_fill_buffer.h @@ -12,6 +12,7 @@ #include "runtime/helpers/kernel_commands.h" #include "runtime/mem_obj/buffer.h" #include "runtime/memory_manager/surface.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/memory_manager.h" #include "runtime/built_ins/built_ins.h" #include @@ -72,7 +73,8 @@ cl_int CommandQueueHw::enqueueFillBuffer( eventWaitList, event); - memoryManager->storeAllocation(std::unique_ptr(patternAllocation), TEMPORARY_ALLOCATION, taskCount); + auto storageForAllocation = device->getCommandStreamReceiver().getInternalAllocationStorage(); + storageForAllocation->storeAllocationWithTaskCount(std::unique_ptr(patternAllocation), TEMPORARY_ALLOCATION, taskCount); return CL_SUCCESS; } diff --git a/runtime/command_queue/enqueue_svm.h b/runtime/command_queue/enqueue_svm.h index 6cfcb8e27a..378acdf680 100644 --- a/runtime/command_queue/enqueue_svm.h +++ b/runtime/command_queue/enqueue_svm.h @@ -271,7 +271,8 @@ cl_int CommandQueueHw::enqueueSVMMemFill(void *svmPtr, eventWaitList, event); - memoryManager->storeAllocation(std::unique_ptr(patternAllocation), REUSABLE_ALLOCATION, taskCount); + auto storageForAllocation = device->getCommandStreamReceiver().getInternalAllocationStorage(); + storageForAllocation->storeAllocationWithTaskCount(std::unique_ptr(patternAllocation), REUSABLE_ALLOCATION, taskCount); return CL_SUCCESS; } diff --git a/runtime/command_queue/hardware_interface.inl b/runtime/command_queue/hardware_interface.inl index 48319c973b..df57c5243c 100644 --- a/runtime/command_queue/hardware_interface.inl +++ b/runtime/command_queue/hardware_interface.inl @@ -67,7 +67,7 @@ void HardwareInterface::dispatchWalker( using UniqueIH = std::unique_ptr; *blockedCommandsData = new KernelOperation(std::unique_ptr(commandStream), UniqueIH(dsh), UniqueIH(ioh), - UniqueIH(ssh), *commandQueue.getDevice().getMemoryManager()); + UniqueIH(ssh), *commandQueue.getDevice().getCommandStreamReceiver().getInternalAllocationStorage()); if (parentKernel) { (*blockedCommandsData)->doNotFreeISH = true; } diff --git a/runtime/command_stream/command_stream_receiver.cpp b/runtime/command_stream/command_stream_receiver.cpp index 27a5ed9d3a..7a6329cde0 100644 --- a/runtime/command_stream/command_stream_receiver.cpp +++ b/runtime/command_stream/command_stream_receiver.cpp @@ -43,7 +43,7 @@ CommandStreamReceiver::~CommandStreamReceiver() { if (indirectHeap[i] != nullptr) { auto allocation = indirectHeap[i]->getGraphicsAllocation(); if (allocation != nullptr) { - getMemoryManager()->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); + internalAllocationStorage->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); } delete indirectHeap[i]; } @@ -143,7 +143,7 @@ LinearStream &CommandStreamReceiver::getCS(size_t minRequiredSize) { //pass current allocation to reusable list if (commandStream.getCpuBase()) { - getMemoryManager()->storeAllocation(std::unique_ptr(commandStream.getGraphicsAllocation()), REUSABLE_ALLOCATION); + internalAllocationStorage->storeAllocation(std::unique_ptr(commandStream.getGraphicsAllocation()), REUSABLE_ALLOCATION); } commandStream.replaceBuffer(allocation->getUnderlyingBuffer(), minRequiredSize - sizeForSubmission); @@ -259,7 +259,7 @@ IndirectHeap &CommandStreamReceiver::getIndirectHeap(IndirectHeap::Type heapType heapMemory = heap->getGraphicsAllocation(); if (heap && heap->getAvailableSpace() < minRequiredSize && heapMemory) { - getMemoryManager()->storeAllocation(std::unique_ptr(heapMemory), REUSABLE_ALLOCATION); + internalAllocationStorage->storeAllocation(std::unique_ptr(heapMemory), REUSABLE_ALLOCATION); heapMemory = nullptr; } @@ -322,7 +322,7 @@ void CommandStreamReceiver::releaseIndirectHeap(IndirectHeap::Type heapType) { if (heap) { auto heapMemory = heap->getGraphicsAllocation(); if (heapMemory != nullptr) - getMemoryManager()->storeAllocation(std::unique_ptr(heapMemory), REUSABLE_ALLOCATION); + internalAllocationStorage->storeAllocation(std::unique_ptr(heapMemory), REUSABLE_ALLOCATION); heap->replaceBuffer(nullptr, 0); heap->replaceGraphicsAllocation(nullptr); } diff --git a/runtime/command_stream/command_stream_receiver_hw.inl b/runtime/command_stream/command_stream_receiver_hw.inl index f7ddebb79a..0b22cf64eb 100644 --- a/runtime/command_stream/command_stream_receiver_hw.inl +++ b/runtime/command_stream/command_stream_receiver_hw.inl @@ -18,6 +18,7 @@ #include "runtime/helpers/state_base_address.h" #include "runtime/helpers/options.h" #include "runtime/indirect_heap/indirect_heap.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/memory_manager.h" #include "runtime/os_interface/debug_settings_manager.h" #include "runtime/command_stream/preemption.h" @@ -247,7 +248,7 @@ CompletionStamp CommandStreamReceiverHw::flushTask( if (requiredScratchSize && (!scratchAllocation || scratchAllocation->getUnderlyingBufferSize() < requiredScratchSizeInBytes)) { if (scratchAllocation) { scratchAllocation->taskCount = this->taskCount; - getMemoryManager()->storeAllocation(std::unique_ptr(scratchAllocation), TEMPORARY_ALLOCATION); + internalAllocationStorage->storeAllocation(std::unique_ptr(scratchAllocation), TEMPORARY_ALLOCATION); } createScratchSpaceAllocation(requiredScratchSizeInBytes); overrideMediaVFEStateDirty(true); diff --git a/runtime/command_stream/experimental_command_buffer.cpp b/runtime/command_stream/experimental_command_buffer.cpp index a0b9610eaf..e347a833e0 100644 --- a/runtime/command_stream/experimental_command_buffer.cpp +++ b/runtime/command_stream/experimental_command_buffer.cpp @@ -8,6 +8,7 @@ #include "runtime/command_stream/command_stream_receiver.h" #include "runtime/command_stream/experimental_command_buffer.h" #include "runtime/command_stream/linear_stream.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/memory_constants.h" #include "runtime/memory_manager/memory_manager.h" #include @@ -66,7 +67,7 @@ void ExperimentalCommandBuffer::getCS(size_t minRequiredSize) { // Deallocate the old block, if not null auto oldAllocation = currentStream->getGraphicsAllocation(); if (oldAllocation) { - memoryManager->storeAllocation(std::unique_ptr(oldAllocation), REUSABLE_ALLOCATION); + commandStreamReceiver->getInternalAllocationStorage()->storeAllocation(std::unique_ptr(oldAllocation), REUSABLE_ALLOCATION); } currentStream->replaceBuffer(allocation->getUnderlyingBuffer(), minRequiredSize - CSRequirements::minCommandQueueCommandStreamSize); currentStream->replaceGraphicsAllocation(allocation); diff --git a/runtime/helpers/task_information.cpp b/runtime/helpers/task_information.cpp index 9e8397d941..d4a9a378c9 100644 --- a/runtime/helpers/task_information.cpp +++ b/runtime/helpers/task_information.cpp @@ -17,19 +17,19 @@ #include "runtime/helpers/string.h" #include "runtime/helpers/task_information.h" #include "runtime/mem_obj/mem_obj.h" -#include "runtime/memory_manager/memory_manager.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/surface.h" namespace OCLRT { KernelOperation::~KernelOperation() { - memoryManager.storeAllocation(std::unique_ptr(dsh->getGraphicsAllocation()), REUSABLE_ALLOCATION); + storageForAllocations.storeAllocation(std::unique_ptr(dsh->getGraphicsAllocation()), REUSABLE_ALLOCATION); if (ioh.get() == dsh.get()) { ioh.release(); } if (ioh) { - memoryManager.storeAllocation(std::unique_ptr(ioh->getGraphicsAllocation()), REUSABLE_ALLOCATION); + storageForAllocations.storeAllocation(std::unique_ptr(ioh->getGraphicsAllocation()), REUSABLE_ALLOCATION); } - memoryManager.storeAllocation(std::unique_ptr(ssh->getGraphicsAllocation()), REUSABLE_ALLOCATION); + storageForAllocations.storeAllocation(std::unique_ptr(ssh->getGraphicsAllocation()), REUSABLE_ALLOCATION); alignedFree(commandStream->getCpuBase()); } diff --git a/runtime/helpers/task_information.h b/runtime/helpers/task_information.h index 98864740cb..abde814543 100644 --- a/runtime/helpers/task_information.h +++ b/runtime/helpers/task_information.h @@ -19,12 +19,12 @@ namespace OCLRT { class CommandQueue; class CommandStreamReceiver; +class InternalAllocationStorage; class Kernel; class MemObj; class Surface; class PrintfHandler; struct HwTimeStamps; -class MemoryManager; class TimestampPacketContainer; enum MapOperationType { @@ -65,10 +65,10 @@ class CommandMapUnmap : public Command { struct KernelOperation { KernelOperation(std::unique_ptr commandStream, std::unique_ptr dsh, std::unique_ptr ioh, std::unique_ptr ssh, - MemoryManager &memoryManager) + InternalAllocationStorage &storageForAllocations) : commandStream(std::move(commandStream)), dsh(std::move(dsh)), ioh(std::move(ioh)), ssh(std::move(ssh)), - surfaceStateHeapSizeEM(0), doNotFreeISH(false), memoryManager(memoryManager) { + surfaceStateHeapSizeEM(0), doNotFreeISH(false), storageForAllocations(storageForAllocations) { } ~KernelOperation(); @@ -80,7 +80,7 @@ struct KernelOperation { size_t surfaceStateHeapSizeEM; bool doNotFreeISH; - MemoryManager &memoryManager; + InternalAllocationStorage &storageForAllocations; }; class CommandComputeKernel : public Command { diff --git a/runtime/mem_obj/mem_obj.cpp b/runtime/mem_obj/mem_obj.cpp index 08e2c2d292..c2a5675b01 100644 --- a/runtime/mem_obj/mem_obj.cpp +++ b/runtime/mem_obj/mem_obj.cpp @@ -10,6 +10,7 @@ #include "runtime/device/device.h" #include "runtime/mem_obj/mem_obj.h" #include "runtime/memory_manager/deferred_deleter.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/memory_manager.h" #include "runtime/gmm_helper/gmm.h" #include "runtime/helpers/aligned_memory.h" @@ -294,9 +295,11 @@ void MemObj::waitForCsrCompletion() { void MemObj::destroyGraphicsAllocation(GraphicsAllocation *allocation, bool asyncDestroy) { if (asyncDestroy && allocation->taskCount != ObjectNotUsed) { - auto currentTag = *memoryManager->getCommandStreamReceiver(0)->getTagAddress(); + auto commandStreamReceiver = memoryManager->getCommandStreamReceiver(0); + auto currentTag = *commandStreamReceiver->getTagAddress(); if (currentTag < allocation->taskCount) { - memoryManager->storeAllocation(std::unique_ptr(allocation), TEMPORARY_ALLOCATION); + auto storageForAllocation = commandStreamReceiver->getInternalAllocationStorage(); + storageForAllocation->storeAllocation(std::unique_ptr(allocation), TEMPORARY_ALLOCATION); return; } } diff --git a/runtime/memory_manager/graphics_allocation.h b/runtime/memory_manager/graphics_allocation.h index 2973f5b129..1441028d30 100644 --- a/runtime/memory_manager/graphics_allocation.h +++ b/runtime/memory_manager/graphics_allocation.h @@ -118,7 +118,6 @@ class GraphicsAllocation : public IDNode { bool isCoherent() { return coherent; }; void setCoherent(bool coherentIn) { this->coherent = coherentIn; }; - void setGpuAddress(uint64_t graphicsAddress) { this->gpuAddress = graphicsAddress; } void setSize(size_t size) { this->size = size; } osHandle peekSharedHandle() { return sharedHandle; } diff --git a/runtime/memory_manager/internal_allocation_storage.cpp b/runtime/memory_manager/internal_allocation_storage.cpp index 507f288204..42412ce0bf 100644 --- a/runtime/memory_manager/internal_allocation_storage.cpp +++ b/runtime/memory_manager/internal_allocation_storage.cpp @@ -24,8 +24,8 @@ void InternalAllocationStorage::storeAllocation(std::unique_ptr gfxAllocation, uint32_t allocationUsage, uint32_t taskCount) { std::lock_guard lock(mutex); - if (DebugManager.flags.DisableResourceRecycling.get()) { - if (allocationUsage == REUSABLE_ALLOCATION) { + if (allocationUsage == REUSABLE_ALLOCATION) { + if (DebugManager.flags.DisableResourceRecycling.get()) { commandStreamReceiver.getMemoryManager()->freeGraphicsMemory(gfxAllocation.release()); return; } @@ -35,7 +35,7 @@ void InternalAllocationStorage::storeAllocationWithTaskCount(std::unique_ptr lock(mutex); freeAllocationsList(waitTaskCount, (allocationUsage == TEMPORARY_ALLOCATION) ? commandStreamReceiver.getTemporaryAllocations() : commandStreamReceiver.getAllocationsForReuse()); } diff --git a/runtime/memory_manager/internal_allocation_storage.h b/runtime/memory_manager/internal_allocation_storage.h index 561cf68080..5438f6b5da 100644 --- a/runtime/memory_manager/internal_allocation_storage.h +++ b/runtime/memory_manager/internal_allocation_storage.h @@ -18,7 +18,7 @@ class GraphicsAllocation; class InternalAllocationStorage { public: InternalAllocationStorage(CommandStreamReceiver &commandStreamReceiver); - void cleanAllocationsList(uint32_t waitTaskCount, uint32_t allocationUsage); + void cleanAllocationList(uint32_t waitTaskCount, uint32_t allocationUsage); void freeAllocationsList(uint32_t waitTaskCount, AllocationsList &allocationsList); void storeAllocation(std::unique_ptr gfxAllocation, uint32_t allocationUsage); void storeAllocationWithTaskCount(std::unique_ptr gfxAllocation, uint32_t allocationUsage, uint32_t taskCount); diff --git a/runtime/memory_manager/memory_manager.cpp b/runtime/memory_manager/memory_manager.cpp index e64de55eab..2e8f5d7a20 100644 --- a/runtime/memory_manager/memory_manager.cpp +++ b/runtime/memory_manager/memory_manager.cpp @@ -148,14 +148,6 @@ void MemoryManager::freeSystemMemory(void *ptr) { ::alignedFree(ptr); } -void MemoryManager::storeAllocation(std::unique_ptr gfxAllocation, uint32_t allocationUsage) { - getCommandStreamReceiver(0)->getInternalAllocationStorage()->storeAllocation(std::move(gfxAllocation), allocationUsage); -} - -void MemoryManager::storeAllocation(std::unique_ptr gfxAllocation, uint32_t allocationUsage, uint32_t taskCount) { - getCommandStreamReceiver(0)->getInternalAllocationStorage()->storeAllocationWithTaskCount(std::move(gfxAllocation), allocationUsage, taskCount); -} - std::unique_ptr MemoryManager::obtainReusableAllocation(size_t requiredSize, bool internalAllocation) { return getCommandStreamReceiver(0)->getInternalAllocationStorage()->obtainReusableAllocation(requiredSize, internalAllocation); } @@ -184,7 +176,7 @@ void MemoryManager::applyCommonCleanup() { } bool MemoryManager::cleanAllocationList(uint32_t waitTaskCount, uint32_t allocationUsage) { - getCommandStreamReceiver(0)->getInternalAllocationStorage()->cleanAllocationsList(waitTaskCount, allocationUsage); + getCommandStreamReceiver(0)->getInternalAllocationStorage()->cleanAllocationList(waitTaskCount, allocationUsage); return false; } @@ -222,7 +214,7 @@ void MemoryManager::checkGpuUsageAndDestroyGraphicsAllocations(GraphicsAllocatio if (gfxAllocation->taskCount == ObjectNotUsed || gfxAllocation->taskCount <= *getCommandStreamReceiver(0)->getTagAddress()) { freeGraphicsMemory(gfxAllocation); } else { - storeAllocation(std::unique_ptr(gfxAllocation), TEMPORARY_ALLOCATION); + getCommandStreamReceiver(0)->getInternalAllocationStorage()->storeAllocation(std::unique_ptr(gfxAllocation), TEMPORARY_ALLOCATION); } } diff --git a/runtime/memory_manager/memory_manager.h b/runtime/memory_manager/memory_manager.h index b1afeaacaf..10f57b3c58 100644 --- a/runtime/memory_manager/memory_manager.h +++ b/runtime/memory_manager/memory_manager.h @@ -204,9 +204,6 @@ class MemoryManager { void freeAllocationsList(uint32_t waitTaskCount, AllocationsList &allocationsList); - void storeAllocation(std::unique_ptr gfxAllocation, uint32_t allocationUsage); - void storeAllocation(std::unique_ptr gfxAllocation, uint32_t allocationUsage, uint32_t taskCount); - TagAllocator *getEventTsAllocator(); TagAllocator *getEventPerfCountAllocator(); TagAllocator *getTimestampPacketAllocator(); diff --git a/runtime/os_interface/windows/wddm_allocation.h b/runtime/os_interface/windows/wddm_allocation.h index db21ae0568..0df360b1f1 100644 --- a/runtime/os_interface/windows/wddm_allocation.h +++ b/runtime/os_interface/windows/wddm_allocation.h @@ -91,6 +91,7 @@ class WddmAllocation : public GraphicsAllocation { void setReservedAddress(void *reserveMem) { this->reservedAddressSpace = reserveMem; } + void setGpuAddress(uint64_t graphicsAddress) { this->gpuAddress = graphicsAddress; } protected: void *alignedCpuPtr; diff --git a/unit_tests/aub_tests/command_queue/enqueue_kernel_aub_tests.cpp b/unit_tests/aub_tests/command_queue/enqueue_kernel_aub_tests.cpp index 3635f2606f..3c8038b16a 100644 --- a/unit_tests/aub_tests/command_queue/enqueue_kernel_aub_tests.cpp +++ b/unit_tests/aub_tests/command_queue/enqueue_kernel_aub_tests.cpp @@ -380,7 +380,7 @@ struct AUBSimpleArgNonUniformFixture : public KernelAUBFixturegetMemoryManager()->allocateGraphicsMemory(sizeUserMemory, destMemory); csr->makeResidentHostPtrAllocation(outBuffer); - csr->getMemoryManager()->storeAllocation(std::unique_ptr(outBuffer), TEMPORARY_ALLOCATION); + csr->getInternalAllocationStorage()->storeAllocation(std::unique_ptr(outBuffer), TEMPORARY_ALLOCATION); ASSERT_NE(nullptr, outBuffer); outBuffer->setAllocationType(GraphicsAllocation::AllocationType::BUFFER); outBuffer->setMemObjectsAllocationWithWritableFlags(true); diff --git a/unit_tests/aub_tests/command_stream/aub_command_stream_fixture.h b/unit_tests/aub_tests/command_stream/aub_command_stream_fixture.h index 729f1e5143..6c103c2f82 100644 --- a/unit_tests/aub_tests/command_stream/aub_command_stream_fixture.h +++ b/unit_tests/aub_tests/command_stream/aub_command_stream_fixture.h @@ -11,6 +11,7 @@ #include "runtime/command_stream/aub_command_stream_receiver_hw.h" #include "runtime/command_stream/command_stream_receiver_with_aub_dump.h" #include "runtime/command_stream/tbx_command_stream_receiver_hw.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/memory_banks.h" #include "unit_tests/command_stream/command_stream_fixture.h" #include "unit_tests/tests_configuration.h" @@ -75,7 +76,7 @@ class AUBCommandStreamFixture : public CommandStreamFixture { GraphicsAllocation *createResidentAllocationAndStoreItInCsr(const void *address, size_t size) { GraphicsAllocation *graphicsAllocation = pCommandStreamReceiver->getMemoryManager()->allocateGraphicsMemory(size, address); pCommandStreamReceiver->makeResidentHostPtrAllocation(graphicsAllocation); - pCommandStreamReceiver->getMemoryManager()->storeAllocation(std::unique_ptr(graphicsAllocation), TEMPORARY_ALLOCATION); + pCommandStreamReceiver->getInternalAllocationStorage()->storeAllocation(std::unique_ptr(graphicsAllocation), TEMPORARY_ALLOCATION); return graphicsAllocation; } CommandStreamReceiver *pCommandStreamReceiver = nullptr; diff --git a/unit_tests/command_queue/command_queue_tests.cpp b/unit_tests/command_queue/command_queue_tests.cpp index 6f91b2f7fd..a29bc65e00 100644 --- a/unit_tests/command_queue/command_queue_tests.cpp +++ b/unit_tests/command_queue/command_queue_tests.cpp @@ -8,6 +8,7 @@ #include "hw_cmds.h" #include "runtime/command_queue/command_queue_hw.h" #include "runtime/command_stream/command_stream_receiver.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/memory_manager.h" #include "runtime/helpers/basic_math.h" #include "runtime/helpers/kernel_commands.h" @@ -320,23 +321,24 @@ TEST_F(CommandQueueCommandStreamTest, getCommandStreamCanRecycle) { EXPECT_GE(commandStream.getMaxAvailableSpace(), requiredSize); } -TEST_F(CommandQueueCommandStreamTest, MemoryManagerWithReusableAllocationsWhenAskedForCommandStreamReturnsAllocationFromReusablePool) { +TEST_F(CommandQueueCommandStreamTest, givenCommandStreamReceiverWithReusableAllocationsWhenAskedForCommandStreamReturnsAllocationFromReusablePool) { const cl_queue_properties props[3] = {CL_QUEUE_PROPERTIES, 0, 0}; CommandQueue cmdQ(context.get(), pDevice, props); auto memoryManager = pDevice->getMemoryManager(); size_t requiredSize = alignUp(100, MemoryConstants::pageSize) + CSRequirements::csOverfetchSize; auto allocation = memoryManager->allocateGraphicsMemory(requiredSize); - memoryManager->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); + auto &commandStreamReceiver = pDevice->getCommandStreamReceiver(); + commandStreamReceiver.getInternalAllocationStorage()->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); - EXPECT_FALSE(memoryManager->getCommandStreamReceiver(0)->getAllocationsForReuse().peekIsEmpty()); - EXPECT_TRUE(memoryManager->getCommandStreamReceiver(0)->getAllocationsForReuse().peekContains(*allocation)); + EXPECT_FALSE(commandStreamReceiver.getAllocationsForReuse().peekIsEmpty()); + EXPECT_TRUE(commandStreamReceiver.getAllocationsForReuse().peekContains(*allocation)); const auto &indirectHeap = cmdQ.getCS(100); EXPECT_EQ(indirectHeap.getGraphicsAllocation(), allocation); - EXPECT_TRUE(memoryManager->getCommandStreamReceiver(0)->getAllocationsForReuse().peekIsEmpty()); + EXPECT_TRUE(commandStreamReceiver.getAllocationsForReuse().peekIsEmpty()); } TEST_F(CommandQueueCommandStreamTest, givenCommandQueueWhenItIsDestroyedThenCommandStreamIsPutOnTheReusabeList) { auto cmdQ = new CommandQueue(context.get(), pDevice, 0); @@ -474,7 +476,7 @@ TEST_P(CommandQueueIndirectHeapTest, alignSizeToCacheLine) { EXPECT_TRUE(isAligned(indirectHeap.getAvailableSpace())); } -TEST_P(CommandQueueIndirectHeapTest, MemoryManagerWithReusableAllocationsWhenAskedForHeapAllocationReturnsAllocationFromReusablePool) { +TEST_P(CommandQueueIndirectHeapTest, givenCommandStreamReceiverWithReusableAllocationsWhenAskedForHeapAllocationReturnsAllocationFromReusablePool) { const cl_queue_properties props[3] = {CL_QUEUE_PROPERTIES, 0, 0}; CommandQueue cmdQ(context.get(), pDevice, props); @@ -493,10 +495,11 @@ TEST_P(CommandQueueIndirectHeapTest, MemoryManagerWithReusableAllocationsWhenAsk allocation->setSize(cmdQ.getDevice().getCommandStreamReceiver().defaultSshSize * 2); } - memoryManager->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); + auto &commandStreamReceiver = pDevice->getCommandStreamReceiver(); + commandStreamReceiver.getInternalAllocationStorage()->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); - EXPECT_FALSE(memoryManager->getCommandStreamReceiver(0)->getAllocationsForReuse().peekIsEmpty()); - EXPECT_TRUE(memoryManager->getCommandStreamReceiver(0)->getAllocationsForReuse().peekContains(*allocation)); + EXPECT_FALSE(commandStreamReceiver.getAllocationsForReuse().peekIsEmpty()); + EXPECT_TRUE(commandStreamReceiver.getAllocationsForReuse().peekContains(*allocation)); const auto &indirectHeap = cmdQ.getIndirectHeap(this->GetParam(), 100); @@ -505,13 +508,12 @@ TEST_P(CommandQueueIndirectHeapTest, MemoryManagerWithReusableAllocationsWhenAsk // if we obtain heap from reusable pool, we need to keep the size of allocation // surface state heap is an exception, it is capped at (max_ssh_size_for_HW - page_size) if (this->GetParam() == IndirectHeap::SURFACE_STATE) { - EXPECT_EQ(cmdQ.getDevice().getCommandStreamReceiver().defaultSshSize - MemoryConstants::pageSize, - indirectHeap.getMaxAvailableSpace()); + EXPECT_EQ(commandStreamReceiver.defaultSshSize - MemoryConstants::pageSize, indirectHeap.getMaxAvailableSpace()); } else { EXPECT_EQ(allocationSize, indirectHeap.getMaxAvailableSpace()); } - EXPECT_TRUE(memoryManager->getCommandStreamReceiver(0)->getAllocationsForReuse().peekIsEmpty()); + EXPECT_TRUE(commandStreamReceiver.getAllocationsForReuse().peekIsEmpty()); } TEST_P(CommandQueueIndirectHeapTest, CommandQueueWhenAskedForNewHeapStoresOldHeapForReuse) { diff --git a/unit_tests/command_stream/command_stream_receiver_flush_task_tests.cpp b/unit_tests/command_stream/command_stream_receiver_flush_task_tests.cpp index a706fced60..75f56f6b61 100644 --- a/unit_tests/command_stream/command_stream_receiver_flush_task_tests.cpp +++ b/unit_tests/command_stream/command_stream_receiver_flush_task_tests.cpp @@ -20,6 +20,7 @@ #include "runtime/helpers/ptr_math.h" #include "runtime/mem_obj/buffer.h" #include "runtime/memory_manager/graphics_allocation.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/memory_manager.h" #include "runtime/os_interface/debug_settings_manager.h" #include "runtime/utilities/linux/debug_env_reader.h" @@ -3119,10 +3120,10 @@ HWTEST_F(CommandStreamReceiverFlushTaskTests, givenCsrWhenTemporaryAndReusableAl auto reusableToClean = memoryManager->allocateGraphicsMemory(4096u); auto reusableToHold = memoryManager->allocateGraphicsMemory(4096u); - memoryManager->storeAllocation(std::unique_ptr(temporaryToClean), TEMPORARY_ALLOCATION); - memoryManager->storeAllocation(std::unique_ptr(temporaryToHold), TEMPORARY_ALLOCATION); - memoryManager->storeAllocation(std::unique_ptr(reusableToClean), REUSABLE_ALLOCATION); - memoryManager->storeAllocation(std::unique_ptr(reusableToHold), REUSABLE_ALLOCATION); + commandStreamReceiver.getInternalAllocationStorage()->storeAllocation(std::unique_ptr(temporaryToClean), TEMPORARY_ALLOCATION); + commandStreamReceiver.getInternalAllocationStorage()->storeAllocation(std::unique_ptr(temporaryToHold), TEMPORARY_ALLOCATION); + commandStreamReceiver.getInternalAllocationStorage()->storeAllocation(std::unique_ptr(reusableToClean), REUSABLE_ALLOCATION); + commandStreamReceiver.getInternalAllocationStorage()->storeAllocation(std::unique_ptr(reusableToHold), REUSABLE_ALLOCATION); temporaryToClean->taskCount = 1; reusableToClean->taskCount = 1; diff --git a/unit_tests/command_stream/command_stream_receiver_tests.cpp b/unit_tests/command_stream/command_stream_receiver_tests.cpp index 411999a8ec..e0994db147 100644 --- a/unit_tests/command_stream/command_stream_receiver_tests.cpp +++ b/unit_tests/command_stream/command_stream_receiver_tests.cpp @@ -11,6 +11,7 @@ #include "runtime/helpers/cache_policy.h" #include "runtime/mem_obj/buffer.h" #include "runtime/memory_manager/graphics_allocation.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/memory_manager.h" #include "test.h" #include "unit_tests/fixtures/device_fixture.h" @@ -166,38 +167,21 @@ TEST_F(CommandStreamReceiverTest, memoryManagerHasAccessToCSR) { EXPECT_EQ(commandStreamReceiver, memoryManager->getCommandStreamReceiver(0)); } -HWTEST_F(CommandStreamReceiverTest, storedAllocationsHaveCSRtaskCount) { +HWTEST_F(CommandStreamReceiverTest, whenStoreAllocationThenStoredAllocationHasTaskCountFromCsr) { auto &csr = pDevice->getUltCommandStreamReceiver(); auto *memoryManager = csr.getMemoryManager(); void *host_ptr = (void *)0x1234; auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); + EXPECT_EQ(ObjectNotUsed, allocation->taskCount); + csr.taskCount = 2u; - memoryManager->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); + csr.getInternalAllocationStorage()->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); EXPECT_EQ(csr.peekTaskCount(), allocation->taskCount); } -HWTEST_F(CommandStreamReceiverTest, dontReuseSurfaceIfStillInUse) { - auto &csr = pDevice->getUltCommandStreamReceiver(); - auto *memoryManager = csr.getMemoryManager(); - void *host_ptr = (void *)0x1234; - auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); - - csr.taskCount = 2u; - - memoryManager->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); - - auto *hwTag = csr.getTagAddress(); - - *hwTag = 1; - - auto newAllocation = memoryManager->obtainReusableAllocation(1, false); - - EXPECT_EQ(nullptr, newAllocation); -} - HWTEST_F(CommandStreamReceiverTest, givenCommandStreamReceiverWhenCheckedForInitialStatusOfStatelessMocsIndexThenUnknownMocsIsReturend) { auto &csr = pDevice->getUltCommandStreamReceiver(); EXPECT_EQ(CacheSettings::unknownMocs, csr.latestSentStatelessMocsConfig); diff --git a/unit_tests/command_stream/experimental_command_buffer_tests.cpp b/unit_tests/command_stream/experimental_command_buffer_tests.cpp index 79789eac13..5c923daa5e 100644 --- a/unit_tests/command_stream/experimental_command_buffer_tests.cpp +++ b/unit_tests/command_stream/experimental_command_buffer_tests.cpp @@ -5,6 +5,7 @@ * */ +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/memory_constants.h" #include "runtime/memory_manager/memory_manager.h" #include "unit_tests/fixtures/ult_command_stream_receiver_fixture.h" @@ -241,14 +242,15 @@ HWTEST_F(MockExperimentalCommandBufferTest, givenEnabledExperimentalCmdBufferWhe HWTEST_F(MockExperimentalCommandBufferTest, givenEnabledExperimentalCmdBufferWhenMemoryManagerAlreadyStoresAllocationThenUseItForLinearSteam) { auto &commandStreamReceiver = pDevice->getUltCommandStreamReceiver(); + auto storage = commandStreamReceiver.getInternalAllocationStorage(); commandStreamReceiver.storeMakeResidentAllocations = true; MemoryManager *memoryManager = commandStreamReceiver.getMemoryManager(); //Make two allocations, since CSR will try to reuse it also auto allocation = memoryManager->allocateGraphicsMemory(3 * MemoryConstants::pageSize); - memoryManager->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); + storage->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); allocation = memoryManager->allocateGraphicsMemory(3 * MemoryConstants::pageSize); - memoryManager->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); + storage->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); MockExperimentalCommandBuffer *mockExCmdBuffer = static_cast(commandStreamReceiver.experimentalCmdBuffer.get()); diff --git a/unit_tests/event/event_tests.cpp b/unit_tests/event/event_tests.cpp index 9a348a54d1..8866b78096 100644 --- a/unit_tests/event/event_tests.cpp +++ b/unit_tests/event/event_tests.cpp @@ -11,6 +11,7 @@ #include "runtime/event/perf_counter.h" #include "runtime/helpers/hw_info.h" #include "runtime/helpers/task_information.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/surface.h" #include "runtime/os_interface/os_interface.h" #include "test.h" @@ -410,7 +411,7 @@ TEST_F(UpdateEventTest, givenEventContainingCommandQueueWhenItsStatusIsUpdatedTo size_t size = 4096; auto temporary = memoryManager->allocateGraphicsMemory(size, ptr); temporary->taskCount = 3; - memoryManager->storeAllocation(std::unique_ptr(temporary), TEMPORARY_ALLOCATION); + device->getCommandStreamReceiver().getInternalAllocationStorage()->storeAllocation(std::unique_ptr(temporary), TEMPORARY_ALLOCATION); Event event(commandQueue.get(), CL_COMMAND_NDRANGE_KERNEL, 3, 3); EXPECT_EQ(1u, hostPtrManager->getFragmentCount()); @@ -462,7 +463,8 @@ TEST_F(InternalsEventTest, processBlockedCommandsKernelOperation) { cmdQ.allocateHeapMemory(IndirectHeap::SURFACE_STATE, 4096u, ssh); using UniqueIH = std::unique_ptr; auto blockedCommandsData = new KernelOperation(std::unique_ptr(cmdStream), UniqueIH(dsh), - UniqueIH(ioh), UniqueIH(ssh), *cmdQ.getDevice().getMemoryManager()); + UniqueIH(ioh), UniqueIH(ssh), + *cmdQ.getDevice().getCommandStreamReceiver().getInternalAllocationStorage()); MockKernelWithInternals mockKernelWithInternals(*pDevice); auto pKernel = mockKernelWithInternals.mockKernel; @@ -500,7 +502,8 @@ TEST_F(InternalsEventTest, processBlockedCommandsAbortKernelOperation) { cmdQ.allocateHeapMemory(IndirectHeap::SURFACE_STATE, 4096u, ssh); using UniqueIH = std::unique_ptr; auto blockedCommandsData = new KernelOperation(std::unique_ptr(cmdStream), UniqueIH(dsh), - UniqueIH(ioh), UniqueIH(ssh), *cmdQ.getDevice().getMemoryManager()); + UniqueIH(ioh), UniqueIH(ssh), + *cmdQ.getDevice().getCommandStreamReceiver().getInternalAllocationStorage()); MockKernelWithInternals mockKernelWithInternals(*pDevice); auto pKernel = mockKernelWithInternals.mockKernel; @@ -534,7 +537,8 @@ TEST_F(InternalsEventTest, givenBlockedKernelWithPrintfWhenSubmittedThenPrintOut cmdQ.allocateHeapMemory(IndirectHeap::SURFACE_STATE, 4096u, ssh); using UniqueIH = std::unique_ptr; auto blockedCommandsData = new KernelOperation(std::unique_ptr(cmdStream), UniqueIH(dsh), - UniqueIH(ioh), UniqueIH(ssh), *cmdQ.getDevice().getMemoryManager()); + UniqueIH(ioh), UniqueIH(ssh), + *cmdQ.getDevice().getCommandStreamReceiver().getInternalAllocationStorage()); SPatchAllocateStatelessPrintfSurface *pPrintfSurface = new SPatchAllocateStatelessPrintfSurface(); pPrintfSurface->DataParamOffset = 0; @@ -1449,7 +1453,7 @@ HWTEST_F(InternalsEventTest, givenAbortedCommandWhenSubmitCalledThenDontUpdateFl pCmdQ->allocateHeapMemory(IndirectHeap::SURFACE_STATE, 4096u, ssh); using UniqueIH = std::unique_ptr; auto blockedCommandsData = new KernelOperation(std::unique_ptr(cmdStream), UniqueIH(dsh), - UniqueIH(ioh), UniqueIH(ssh), *pCmdQ->getDevice().getMemoryManager()); + UniqueIH(ioh), UniqueIH(ssh), *pCmdQ->getDevice().getCommandStreamReceiver().getInternalAllocationStorage()); PreemptionMode preemptionMode = pDevice->getPreemptionMode(); std::vector v; auto cmd = new CommandComputeKernel(*pCmdQ, std::unique_ptr(blockedCommandsData), v, false, false, false, nullptr, preemptionMode, pKernel, 1); diff --git a/unit_tests/event/user_events_tests.cpp b/unit_tests/event/user_events_tests.cpp index 3f56a2d313..d867bec398 100644 --- a/unit_tests/event/user_events_tests.cpp +++ b/unit_tests/event/user_events_tests.cpp @@ -10,6 +10,7 @@ #include "unit_tests/fixtures/hello_world_fixture.h" #include "unit_tests/helpers/debug_manager_state_restore.h" #include "unit_tests/mocks/mock_event.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/memory_manager.h" TEST(UserEvent, testInitialStatusOfUserEventCmdQueue) { @@ -899,7 +900,7 @@ TEST_F(EventTests, waitForEventsDestroysTemporaryAllocations) { EXPECT_TRUE(csr.getTemporaryAllocations().peekIsEmpty()); GraphicsAllocation *temporaryAllocation = memoryManager->allocateGraphicsMemory(MemoryConstants::pageSize); - memoryManager->storeAllocation(std::unique_ptr(temporaryAllocation), TEMPORARY_ALLOCATION); + csr.getInternalAllocationStorage()->storeAllocation(std::unique_ptr(temporaryAllocation), TEMPORARY_ALLOCATION); EXPECT_EQ(temporaryAllocation, csr.getTemporaryAllocations().peekHead()); diff --git a/unit_tests/execution_model/submit_blocked_parent_kernel_tests.cpp b/unit_tests/execution_model/submit_blocked_parent_kernel_tests.cpp index 6a446dcb89..f280fae330 100644 --- a/unit_tests/execution_model/submit_blocked_parent_kernel_tests.cpp +++ b/unit_tests/execution_model/submit_blocked_parent_kernel_tests.cpp @@ -100,7 +100,7 @@ HWTEST_F(ParentKernelCommandQueueFixture, givenLockedEMcritcalSectionWhenParentK std::unique_ptr(dsh), std::unique_ptr(ioh), std::unique_ptr(ssh), - *pCmdQ->getDevice().getMemoryManager()); + *pCmdQ->getDevice().getCommandStreamReceiver().getInternalAllocationStorage()); blockedCommandData->surfaceStateHeapSizeEM = minSizeSSHForEM; PreemptionMode preemptionMode = device->getPreemptionMode(); @@ -159,7 +159,7 @@ HWTEST_F(ParentKernelCommandQueueFixture, givenParentKernelWhenCommandIsSubmitte std::unique_ptr(dsh), std::unique_ptr(ioh), std::unique_ptr(ssh), - *pCmdQ->getDevice().getMemoryManager()); + *pCmdQ->getDevice().getCommandStreamReceiver().getInternalAllocationStorage()); size_t minSizeSSHForEM = KernelCommandsHelper::template getSizeRequiredForExecutionModel(*parentKernel); @@ -201,7 +201,7 @@ HWTEST_F(ParentKernelCommandQueueFixture, givenParentKernelWhenCommandIsSubmitte std::unique_ptr(dsh), std::unique_ptr(ioh), std::unique_ptr(ssh), - *pCmdQ->getDevice().getMemoryManager()); + *pCmdQ->getDevice().getCommandStreamReceiver().getInternalAllocationStorage()); size_t minSizeSSHForEM = KernelCommandsHelper::template getSizeRequiredForExecutionModel(*parentKernel); @@ -240,7 +240,7 @@ HWTEST_F(ParentKernelCommandQueueFixture, givenBlockedParentKernelWithProfilingW std::unique_ptr(dsh), std::unique_ptr(ioh), std::unique_ptr(ssh), - *pCmdQ->getDevice().getMemoryManager()); + *pCmdQ->getDevice().getCommandStreamReceiver().getInternalAllocationStorage()); size_t minSizeSSHForEM = KernelCommandsHelper::template getSizeRequiredForExecutionModel(*parentKernel); @@ -283,7 +283,7 @@ HWTEST_F(ParentKernelCommandQueueFixture, givenParentKernelWhenCommandIsSubmitte std::unique_ptr(dsh), std::unique_ptr(ioh), std::unique_ptr(ssh), - *pCmdQ->getDevice().getMemoryManager()); + *pCmdQ->getDevice().getCommandStreamReceiver().getInternalAllocationStorage()); size_t minSizeSSHForEM = KernelCommandsHelper::template getSizeRequiredForExecutionModel(*parentKernel); @@ -337,7 +337,7 @@ HWCMDTEST_F(IGFX_GEN8_CORE, ParentKernelCommandQueueFixture, givenUsedCommandQue std::unique_ptr(dsh), std::unique_ptr(ioh), std::unique_ptr(ssh), - *pCmdQ->getDevice().getMemoryManager()); + *pCmdQ->getDevice().getCommandStreamReceiver().getInternalAllocationStorage()); blockedCommandData->surfaceStateHeapSizeEM = minSizeSSHForEM; PreemptionMode preemptionMode = device->getPreemptionMode(); @@ -386,7 +386,7 @@ HWCMDTEST_F(IGFX_GEN8_CORE, ParentKernelCommandQueueFixture, givenNotUsedSSHWhen std::unique_ptr(dsh), std::unique_ptr(ioh), std::unique_ptr(ssh), - *pCmdQ->getDevice().getMemoryManager()); + *pCmdQ->getDevice().getCommandStreamReceiver().getInternalAllocationStorage()); blockedCommandData->surfaceStateHeapSizeEM = minSizeSSHForEM; PreemptionMode preemptionMode = device->getPreemptionMode(); diff --git a/unit_tests/memory_manager/CMakeLists.txt b/unit_tests/memory_manager/CMakeLists.txt index 78a1ed5637..95d70fdf3f 100644 --- a/unit_tests/memory_manager/CMakeLists.txt +++ b/unit_tests/memory_manager/CMakeLists.txt @@ -9,6 +9,7 @@ set(IGDRCL_SRCS_tests_memory_manager ${CMAKE_CURRENT_SOURCE_DIR}/address_mapper_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/deferred_deleter_mt_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/host_ptr_manager_tests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/internal_allocation_storage_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory_manager_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/memory_manager_allocate_in_device_pool_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory_manager_allocate_in_device_pool_tests.inl diff --git a/unit_tests/memory_manager/internal_allocation_storage_tests.cpp b/unit_tests/memory_manager/internal_allocation_storage_tests.cpp new file mode 100644 index 0000000000..0dd0069e19 --- /dev/null +++ b/unit_tests/memory_manager/internal_allocation_storage_tests.cpp @@ -0,0 +1,206 @@ +/* + * Copyright (C) 2018 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "runtime/memory_manager/internal_allocation_storage.h" +#include "unit_tests/fixtures/memory_allocator_fixture.h" +#include "unit_tests/gen_common/test.h" +#include "unit_tests/helpers/debug_manager_state_restore.h" +#include "unit_tests/utilities/containers_tests_helpers.h" + +struct InternalAllocationStorageTest : public MemoryAllocatorFixture, + public ::testing::Test { + using MemoryAllocatorFixture::TearDown; + void SetUp() override { + MemoryAllocatorFixture::SetUp(); + csr->initializeTagAllocation(); + storage = csr->getInternalAllocationStorage(); + } + InternalAllocationStorage *storage; +}; + +TEST_F(InternalAllocationStorageTest, givenDebugFlagThatDisablesAllocationReuseWhenStoreReusableAllocationIsCalledThenAllocationIsReleased) { + DebugManagerStateRestore stateRestorer; + DebugManager.flags.DisableResourceRecycling.set(true); + void *host_ptr = (void *)0x1234; + + auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); + + storage->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); + EXPECT_NE(allocation, csr->getAllocationsForReuse().peekHead()); + EXPECT_TRUE(csr->getAllocationsForReuse().peekIsEmpty()); +} + +TEST_F(InternalAllocationStorageTest, whenCleanAllocationListThenRemoveOnlyCompletedAllocations) { + void *host_ptr = (void *)0x1234; + + auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); + auto allocation2 = memoryManager->allocateGraphicsMemory(1, host_ptr); + auto allocation3 = memoryManager->allocateGraphicsMemory(1, host_ptr); + + allocation->taskCount = 10; + allocation2->taskCount = 5; + allocation3->taskCount = 15; + + storage->storeAllocation(std::unique_ptr(allocation), TEMPORARY_ALLOCATION); + storage->storeAllocation(std::unique_ptr(allocation2), TEMPORARY_ALLOCATION); + storage->storeAllocation(std::unique_ptr(allocation3), TEMPORARY_ALLOCATION); + + //head point to alloc 2, tail points to alloc3 + EXPECT_TRUE(csr->getTemporaryAllocations().peekContains(*allocation)); + EXPECT_TRUE(csr->getTemporaryAllocations().peekContains(*allocation2)); + EXPECT_TRUE(csr->getTemporaryAllocations().peekContains(*allocation3)); + EXPECT_EQ(-1, verifyDListOrder(csr->getTemporaryAllocations().peekHead(), allocation, allocation2, allocation3)); + + //now remove element form the middle + storage->cleanAllocationList(6, TEMPORARY_ALLOCATION); + EXPECT_TRUE(csr->getTemporaryAllocations().peekContains(*allocation)); + EXPECT_FALSE(csr->getTemporaryAllocations().peekContains(*allocation2)); + EXPECT_TRUE(csr->getTemporaryAllocations().peekContains(*allocation3)); + EXPECT_EQ(-1, verifyDListOrder(csr->getTemporaryAllocations().peekHead(), allocation, allocation3)); + + //now remove head + storage->cleanAllocationList(11, TEMPORARY_ALLOCATION); + EXPECT_FALSE(csr->getTemporaryAllocations().peekContains(*allocation)); + EXPECT_FALSE(csr->getTemporaryAllocations().peekContains(*allocation2)); + EXPECT_TRUE(csr->getTemporaryAllocations().peekContains(*allocation3)); + + //now remove tail + storage->cleanAllocationList(16, TEMPORARY_ALLOCATION); + EXPECT_TRUE(csr->getTemporaryAllocations().peekIsEmpty()); +} + +TEST_F(InternalAllocationStorageTest, whenAllocationIsStoredAsReusableButIsStillUsedThenCannotBeObtained) { + void *host_ptr = (void *)0x1234; + auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); + + storage->storeAllocationWithTaskCount(std::unique_ptr(allocation), REUSABLE_ALLOCATION, 2u); + + auto *hwTag = csr->getTagAddress(); + + *hwTag = 1u; + auto newAllocation = storage->obtainReusableAllocation(1, false); + EXPECT_EQ(nullptr, newAllocation); + storage->cleanAllocationList(2u, REUSABLE_ALLOCATION); +} + +TEST_F(InternalAllocationStorageTest, whenObtainAllocationFromEmptyReuseListThenReturnNullptr) { + auto allocation2 = storage->obtainReusableAllocation(1, false); + EXPECT_EQ(nullptr, allocation2); +} + +TEST_F(InternalAllocationStorageTest, whenAllocationIsStoredAsReusableAndNotUsedThenCanBeObtained) { + void *host_ptr = (void *)0x1234; + auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); + EXPECT_NE(nullptr, allocation); + + storage->storeAllocationWithTaskCount(std::unique_ptr(allocation), REUSABLE_ALLOCATION, 2u); + EXPECT_FALSE(csr->getAllocationsForReuse().peekIsEmpty()); + + auto *hwTag = csr->getTagAddress(); + + *hwTag = 3u; + auto reusedAllocation = storage->obtainReusableAllocation(1, false).release(); + + EXPECT_EQ(allocation, reusedAllocation); + EXPECT_TRUE(csr->getAllocationsForReuse().peekIsEmpty()); + memoryManager->freeGraphicsMemory(allocation); +} + +TEST_F(InternalAllocationStorageTest, whenObtainAllocationFromMidlleOfReusableListThenItIsDetachedFromLinkedList) { + auto &reusableAllocations = csr->getAllocationsForReuse(); + EXPECT_TRUE(reusableAllocations.peekIsEmpty()); + + auto allocation = memoryManager->allocateGraphicsMemory(1); + auto allocation2 = memoryManager->allocateGraphicsMemory(10000); + auto allocation3 = memoryManager->allocateGraphicsMemory(1); + + EXPECT_TRUE(reusableAllocations.peekIsEmpty()); + EXPECT_EQ(nullptr, allocation2->next); + EXPECT_EQ(nullptr, allocation2->prev); + + storage->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); + + EXPECT_TRUE(reusableAllocations.peekContains(*allocation)); + EXPECT_FALSE(reusableAllocations.peekContains(*allocation2)); + EXPECT_FALSE(reusableAllocations.peekContains(*allocation3)); + EXPECT_EQ(nullptr, allocation2->next); + EXPECT_EQ(nullptr, allocation2->prev); + + storage->storeAllocation(std::unique_ptr(allocation2), REUSABLE_ALLOCATION); + + EXPECT_TRUE(reusableAllocations.peekContains(*allocation)); + EXPECT_TRUE(reusableAllocations.peekContains(*allocation2)); + EXPECT_FALSE(reusableAllocations.peekContains(*allocation3)); + EXPECT_EQ(nullptr, allocation2->next); + EXPECT_EQ(allocation, allocation2->prev); + + storage->storeAllocation(std::unique_ptr(allocation3), REUSABLE_ALLOCATION); + + EXPECT_TRUE(reusableAllocations.peekContains(*allocation)); + EXPECT_TRUE(reusableAllocations.peekContains(*allocation2)); + EXPECT_TRUE(reusableAllocations.peekContains(*allocation3)); + EXPECT_EQ(allocation3, allocation2->next); + EXPECT_EQ(allocation, allocation2->prev); + + auto reusableAllocation = storage->obtainReusableAllocation(10000, false).release(); + EXPECT_EQ(reusableAllocation, allocation2); + EXPECT_EQ(nullptr, allocation2->next); + EXPECT_EQ(nullptr, allocation2->prev); + + EXPECT_EQ(nullptr, reusableAllocation->next); + EXPECT_EQ(nullptr, reusableAllocation->prev); + + EXPECT_FALSE(reusableAllocations.peekContains(*reusableAllocation)); + EXPECT_TRUE(reusableAllocations.peekContains(*allocation)); + EXPECT_FALSE(reusableAllocations.peekContains(*allocation2)); + EXPECT_TRUE(reusableAllocations.peekContains(*allocation3)); + + memoryManager->freeGraphicsMemory(allocation2); + storage->cleanAllocationList(ObjectNotUsed, REUSABLE_ALLOCATION); +} + +TEST_F(InternalAllocationStorageTest, givenNonInternalAllocationWhenItIsPutOnReusableListWhenInternalAllocationIsRequestedThenNullIsReturned) { + EXPECT_TRUE(csr->getAllocationsForReuse().peekIsEmpty()); + + auto allocation = memoryManager->allocateGraphicsMemory(4096); + storage->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); + + EXPECT_FALSE(csr->getAllocationsForReuse().peekIsEmpty()); + + auto internalAllocation = storage->obtainReusableAllocation(1, true); + EXPECT_EQ(nullptr, internalAllocation); +} + +TEST_F(InternalAllocationStorageTest, givenInternalAllocationWhenItIsPutOnReusableListWhenNonInternalAllocationIsRequestedThenNullIsReturned) { + EXPECT_TRUE(csr->getAllocationsForReuse().peekIsEmpty()); + + auto allocation = memoryManager->allocateGraphicsMemory(4096); + allocation->is32BitAllocation = true; + + storage->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); + + EXPECT_FALSE(csr->getAllocationsForReuse().peekIsEmpty()); + + auto internalAllocation = storage->obtainReusableAllocation(1, false); + EXPECT_EQ(nullptr, internalAllocation); +} + +TEST_F(InternalAllocationStorageTest, givenInternalAllocationWhenItIsPutOnReusableListWhenInternalAllocationIsRequestedThenItIsReturned) { + EXPECT_TRUE(csr->getAllocationsForReuse().peekIsEmpty()); + + auto allocation = memoryManager->allocateGraphicsMemory(4096); + allocation->is32BitAllocation = true; + + storage->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); + + EXPECT_FALSE(csr->getAllocationsForReuse().peekIsEmpty()); + + auto internalAllocation = storage->obtainReusableAllocation(1, true); + EXPECT_EQ(allocation, internalAllocation.get()); + internalAllocation.release(); + memoryManager->freeGraphicsMemory(allocation); +} \ No newline at end of file diff --git a/unit_tests/memory_manager/memory_manager_tests.cpp b/unit_tests/memory_manager/memory_manager_tests.cpp index 6d2cd39b9b..a44b6b959f 100644 --- a/unit_tests/memory_manager/memory_manager_tests.cpp +++ b/unit_tests/memory_manager/memory_manager_tests.cpp @@ -9,6 +9,7 @@ #include "runtime/helpers/dispatch_info.h" #include "runtime/helpers/kernel_commands.h" #include "runtime/helpers/timestamp_packet.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/memory_constants.h" #include "runtime/mem_obj/image.h" #include "runtime/os_interface/os_context.h" @@ -31,7 +32,6 @@ #include "unit_tests/mocks/mock_kernel.h" #include "unit_tests/mocks/mock_mdi.h" #include "unit_tests/mocks/mock_memory_manager.h" -#include "unit_tests/utilities/containers_tests_helpers.h" #include "test.h" #include @@ -228,243 +228,31 @@ TEST_F(MemoryAllocatorTest, allocateGraphicsMoreThanPageAligned) { memoryManager->freeGraphicsMemory(allocation); } -TEST_F(MemoryAllocatorTest, storeTemporaryAllocation) { - void *host_ptr = (void *)0x1234; - auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); - allocation->taskCount = 1; - memoryManager->storeAllocation(std::unique_ptr(allocation), TEMPORARY_ALLOCATION); - EXPECT_TRUE(csr->getTemporaryAllocations().peekIsEmpty() == false); -} - -TEST_F(MemoryAllocatorTest, givenMemoryManagerWhenStoreOrCleanTempoaryAllocationsThenUseFirstCommandStreamReceiver) { +TEST_F(MemoryAllocatorTest, givenMemoryManagerWhenCleanTempoaryAllocationsThenUseFirstCommandStreamReceiver) { void *host_ptr = (void *)0x1234; auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); allocation->taskCount = 1; auto csr = memoryManager->getCommandStreamReceiver(0); EXPECT_TRUE(csr->getTemporaryAllocations().peekIsEmpty()); - memoryManager->storeAllocation(std::unique_ptr(allocation), TEMPORARY_ALLOCATION); + + csr->getInternalAllocationStorage()->storeAllocation(std::unique_ptr(allocation), TEMPORARY_ALLOCATION); EXPECT_FALSE(csr->getTemporaryAllocations().peekIsEmpty()); memoryManager->cleanAllocationList(1, TEMPORARY_ALLOCATION); EXPECT_TRUE(csr->getTemporaryAllocations().peekIsEmpty()); } -TEST_F(MemoryAllocatorTest, givenMemoryManagerWhenStoreOrCleanReusableAllocationsThenUseFirstCommandStreamReceiver) { +TEST_F(MemoryAllocatorTest, givenMemoryManagerWhenCleanReusableAllocationsThenUseFirstCommandStreamReceiver) { void *host_ptr = (void *)0x1234; auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); allocation->taskCount = 1; auto csr = memoryManager->getCommandStreamReceiver(0); EXPECT_TRUE(csr->getAllocationsForReuse().peekIsEmpty()); - memoryManager->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); + csr->getInternalAllocationStorage()->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); EXPECT_FALSE(csr->getAllocationsForReuse().peekIsEmpty()); memoryManager->cleanAllocationList(1, REUSABLE_ALLOCATION); EXPECT_TRUE(csr->getAllocationsForReuse().peekIsEmpty()); } -TEST_F(MemoryAllocatorTest, selectiveDestroy) { - void *host_ptr = (void *)0x1234; - auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); - - allocation->taskCount = 10; - memoryManager->storeAllocation(std::unique_ptr(allocation), TEMPORARY_ALLOCATION); - - auto allocation2 = memoryManager->allocateGraphicsMemory(1, host_ptr); - allocation2->taskCount = 15; - - memoryManager->storeAllocation(std::unique_ptr(allocation2), TEMPORARY_ALLOCATION); - - //check the same task count first, nothign should be killed - memoryManager->cleanAllocationList(11, TEMPORARY_ALLOCATION); - - EXPECT_EQ(allocation2, csr->getTemporaryAllocations().peekHead()); - - memoryManager->cleanAllocationList(16, TEMPORARY_ALLOCATION); - - EXPECT_TRUE(csr->getTemporaryAllocations().peekIsEmpty()); -} - -TEST_F(MemoryAllocatorTest, intrusiveListsInjectionsAndRemoval) { - void *host_ptr = (void *)0x1234; - void *host_ptr2 = (void *)0x1234; - void *host_ptr3 = (void *)0x1234; - - auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); - auto allocation2 = memoryManager->allocateGraphicsMemory(1, host_ptr2); - auto allocation3 = memoryManager->allocateGraphicsMemory(1, host_ptr3); - - allocation->taskCount = 10; - allocation2->taskCount = 5; - allocation3->taskCount = 15; - - memoryManager->storeAllocation(std::unique_ptr(allocation), TEMPORARY_ALLOCATION); - memoryManager->storeAllocation(std::unique_ptr(allocation2), TEMPORARY_ALLOCATION); - memoryManager->storeAllocation(std::unique_ptr(allocation3), TEMPORARY_ALLOCATION); - - //head point to alloc 2, tail points to alloc3 - EXPECT_TRUE(csr->getTemporaryAllocations().peekContains(*allocation)); - EXPECT_TRUE(csr->getTemporaryAllocations().peekContains(*allocation2)); - EXPECT_TRUE(csr->getTemporaryAllocations().peekContains(*allocation3)); - EXPECT_EQ(-1, verifyDListOrder(csr->getTemporaryAllocations().peekHead(), allocation, allocation2, allocation3)); - - //now remove element form the middle - memoryManager->cleanAllocationList(6, TEMPORARY_ALLOCATION); - EXPECT_TRUE(csr->getTemporaryAllocations().peekContains(*allocation)); - EXPECT_FALSE(csr->getTemporaryAllocations().peekContains(*allocation2)); - EXPECT_TRUE(csr->getTemporaryAllocations().peekContains(*allocation3)); - EXPECT_EQ(-1, verifyDListOrder(csr->getTemporaryAllocations().peekHead(), allocation, allocation3)); - - //now remove head - memoryManager->cleanAllocationList(11, TEMPORARY_ALLOCATION); - EXPECT_FALSE(csr->getTemporaryAllocations().peekContains(*allocation)); - EXPECT_FALSE(csr->getTemporaryAllocations().peekContains(*allocation2)); - EXPECT_TRUE(csr->getTemporaryAllocations().peekContains(*allocation3)); - - //now remove tail - memoryManager->cleanAllocationList(16, TEMPORARY_ALLOCATION); - EXPECT_TRUE(csr->getTemporaryAllocations().peekIsEmpty()); -} - -TEST_F(MemoryAllocatorTest, addAllocationToReuseList) { - void *host_ptr = (void *)0x1234; - - auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); - - memoryManager->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); - EXPECT_EQ(allocation, csr->getAllocationsForReuse().peekHead()); -} - -TEST_F(MemoryAllocatorTest, givenDebugFlagThatDisablesAllocationReuseWhenApiIsCalledThenAllocationIsReleased) { - DebugManager.flags.DisableResourceRecycling.set(true); - void *host_ptr = (void *)0x1234; - - auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); - - memoryManager->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); - EXPECT_NE(allocation, csr->getAllocationsForReuse().peekHead()); - EXPECT_TRUE(csr->getAllocationsForReuse().peekIsEmpty()); - DebugManager.flags.DisableResourceRecycling.set(false); -} - -TEST_F(MemoryAllocatorTest, givenDebugFlagThatDisablesAllocationReuseWhenStoreIsCalledWithTEmporaryAllocationThenItIsStored) { - DebugManager.flags.DisableResourceRecycling.set(true); - void *host_ptr = (void *)0x1234; - - auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); - - memoryManager->storeAllocation(std::unique_ptr(allocation), TEMPORARY_ALLOCATION); - EXPECT_EQ(allocation, csr->getTemporaryAllocations().peekHead()); - EXPECT_FALSE(csr->getTemporaryAllocations().peekIsEmpty()); - allocation->setGpuAddress(allocation->getGpuAddress()); - - DebugManager.flags.DisableResourceRecycling.set(false); -} - -TEST_F(MemoryAllocatorTest, obtainAllocationFromEmptyReuseListReturnNullPtr) { - void *host_ptr = (void *)0x1234; - - auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); - - auto allocation2 = memoryManager->obtainReusableAllocation(1, false); - EXPECT_EQ(nullptr, allocation2); - memoryManager->freeGraphicsMemory(allocation); -} - -TEST_F(MemoryAllocatorTest, obtainAllocationFromReusableList) { - void *host_ptr = (void *)0x1234; - - auto allocation = memoryManager->allocateGraphicsMemory(1, host_ptr); - - memoryManager->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); - - auto allocation2 = memoryManager->obtainReusableAllocation(1, false); - EXPECT_EQ(allocation, allocation2.get()); - - EXPECT_TRUE(csr->getAllocationsForReuse().peekIsEmpty()); - - memoryManager->freeGraphicsMemory(allocation2.release()); -} - -TEST_F(MemoryAllocatorTest, obtainAllocationFromMidlleOfReusableList) { - EXPECT_TRUE(csr->getAllocationsForReuse().peekIsEmpty()); - - auto allocation = memoryManager->allocateGraphicsMemory(1); - auto allocation2 = memoryManager->allocateGraphicsMemory(10000); - auto allocation3 = memoryManager->allocateGraphicsMemory(1); - - EXPECT_TRUE(csr->getAllocationsForReuse().peekIsEmpty()); - - memoryManager->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); - - EXPECT_TRUE(csr->getAllocationsForReuse().peekContains(*allocation)); - EXPECT_FALSE(csr->getAllocationsForReuse().peekContains(*allocation2)); - EXPECT_FALSE(csr->getAllocationsForReuse().peekContains(*allocation3)); - - memoryManager->storeAllocation(std::unique_ptr(allocation2), REUSABLE_ALLOCATION); - - EXPECT_TRUE(csr->getAllocationsForReuse().peekContains(*allocation)); - EXPECT_TRUE(csr->getAllocationsForReuse().peekContains(*allocation2)); - EXPECT_FALSE(csr->getAllocationsForReuse().peekContains(*allocation3)); - - memoryManager->storeAllocation(std::unique_ptr(allocation3), REUSABLE_ALLOCATION); - - EXPECT_TRUE(csr->getAllocationsForReuse().peekContains(*allocation)); - EXPECT_TRUE(csr->getAllocationsForReuse().peekContains(*allocation2)); - EXPECT_TRUE(csr->getAllocationsForReuse().peekContains(*allocation3)); - - auto reusableAllocation = memoryManager->obtainReusableAllocation(10000, false); - - EXPECT_EQ(nullptr, reusableAllocation->next); - EXPECT_EQ(nullptr, reusableAllocation->prev); - - EXPECT_FALSE(csr->getAllocationsForReuse().peekContains(*reusableAllocation)); - EXPECT_TRUE(csr->getAllocationsForReuse().peekContains(*allocation) || (allocation == reusableAllocation.get())); - EXPECT_TRUE(csr->getAllocationsForReuse().peekContains(*allocation2) || (allocation2 == reusableAllocation.get())); - EXPECT_TRUE(csr->getAllocationsForReuse().peekContains(*allocation3) || (allocation3 == reusableAllocation.get())); - - memoryManager->freeGraphicsMemory(reusableAllocation.release()); -} - -TEST_F(MemoryAllocatorTest, givenNonInternalAllocationWhenItIsPutOnReusableListWhenInternalAllocationIsRequestedThenNullIsReturned) { - EXPECT_TRUE(csr->getAllocationsForReuse().peekIsEmpty()); - - auto allocation = memoryManager->allocateGraphicsMemory(4096); - memoryManager->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); - - EXPECT_FALSE(csr->getAllocationsForReuse().peekIsEmpty()); - - auto internalAllocation = memoryManager->obtainReusableAllocation(1, true); - EXPECT_EQ(nullptr, internalAllocation); -} - -TEST_F(MemoryAllocatorTest, givenInternalAllocationWhenItIsPutOnReusableListWhenNonInternalAllocationIsRequestedThenNullIsReturned) { - EXPECT_TRUE(csr->getAllocationsForReuse().peekIsEmpty()); - - auto allocation = memoryManager->allocateGraphicsMemory(4096); - allocation->is32BitAllocation = true; - - memoryManager->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); - - EXPECT_FALSE(csr->getAllocationsForReuse().peekIsEmpty()); - - auto internalAllocation = memoryManager->obtainReusableAllocation(1, false); - EXPECT_EQ(nullptr, internalAllocation); -} - -TEST_F(MemoryAllocatorTest, givenInternalAllocationWhenItIsPutOnReusableListWhenInternalAllocationIsRequestedThenItIsReturned) { - EXPECT_TRUE(csr->getAllocationsForReuse().peekIsEmpty()); - - auto allocation = memoryManager->allocateGraphicsMemory(4096); - allocation->is32BitAllocation = true; - - memoryManager->storeAllocation(std::unique_ptr(allocation), REUSABLE_ALLOCATION); - - EXPECT_FALSE(csr->getAllocationsForReuse().peekIsEmpty()); - - auto internalAllocation = memoryManager->obtainReusableAllocation(1, true); - EXPECT_EQ(allocation, internalAllocation.get()); - internalAllocation.release(); - memoryManager->freeGraphicsMemory(allocation); -} - TEST_F(MemoryAllocatorTest, AlignedHostPtrWithAlignedSizeWhenAskedForGraphicsAllocationReturnsNullStorageFromHostPtrManager) { auto ptr = (void *)0x1000; MockMemoryManager mockMemoryManager(*executionEnvironment); @@ -1410,8 +1198,9 @@ TEST_F(MemoryManagerWithCsrTest, GivenAllocationsInHostPtrManagerReadyForCleanin EXPECT_NE(nullptr, fragment4); uint32_t taskCountReady = 1; - memoryManager->storeAllocation(std::unique_ptr(graphicsAllocation1), TEMPORARY_ALLOCATION, taskCountReady); - memoryManager->storeAllocation(std::unique_ptr(graphicsAllocation2), TEMPORARY_ALLOCATION, taskCountReady); + auto storage = csr->getInternalAllocationStorage(); + storage->storeAllocationWithTaskCount(std::unique_ptr(graphicsAllocation1), TEMPORARY_ALLOCATION, taskCountReady); + storage->storeAllocationWithTaskCount(std::unique_ptr(graphicsAllocation2), TEMPORARY_ALLOCATION, taskCountReady); EXPECT_EQ(4u, hostPtrManager->getFragmentCount()); @@ -1503,7 +1292,8 @@ TEST_F(MemoryManagerWithCsrTest, checkAllocationsForOverlappingWithBiggerOverlap EXPECT_NE(nullptr, fragment2); uint32_t taskCountReady = 1; - memoryManager->storeAllocation(std::unique_ptr(graphicsAllocation1), TEMPORARY_ALLOCATION, taskCountReady); + auto storage = csr->getInternalAllocationStorage(); + storage->storeAllocationWithTaskCount(std::unique_ptr(graphicsAllocation1), TEMPORARY_ALLOCATION, taskCountReady); // All fragments ready for release taskCount = taskCountReady; @@ -1548,7 +1338,8 @@ TEST_F(MemoryManagerWithCsrTest, checkAllocationsForOverlappingWithBiggerOverlap EXPECT_NE(nullptr, fragment2); uint32_t taskCountReady = 2; - memoryManager->storeAllocation(std::unique_ptr(graphicsAllocation1), TEMPORARY_ALLOCATION, taskCountReady); + auto storage = csr->getInternalAllocationStorage(); + storage->storeAllocationWithTaskCount(std::unique_ptr(graphicsAllocation1), TEMPORARY_ALLOCATION, taskCountReady); // All fragments ready for release currentGpuTag = 1; @@ -1605,7 +1396,8 @@ TEST_F(MemoryManagerWithCsrTest, checkAllocationsForOverlappingWithBiggerOverlap EXPECT_NE(nullptr, fragment2); uint32_t taskCountReady = 2; - memoryManager->storeAllocation(std::unique_ptr(graphicsAllocation1), TEMPORARY_ALLOCATION, taskCountReady); + auto storage = csr->getInternalAllocationStorage(); + storage->storeAllocationWithTaskCount(std::unique_ptr(graphicsAllocation1), TEMPORARY_ALLOCATION, taskCountReady); // All fragments ready for release currentGpuTag = taskCountReady - 1; diff --git a/unit_tests/os_interface/linux/drm_command_stream_tests.cpp b/unit_tests/os_interface/linux/drm_command_stream_tests.cpp index 3f158b9bfb..5e365f9e82 100644 --- a/unit_tests/os_interface/linux/drm_command_stream_tests.cpp +++ b/unit_tests/os_interface/linux/drm_command_stream_tests.cpp @@ -12,6 +12,7 @@ #include "runtime/helpers/aligned_memory.h" #include "runtime/helpers/options.h" #include "runtime/mem_obj/buffer.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/os_interface/os_context.h" #include "runtime/os_interface/linux/drm_allocation.h" #include "runtime/os_interface/linux/drm_buffer_object.h" @@ -1515,7 +1516,7 @@ TEST_F(DrmCommandStreamLeaksTest, FlushMultipleTimes) { csr->makeResident(*allocation); csr->makeResident(*allocation2); - mm->storeAllocation(std::unique_ptr(commandBuffer), REUSABLE_ALLOCATION); + csr->getInternalAllocationStorage()->storeAllocation(std::unique_ptr(commandBuffer), REUSABLE_ALLOCATION); auto commandBuffer2 = mm->allocateGraphicsMemory(1024); ASSERT_NE(nullptr, commandBuffer2); @@ -1529,7 +1530,7 @@ TEST_F(DrmCommandStreamLeaksTest, FlushMultipleTimes) { mm->freeGraphicsMemory(allocation); mm->freeGraphicsMemory(allocation2); - mm->storeAllocation(std::unique_ptr(commandBuffer2), REUSABLE_ALLOCATION); + csr->getInternalAllocationStorage()->storeAllocation(std::unique_ptr(commandBuffer2), REUSABLE_ALLOCATION); commandBuffer2 = mm->allocateGraphicsMemory(1024); ASSERT_NE(nullptr, commandBuffer2); cs.replaceBuffer(commandBuffer2->getUnderlyingBuffer(), commandBuffer2->getUnderlyingBufferSize()); diff --git a/unit_tests/os_interface/windows/device_command_stream_tests.cpp b/unit_tests/os_interface/windows/device_command_stream_tests.cpp index 1c49c30ff9..17ae742626 100644 --- a/unit_tests/os_interface/windows/device_command_stream_tests.cpp +++ b/unit_tests/os_interface/windows/device_command_stream_tests.cpp @@ -17,6 +17,7 @@ #include "runtime/helpers/flush_stamp.h" #include "runtime/helpers/options.h" #include "runtime/mem_obj/buffer.h" +#include "runtime/memory_manager/internal_allocation_storage.h" #include "runtime/memory_manager/memory_manager.h" #include "runtime/os_interface/windows/os_context_win.h" #include "runtime/os_interface/windows/os_interface.h" @@ -634,8 +635,8 @@ TEST_F(WddmCommandStreamTest, givenTwoTemporaryAllocationsWhenCleanTemporaryAllo GraphicsAllocation *graphicsAllocation = memoryManager->allocateGraphicsMemory(size, host_ptr); GraphicsAllocation *graphicsAllocation2 = memoryManager->allocateGraphicsMemory(size, host_ptr2); - memoryManager->storeAllocation(std::unique_ptr(graphicsAllocation), TEMPORARY_ALLOCATION); - memoryManager->storeAllocation(std::unique_ptr(graphicsAllocation2), TEMPORARY_ALLOCATION); + csr->getInternalAllocationStorage()->storeAllocation(std::unique_ptr(graphicsAllocation), TEMPORARY_ALLOCATION); + csr->getInternalAllocationStorage()->storeAllocation(std::unique_ptr(graphicsAllocation2), TEMPORARY_ALLOCATION); graphicsAllocation->taskCount = 1; graphicsAllocation2->taskCount = 100;