From b3d72ddd3d1e2077936370def6bcbd0981bec316 Mon Sep 17 00:00:00 2001 From: Mateusz Hoppe Date: Fri, 14 Jun 2024 14:15:14 +0000 Subject: [PATCH] fix: write memory for resident allocations in simulation mode - refactor and call proceesFlushResdiency() on memoryOperationsHandler - call free() to remove allocation from resident allocations when graphics allocation is released Related-To: NEO-11719 Signed-off-by: Mateusz Hoppe --- .../aub_command_stream_receiver_hw_base.inl | 5 + .../tbx_command_stream_receiver_hw.inl | 5 + .../source/memory_manager/memory_manager.cpp | 13 +- .../memory_operations_handler.h | 3 + .../aub_memory_operations_handler.cpp | 36 ++--- .../aub_memory_operations_handler.h | 5 +- shared/test/common/mocks/mock_aub_csr.h | 7 +- .../mock_aub_memory_operations_handler.h | 8 +- .../mocks/mock_memory_operations_handler.h | 14 ++ shared/test/common/mocks/mock_tbx_csr.h | 8 +- .../aub_command_stream_receiver_1_tests.cpp | 42 ++++++ .../tbx_command_stream_tests.cpp | 40 ++++++ .../memory_manager/memory_manager_tests.cpp | 32 +++++ .../aub_memory_operations_handler_tests.cpp | 136 +++++++++++++----- .../aub_memory_operations_handler_tests.h | 4 +- 15 files changed, 285 insertions(+), 73 deletions(-) diff --git a/shared/source/command_stream/aub_command_stream_receiver_hw_base.inl b/shared/source/command_stream/aub_command_stream_receiver_hw_base.inl index 8e78825189..3f87981dd2 100644 --- a/shared/source/command_stream/aub_command_stream_receiver_hw_base.inl +++ b/shared/source/command_stream/aub_command_stream_receiver_hw_base.inl @@ -36,6 +36,7 @@ #include "shared/source/memory_manager/memory_banks.h" #include "shared/source/memory_manager/memory_manager.h" #include "shared/source/memory_manager/page_table.h" +#include "shared/source/os_interface/aub_memory_operations_handler.h" #include "shared/source/os_interface/product_helper.h" #include "aubstream/aubstream.h" @@ -779,6 +780,10 @@ SubmissionStatus AUBCommandStreamReceiverHw::processResidency(const R gfxAllocation->updateResidencyTaskCount(this->taskCount + 1, this->osContext->getContextId()); } + if (this->executionEnvironment.rootDeviceEnvironments[this->rootDeviceIndex]->memoryOperationsInterface) { + this->executionEnvironment.rootDeviceEnvironments[this->rootDeviceIndex]->memoryOperationsInterface->processFlushResidency(this); + } + dumpAubNonWritable = false; return SubmissionStatus::success; } diff --git a/shared/source/command_stream/tbx_command_stream_receiver_hw.inl b/shared/source/command_stream/tbx_command_stream_receiver_hw.inl index 1e4676bae5..0d836ca2ef 100644 --- a/shared/source/command_stream/tbx_command_stream_receiver_hw.inl +++ b/shared/source/command_stream/tbx_command_stream_receiver_hw.inl @@ -27,6 +27,7 @@ #include "shared/source/helpers/hw_info.h" #include "shared/source/helpers/ptr_math.h" #include "shared/source/memory_manager/graphics_allocation.h" +#include "shared/source/os_interface/aub_memory_operations_handler.h" #include "shared/source/os_interface/product_helper.h" #include @@ -539,6 +540,10 @@ SubmissionStatus TbxCommandStreamReceiverHw::processResidency(const R gfxAllocation->updateResidencyTaskCount(this->taskCount + 1, this->osContext->getContextId()); } + if (this->executionEnvironment.rootDeviceEnvironments[this->rootDeviceIndex]->memoryOperationsInterface) { + this->executionEnvironment.rootDeviceEnvironments[this->rootDeviceIndex]->memoryOperationsInterface->processFlushResidency(this); + } + dumpTbxNonWritable = false; return SubmissionStatus::success; } diff --git a/shared/source/memory_manager/memory_manager.cpp b/shared/source/memory_manager/memory_manager.cpp index 46f412f60e..910cdf13fd 100644 --- a/shared/source/memory_manager/memory_manager.cpp +++ b/shared/source/memory_manager/memory_manager.cpp @@ -35,6 +35,7 @@ #include "shared/source/memory_manager/host_ptr_manager.h" #include "shared/source/memory_manager/internal_allocation_storage.h" #include "shared/source/memory_manager/local_memory_usage.h" +#include "shared/source/memory_manager/memory_operations_handler.h" #include "shared/source/memory_manager/multi_graphics_allocation.h" #include "shared/source/memory_manager/prefetch_manager.h" #include "shared/source/os_interface/os_context.h" @@ -259,10 +260,18 @@ void MemoryManager::freeGraphicsMemory(GraphicsAllocation *gfxAllocation, bool i if (!gfxAllocation) { return; } + bool rootEnvAvailable = executionEnvironment.rootDeviceEnvironments.size() > 0; - if (executionEnvironment.rootDeviceEnvironments.size() > 0 && executionEnvironment.rootDeviceEnvironments[gfxAllocation->getRootDeviceIndex()]->getBindlessHeapsHelper() != nullptr) { - executionEnvironment.rootDeviceEnvironments[gfxAllocation->getRootDeviceIndex()]->getBindlessHeapsHelper()->releaseSSToReusePool(gfxAllocation->getBindlessInfo()); + if (rootEnvAvailable) { + if (executionEnvironment.rootDeviceEnvironments[gfxAllocation->getRootDeviceIndex()]->getBindlessHeapsHelper() != nullptr) { + executionEnvironment.rootDeviceEnvironments[gfxAllocation->getRootDeviceIndex()]->getBindlessHeapsHelper()->releaseSSToReusePool(gfxAllocation->getBindlessInfo()); + } + + if (this->peekExecutionEnvironment().rootDeviceEnvironments[gfxAllocation->getRootDeviceIndex()]->memoryOperationsInterface) { + this->peekExecutionEnvironment().rootDeviceEnvironments[gfxAllocation->getRootDeviceIndex()]->memoryOperationsInterface->free(nullptr, *gfxAllocation); + } } + const bool hasFragments = gfxAllocation->fragmentsStorage.fragmentCount != 0; const bool isLocked = gfxAllocation->isLocked(); DEBUG_BREAK_IF(hasFragments && isLocked); diff --git a/shared/source/memory_manager/memory_operations_handler.h b/shared/source/memory_manager/memory_operations_handler.h index e12b67f2e5..92b4674818 100644 --- a/shared/source/memory_manager/memory_operations_handler.h +++ b/shared/source/memory_manager/memory_operations_handler.h @@ -13,6 +13,7 @@ namespace NEO { class Device; class GraphicsAllocation; class OsContext; +class CommandStreamReceiver; class MemoryOperationsHandler { public: @@ -23,8 +24,10 @@ class MemoryOperationsHandler { virtual MemoryOperationsStatus lock(Device *device, ArrayRef gfxAllocations) = 0; virtual MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) = 0; virtual MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) = 0; + virtual MemoryOperationsStatus free(Device *device, GraphicsAllocation &gfxAllocation) { return MemoryOperationsStatus::success; } virtual MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) = 0; virtual MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) = 0; + virtual void processFlushResidency(CommandStreamReceiver *csr) {} }; } // namespace NEO diff --git a/shared/source/os_interface/aub_memory_operations_handler.cpp b/shared/source/os_interface/aub_memory_operations_handler.cpp index 95a94c2db6..ae603119bd 100644 --- a/shared/source/os_interface/aub_memory_operations_handler.cpp +++ b/shared/source/os_interface/aub_memory_operations_handler.cpp @@ -83,6 +83,14 @@ MemoryOperationsStatus AubMemoryOperationsHandler::evict(Device *device, Graphic return MemoryOperationsStatus::success; } } +MemoryOperationsStatus AubMemoryOperationsHandler::free(Device *device, GraphicsAllocation &gfxAllocation) { + auto lock = acquireLock(resourcesLock); + auto itor = std::find(residentAllocations.begin(), residentAllocations.end(), &gfxAllocation); + if (itor != residentAllocations.end()) { + residentAllocations.erase(itor, itor + 1); + } + return MemoryOperationsStatus::success; +} MemoryOperationsStatus AubMemoryOperationsHandler::makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) { return makeResident(nullptr, gfxAllocations); @@ -141,32 +149,10 @@ DeviceBitfield AubMemoryOperationsHandler::getMemoryBanksBitfield(GraphicsAlloca return {}; } -void AubMemoryOperationsHandler::processFlushResidency(Device *device) { +void AubMemoryOperationsHandler::processFlushResidency(CommandStreamReceiver *csr) { + auto lock = acquireLock(resourcesLock); for (const auto &allocation : this->residentAllocations) { - if (!isAubWritable(*allocation, device)) { - continue; - } - - uint64_t gpuAddress = device->getGmmHelper()->decanonize(allocation->getGpuAddress()); - aub_stream::AllocationParams params(gpuAddress, - allocation->getUnderlyingBuffer(), - allocation->getUnderlyingBufferSize(), - allocation->storageInfo.getMemoryBanks(), - AubMemDump::DataTypeHintValues::TraceNotype, - allocation->getUsedPageSize()); - - auto gmm = allocation->getDefaultGmm(); - - if (gmm) { - params.additionalParams.compressionEnabled = gmm->isCompressionEnabled(); - params.additionalParams.uncached = CacheSettingsHelper::isUncachedType(gmm->resourceParams.Usage); - } - - if (allocation->storageInfo.cloningOfPageTables || !allocation->isAllocatedInLocalMemoryPool()) { - aubManager->writeMemory2(params); - } else { - device->getDefaultEngine().commandStreamReceiver->writeMemoryAub(params); - } + csr->writeMemory(*allocation); } } diff --git a/shared/source/os_interface/aub_memory_operations_handler.h b/shared/source/os_interface/aub_memory_operations_handler.h index 98568edbb0..b80f78450f 100644 --- a/shared/source/os_interface/aub_memory_operations_handler.h +++ b/shared/source/os_interface/aub_memory_operations_handler.h @@ -26,17 +26,18 @@ class AubMemoryOperationsHandler : public MemoryOperationsHandler { MemoryOperationsStatus lock(Device *device, ArrayRef gfxAllocations) override; MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override; MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override; + MemoryOperationsStatus free(Device *device, GraphicsAllocation &gfxAllocation) override; MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef gfxAllocations, bool evictable) override; MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override; + void processFlushResidency(CommandStreamReceiver *csr) override; + void setAubManager(aub_stream::AubManager *aubManager); bool isAubWritable(GraphicsAllocation &graphicsAllocation, Device *device) const; void setAubWritable(bool writable, GraphicsAllocation &graphicsAllocation, Device *device); - void processFlushResidency(Device *device); - protected: DeviceBitfield getMemoryBanksBitfield(GraphicsAllocation *allocation, Device *device) const; [[nodiscard]] MOCKABLE_VIRTUAL std::unique_lock acquireLock(SpinLock &lock) { diff --git a/shared/test/common/mocks/mock_aub_csr.h b/shared/test/common/mocks/mock_aub_csr.h index 53ef439a4c..9e351b2af1 100644 --- a/shared/test/common/mocks/mock_aub_csr.h +++ b/shared/test/common/mocks/mock_aub_csr.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2023 Intel Corporation + * Copyright (C) 2018-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -93,6 +93,10 @@ struct MockAubCsr : public AUBCommandStreamReceiverHw { AUBCommandStreamReceiverHw::writeMemory(gpuAddress, cpuAddress, size, memoryBank, entryBits); writeMemoryCalled = true; } + bool writeMemory(GraphicsAllocation &gfxAllocation) override { + writeMemoryGfxAllocCalled = true; + return AUBCommandStreamReceiverHw::writeMemory(gfxAllocation); + } void writeMMIO(uint32_t offset, uint32_t value) override { AUBCommandStreamReceiverHw::writeMMIO(offset, value); writeMMIOCalled = true; @@ -146,6 +150,7 @@ struct MockAubCsr : public AUBCommandStreamReceiverHw { bool initProgrammingFlagsCalled = false; bool initializeEngineCalled = false; bool writeMemoryCalled = false; + bool writeMemoryGfxAllocCalled = false; bool writeMemoryWithAubManagerCalled = false; bool writeMMIOCalled = false; bool submitBatchBufferCalled = false; diff --git a/shared/test/common/mocks/mock_aub_memory_operations_handler.h b/shared/test/common/mocks/mock_aub_memory_operations_handler.h index 42aa5d0f9e..9d014ffed7 100644 --- a/shared/test/common/mocks/mock_aub_memory_operations_handler.h +++ b/shared/test/common/mocks/mock_aub_memory_operations_handler.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -46,11 +46,17 @@ struct MockAubMemoryOperationsHandler : public AubMemoryOperationsHandler { return AubMemoryOperationsHandler::evictWithinOsContext(osContext, gfxAllocation); } + MemoryOperationsStatus free(Device *device, GraphicsAllocation &gfxAllocation) override { + freeCalled = true; + return AubMemoryOperationsHandler::free(device, gfxAllocation); + } + bool makeResidentCalled = false; bool evictCalled = false; bool isResidentCalled = false; bool makeResidentWithinOsContextCalled = false; bool evictWithinOsContextCalled = false; + bool freeCalled = false; }; } // namespace NEO diff --git a/shared/test/common/mocks/mock_memory_operations_handler.h b/shared/test/common/mocks/mock_memory_operations_handler.h index 3ebd6aaaeb..1b02d06f8c 100644 --- a/shared/test/common/mocks/mock_memory_operations_handler.h +++ b/shared/test/common/mocks/mock_memory_operations_handler.h @@ -102,9 +102,23 @@ class MockMemoryOperations : public MemoryOperationsHandler { return MemoryOperationsStatus::success; } + MemoryOperationsStatus free(Device *device, GraphicsAllocation &gfxAllocation) override { + freeCalledCount++; + + if (captureGfxAllocationsForMakeResident) { + auto itor = std::find(gfxAllocationsForMakeResident.begin(), gfxAllocationsForMakeResident.end(), &gfxAllocation); + if (itor != gfxAllocationsForMakeResident.end()) { + gfxAllocationsForMakeResident.erase(itor, itor + 1); + } + } + + return MemoryOperationsStatus::success; + } + std::vector gfxAllocationsForMakeResident{}; int makeResidentCalledCount = 0; int evictCalledCount = 0; + int freeCalledCount = 0; uint32_t isResidentCalledCount = 0; uint32_t lockCalledCount = 0; uint32_t makeResidentContextId = std::numeric_limits::max(); diff --git a/shared/test/common/mocks/mock_tbx_csr.h b/shared/test/common/mocks/mock_tbx_csr.h index 2660e114a4..d4a9c7e6fc 100644 --- a/shared/test/common/mocks/mock_tbx_csr.h +++ b/shared/test/common/mocks/mock_tbx_csr.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2023 Intel Corporation + * Copyright (C) 2018-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -42,6 +42,11 @@ class MockTbxCsr : public TbxCommandStreamReceiverHw { TbxCommandStreamReceiverHw::writeMemory(gpuAddress, cpuAddress, size, memoryBank, entryBits); writeMemoryCalled = true; } + bool writeMemory(GraphicsAllocation &graphicsAllocation) override { + writeMemoryGfxAllocCalled = true; + return TbxCommandStreamReceiverHw::writeMemory(graphicsAllocation); + } + void submitBatchBufferTbx(uint64_t batchBufferGpuAddress, const void *batchBuffer, size_t batchBufferSize, uint32_t memoryBank, uint64_t entryBits, bool overrideRingHead) override { TbxCommandStreamReceiverHw::submitBatchBufferTbx(batchBufferGpuAddress, batchBuffer, batchBufferSize, memoryBank, entryBits, overrideRingHead); overrideRingHeadPassed = overrideRingHead; @@ -62,6 +67,7 @@ class MockTbxCsr : public TbxCommandStreamReceiverHw { bool initializeEngineCalled = false; bool writeMemoryWithAubManagerCalled = false; bool writeMemoryCalled = false; + bool writeMemoryGfxAllocCalled = false; bool submitBatchBufferCalled = false; bool overrideRingHeadPassed = false; bool pollForCompletionCalled = false; diff --git a/shared/test/unit_test/command_stream/aub_command_stream_receiver_1_tests.cpp b/shared/test/unit_test/command_stream/aub_command_stream_receiver_1_tests.cpp index 7907045554..1fc272918a 100644 --- a/shared/test/unit_test/command_stream/aub_command_stream_receiver_1_tests.cpp +++ b/shared/test/unit_test/command_stream/aub_command_stream_receiver_1_tests.cpp @@ -20,12 +20,14 @@ #include "shared/test/common/mocks/mock_aub_center.h" #include "shared/test/common/mocks/mock_aub_csr.h" #include "shared/test/common/mocks/mock_aub_manager.h" +#include "shared/test/common/mocks/mock_aub_memory_operations_handler.h" #include "shared/test/common/mocks/mock_aub_subcapture_manager.h" #include "shared/test/common/mocks/mock_execution_environment.h" #include "shared/test/common/mocks/mock_gmm.h" #include "shared/test/common/mocks/mock_graphics_allocation.h" #include "shared/test/common/mocks/mock_os_context.h" #include "shared/test/common/test_macros/hw_test.h" + using namespace NEO; using AubCommandStreamReceiverTests = Test; @@ -336,6 +338,46 @@ HWTEST_F(AubCommandStreamReceiverTests, givenAubCommandStreamReceiverInSubCaptur EXPECT_FALSE(aubCsr->writeMemoryCalled); } +HWTEST_F(AubCommandStreamReceiverTests, givenAubCsrAndResidentAllocationWhenProcessResidencyIsCalledThenWriteMemoryIsCalledOnResidentAllocations) { + + auto mockManager = new MockAubManager(); + auto mockAubCenter = new MockAubCenter(*pDevice->getExecutionEnvironment()->rootDeviceEnvironments[0], false, "aubfile", CommandStreamReceiverType::aub); + mockAubCenter->aubManager.reset(mockManager); + pDevice->getExecutionEnvironment()->rootDeviceEnvironments[0]->aubCenter.reset(mockAubCenter); + + auto memoryOperationsHandler = new NEO::MockAubMemoryOperationsHandler(mockManager); + pDevice->getExecutionEnvironment()->rootDeviceEnvironments[0]->memoryOperationsInterface.reset(memoryOperationsHandler); + + auto commandStreamReceiver = std::make_unique>("", true, *pDevice->getExecutionEnvironment(), 0, pDevice->getDeviceBitfield()); + + auto osContext = pDevice->getExecutionEnvironment()->memoryManager->createAndRegisterOsContext(commandStreamReceiver.get(), + EngineDescriptorHelper::getDefaultDescriptor({getChosenEngineType(*defaultHwInfo), EngineUsage::regular}, + PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo))); + commandStreamReceiver->setupContext(*osContext); + commandStreamReceiver->initializeTagAllocation(); + commandStreamReceiver->createGlobalFenceAllocation(); + + MockGraphicsAllocation allocation(reinterpret_cast(0x1000), 0x1000); + ResidencyContainer allocationsForResidency = {&allocation}; + + MockGraphicsAllocation allocation2(reinterpret_cast(0x5000), 0x5000, 0x1000); + GraphicsAllocation *allocPtr = &allocation2; + memoryOperationsHandler->makeResident(pDevice, ArrayRef(&allocPtr, 1)); + EXPECT_TRUE(mockManager->writeMemory2Called); + + mockManager->storeAllocationParams = true; + commandStreamReceiver->writeMemoryGfxAllocCalled = false; + mockManager->writeMemory2Called = false; + + commandStreamReceiver->processResidency(allocationsForResidency, 0u); + EXPECT_TRUE(mockManager->writeMemory2Called); + EXPECT_TRUE(commandStreamReceiver->writeMemoryGfxAllocCalled); + ASSERT_EQ(2u, mockManager->storedAllocationParams.size()); + EXPECT_EQ(allocation2.getGpuAddress(), mockManager->storedAllocationParams[1].gfxAddress); + ASSERT_EQ(1u, memoryOperationsHandler->residentAllocations.size()); + EXPECT_EQ(&allocation2, memoryOperationsHandler->residentAllocations[0]); +} + HWTEST_F(AubCommandStreamReceiverTests, givenAubCommandStreamReceiverInSubCaptureModeWhenProcessResidencyIsCalledButAllocationSizeIsZeroThenItShouldntWriteMemory) { DebugManagerStateRestore stateRestore; AubSubCaptureCommon aubSubCaptureCommon; diff --git a/shared/test/unit_test/command_stream/tbx_command_stream_tests.cpp b/shared/test/unit_test/command_stream/tbx_command_stream_tests.cpp index 73a0fda376..e43fcdf8da 100644 --- a/shared/test/unit_test/command_stream/tbx_command_stream_tests.cpp +++ b/shared/test/unit_test/command_stream/tbx_command_stream_tests.cpp @@ -26,6 +26,7 @@ #include "shared/test/common/mocks/mock_allocation_properties.h" #include "shared/test/common/mocks/mock_aub_center.h" #include "shared/test/common/mocks/mock_aub_manager.h" +#include "shared/test/common/mocks/mock_aub_memory_operations_handler.h" #include "shared/test/common/mocks/mock_aub_subcapture_manager.h" #include "shared/test/common/mocks/mock_device.h" #include "shared/test/common/mocks/mock_execution_environment.h" @@ -344,6 +345,45 @@ HWTEST_F(TbxCommandSteamSimpleTest, givenTbxCsrWhenCallingMakeSurfacePackNonResi EXPECT_EQ(expectedAllocationsForDownload, tbxCsr.allocationsForDownload); } +HWTEST_F(TbxCommandSteamSimpleTest, givenTbxCsrAndResidentAllocationWhenProcessResidencyIsCalledThenWriteMemoryIsCalledOnResidentAllocations) { + auto mockManager = new MockAubManager(); + auto mockAubCenter = new MockAubCenter(*pDevice->getExecutionEnvironment()->rootDeviceEnvironments[0], false, "aubfile", CommandStreamReceiverType::aub); + mockAubCenter->aubManager.reset(mockManager); + pDevice->getExecutionEnvironment()->rootDeviceEnvironments[0]->aubCenter.reset(mockAubCenter); + + auto memoryOperationsHandler = new NEO::MockAubMemoryOperationsHandler(mockManager); + pDevice->getExecutionEnvironment()->rootDeviceEnvironments[0]->memoryOperationsInterface.reset(memoryOperationsHandler); + + auto commandStreamReceiver = std::make_unique>(*pDevice->getExecutionEnvironment(), pDevice->getDeviceBitfield()); + + auto osContext = pDevice->getExecutionEnvironment()->memoryManager->createAndRegisterOsContext(commandStreamReceiver.get(), + EngineDescriptorHelper::getDefaultDescriptor({getChosenEngineType(*defaultHwInfo), EngineUsage::regular}, + PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo))); + commandStreamReceiver->setupContext(*osContext); + commandStreamReceiver->initializeTagAllocation(); + commandStreamReceiver->createGlobalFenceAllocation(); + + MockGraphicsAllocation allocation(reinterpret_cast(0x1000), 0x1000); + ResidencyContainer allocationsForResidency = {&allocation}; + + MockGraphicsAllocation allocation2(reinterpret_cast(0x5000), 0x5000, 0x1000); + GraphicsAllocation *allocPtr = &allocation2; + memoryOperationsHandler->makeResident(pDevice, ArrayRef(&allocPtr, 1)); + EXPECT_TRUE(mockManager->writeMemory2Called); + + mockManager->storeAllocationParams = true; + commandStreamReceiver->writeMemoryGfxAllocCalled = false; + mockManager->writeMemory2Called = false; + + commandStreamReceiver->processResidency(allocationsForResidency, 0u); + EXPECT_TRUE(mockManager->writeMemory2Called); + EXPECT_TRUE(commandStreamReceiver->writeMemoryGfxAllocCalled); + ASSERT_EQ(2u, mockManager->storedAllocationParams.size()); + EXPECT_EQ(allocation2.getGpuAddress(), mockManager->storedAllocationParams[1].gfxAddress); + ASSERT_EQ(1u, memoryOperationsHandler->residentAllocations.size()); + EXPECT_EQ(&allocation2, memoryOperationsHandler->residentAllocations[0]); +} + HWTEST_F(TbxCommandSteamSimpleTest, givenTbxCsrWhenCallingWaitForTaskCountWithKmdNotifyFallbackThenTagAllocationAndScheduledAllocationsAreDownloaded) { MockTbxCsrRegisterDownloadedAllocations tbxCsr{*pDevice->executionEnvironment, pDevice->getRootDeviceIndex(), pDevice->getDeviceBitfield()}; MockOsContext osContext(0, EngineDescriptorHelper::getDefaultDescriptor(pDevice->getDeviceBitfield())); diff --git a/shared/test/unit_test/memory_manager/memory_manager_tests.cpp b/shared/test/unit_test/memory_manager/memory_manager_tests.cpp index b471ee45ba..ac0496a228 100644 --- a/shared/test/unit_test/memory_manager/memory_manager_tests.cpp +++ b/shared/test/unit_test/memory_manager/memory_manager_tests.cpp @@ -23,6 +23,7 @@ #include "shared/test/common/mocks/mock_allocation_properties.h" #include "shared/test/common/mocks/mock_aub_center.h" #include "shared/test/common/mocks/mock_aub_manager.h" +#include "shared/test/common/mocks/mock_aub_memory_operations_handler.h" #include "shared/test/common/mocks/mock_csr.h" #include "shared/test/common/mocks/mock_deferrable_deletion.h" #include "shared/test/common/mocks/mock_deferred_deleter.h" @@ -3163,6 +3164,37 @@ TEST(MemoryManagerTest, givenDuplicateRootDeviceIndicesWhenCreatingMultiGraphics memoryManager.freeGraphicsMemory(allocation); } +TEST(MemoryManagerTest, givenMemoryAllocationWhenFreedThenFreeCalledOnMemoryOperationsHandler) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + executionEnvironment.initGmm(); + auto mockManager = new MockAubManager(); + auto mockAubCenter = new MockAubCenter(*executionEnvironment.rootDeviceEnvironments[0], false, "aubfile", CommandStreamReceiverType::aub); + mockAubCenter->aubManager.reset(mockManager); + executionEnvironment.rootDeviceEnvironments[0]->aubCenter.reset(mockAubCenter); + executionEnvironment.incRefInternal(); + auto device = std::unique_ptr(MockDevice::createWithExecutionEnvironment(nullptr, &executionEnvironment, 0)); + + auto memoryOperationsHandler = new NEO::MockAubMemoryOperationsHandler(mockManager); + executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface.reset(memoryOperationsHandler); + + MockMemoryManager memoryManager(true, true, executionEnvironment); + + DeviceBitfield localMemoryBitfield{1}; + AllocationProperties allocationProperties{mockRootDeviceIndex, MemoryConstants::pageSize, AllocationType::buffer, localMemoryBitfield}; + + auto memoryAllocation = memoryManager.allocateGraphicsMemoryWithProperties(allocationProperties); + EXPECT_NE(nullptr, memoryAllocation); + + memoryOperationsHandler->makeResident(device.get(), ArrayRef(&memoryAllocation, 1)); + + EXPECT_EQ(1u, memoryOperationsHandler->residentAllocations.size()); + + memoryManager.freeGraphicsMemory(memoryAllocation); + + EXPECT_TRUE(memoryOperationsHandler->freeCalled); + EXPECT_EQ(0u, memoryOperationsHandler->residentAllocations.size()); +} + TEST(AllocationListTest, givenAllocationInListWhenFreeAllGraphicsAllocationsCalledThenHeadAndTailIsNullptr) { AllocationsList allocList; EXPECT_EQ(nullptr, allocList.peekHead()); diff --git a/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.cpp b/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.cpp index d27193d296..dff79b4fe1 100644 --- a/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.cpp +++ b/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.cpp @@ -11,9 +11,14 @@ #include "shared/source/aub_mem_dump/aub_mem_dump.h" #include "shared/test/common/helpers/engine_descriptor_helper.h" #include "shared/test/common/helpers/variable_backup.h" +#include "shared/test/common/mocks/mock_aub_center.h" +#include "shared/test/common/mocks/mock_aub_csr.h" #include "shared/test/common/mocks/mock_aub_manager.h" #include "shared/test/common/mocks/mock_command_stream_receiver.h" +#include "shared/test/common/mocks/mock_csr.h" #include "shared/test/common/mocks/mock_gmm.h" +#include "shared/test/common/mocks/mock_os_context.h" +#include "shared/test/common/test_macros/hw_test.h" TEST_F(AubMemoryOperationsHandlerTests, givenNullPtrAsAubManagerWhenMakeResidentCalledThenFalseReturned) { getMemoryOperationsHandler()->setAubManager(nullptr); @@ -426,90 +431,141 @@ TEST_F(AubMemoryOperationsHandlerTests, givenGfxAllocationWhenSetAubWritableForN EXPECT_TRUE(memoryOperationsInterface->isAubWritable(allocation, device.get())); } -TEST_F(AubMemoryOperationsHandlerTests, givenGfxAllocationWriteableWhenProcessingFlushResidencyThenPerformAllocationWriteMemory) { - MockAubManager aubManager; - getMemoryOperationsHandler()->setAubManager(&aubManager); +HWTEST_F(AubMemoryOperationsHandlerTests, givenCloningOfPageTablesWhenProcessingFlushResidencyThenPerformAUbManagerWriteMemory) { + auto mockAubManager = std::make_unique(); + auto *aubManager = mockAubManager.get(); auto memoryOperationsInterface = getMemoryOperationsHandler(); + memoryOperationsInterface->setAubManager(aubManager); + auto executionEnvironment = std::unique_ptr(MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u)); + executionEnvironment->rootDeviceEnvironments[0]->initGmm(); + executionEnvironment->rootDeviceEnvironments[0]->setHwInfoAndInitHelpers(&hardwareInfo); + executionEnvironment->initializeMemoryManager(); + auto mockAubCenter = std::make_unique(); + mockAubCenter->aubManager.reset(mockAubManager.release()); + executionEnvironment->rootDeviceEnvironments[0]->aubCenter.reset(mockAubCenter.release()); + auto csr = std::make_unique>("", true, *executionEnvironment, device->getRootDeviceIndex(), device->deviceBitfield); + auto osContext = executionEnvironment->memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor()); + csr->setupContext(*osContext); allocPtr->storageInfo.cloningOfPageTables = true; auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1)); EXPECT_EQ(result, MemoryOperationsStatus::success); - EXPECT_TRUE(aubManager.writeMemory2Called); + EXPECT_TRUE(aubManager->writeMemory2Called); - aubManager.writeMemory2Called = false; - - memoryOperationsInterface->processFlushResidency(device.get()); - EXPECT_TRUE(aubManager.writeMemory2Called); + aubManager->writeMemory2Called = false; + memoryOperationsInterface->processFlushResidency(csr.get()); + EXPECT_TRUE(aubManager->writeMemory2Called); } -TEST_F(AubMemoryOperationsHandlerTests, givenNonLocalGfxAllocationWriteableWhenProcessingFlushResidencyThenPerformAllocationAubManagerWriteMemory) { - MockAubManager aubManager; - getMemoryOperationsHandler()->setAubManager(&aubManager); +HWTEST_F(AubMemoryOperationsHandlerTests, givenNonLocalGfxAllocationWriteableWhenProcessingFlushResidencyThenPerformAubManagerWriteMemory) { + auto mockAubManager = std::make_unique(); + auto *aubManager = mockAubManager.get(); auto memoryOperationsInterface = getMemoryOperationsHandler(); + memoryOperationsInterface->setAubManager(aubManager); + auto executionEnvironment = std::unique_ptr(MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u)); + executionEnvironment->rootDeviceEnvironments[0]->initGmm(); + executionEnvironment->rootDeviceEnvironments[0]->setHwInfoAndInitHelpers(&hardwareInfo); + executionEnvironment->initializeMemoryManager(); + auto mockAubCenter = std::make_unique(); + mockAubCenter->aubManager.reset(mockAubManager.release()); + executionEnvironment->rootDeviceEnvironments[0]->aubCenter.reset(mockAubCenter.release()); + auto csr = std::make_unique>("", true, *executionEnvironment, device->getRootDeviceIndex(), device->deviceBitfield); + auto osContext = executionEnvironment->memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor()); + csr->setupContext(*osContext); + MemoryAllocation allocation(0, AllocationType::unknown, nullptr, reinterpret_cast(0x1000), 0x1000u, + MemoryConstants::pageSize, 0, MemoryPool::system4KBPages, false, false, MemoryManager::maxOsContextCount); + allocPtr = &allocation; allocPtr->storageInfo.cloningOfPageTables = false; auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1)); EXPECT_EQ(result, MemoryOperationsStatus::success); - EXPECT_TRUE(aubManager.writeMemory2Called); + EXPECT_TRUE(aubManager->writeMemory2Called); - aubManager.writeMemory2Called = false; + aubManager->writeMemory2Called = false; - memoryOperationsInterface->processFlushResidency(device.get()); - EXPECT_TRUE(aubManager.writeMemory2Called); + memoryOperationsInterface->processFlushResidency(csr.get()); + EXPECT_TRUE(aubManager->writeMemory2Called); } -TEST_F(AubMemoryOperationsHandlerTests, givenLocalGfxAllocationWriteableWhenProcessingFlushResidencyThenPerformAllocationDefaultCsrWriteMemory) { +HWTEST_F(AubMemoryOperationsHandlerTests, givenLocalGfxAllocationWriteableWhenProcessingFlushResidencyThenPerformAllocationDefaultCsrWriteMemory) { + auto mockAubManager = std::make_unique(); + auto *aubManager = mockAubManager.get(); + auto memoryOperationsInterface = getMemoryOperationsHandler(); + memoryOperationsInterface->setAubManager(aubManager); + auto executionEnvironment = std::unique_ptr(MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u)); + executionEnvironment->rootDeviceEnvironments[0]->initGmm(); + executionEnvironment->rootDeviceEnvironments[0]->setHwInfoAndInitHelpers(&hardwareInfo); + executionEnvironment->initializeMemoryManager(); + auto mockAubCenter = std::make_unique(); + mockAubCenter->aubManager.reset(mockAubManager.release()); + executionEnvironment->rootDeviceEnvironments[0]->aubCenter.reset(mockAubCenter.release()); + auto csr = std::make_unique>("", true, *executionEnvironment, device->getRootDeviceIndex(), device->deviceBitfield); + auto osContext = executionEnvironment->memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor()); + csr->setupContext(*osContext); + csr->localMemoryEnabled = true; + MemoryAllocation allocation(0, AllocationType::unknown, nullptr, reinterpret_cast(0x1000), 0x1000u, MemoryConstants::pageSize, 0, MemoryPool::localMemory, false, false, MemoryManager::maxOsContextCount); allocPtr = &allocation; allocPtr->storageInfo.cloningOfPageTables = false; - DeviceBitfield deviceBitfield(1); - const auto csr = std::make_unique(*device->getExecutionEnvironment(), 0, deviceBitfield); - csr->localMemoryEnabled = true; - auto backupDefaultCsr = std::make_unique>(&device->getDefaultEngine().commandStreamReceiver); device->getDefaultEngine().commandStreamReceiver = csr.get(); - MockAubManager aubManager; - getMemoryOperationsHandler()->setAubManager(&aubManager); - auto memoryOperationsInterface = getMemoryOperationsHandler(); auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1)); EXPECT_EQ(result, MemoryOperationsStatus::success); - EXPECT_FALSE(aubManager.writeMemory2Called); - EXPECT_EQ(1u, csr->writeMemoryAubCalled); + EXPECT_FALSE(aubManager->writeMemory2Called); - memoryOperationsInterface->processFlushResidency(device.get()); - EXPECT_FALSE(aubManager.writeMemory2Called); - EXPECT_EQ(2u, csr->writeMemoryAubCalled); + auto mockHwContext0 = static_cast(csr->hardwareContextController->hardwareContexts[0].get()); + + EXPECT_TRUE(mockHwContext0->writeMemory2Called); + + mockHwContext0->writeMemory2Called = false; + memoryOperationsInterface->processFlushResidency(csr.get()); + EXPECT_FALSE(aubManager->writeMemory2Called); + EXPECT_TRUE(mockHwContext0->writeMemory2Called); } -TEST_F(AubMemoryOperationsHandlerTests, givenGfxAllocationWriteableWithGmmPresentWhenProcessingFlushResidencyThenPerformAllocationWriteMemoryAndReflectGmmFlags) { - MockAubManager aubManager; - getMemoryOperationsHandler()->setAubManager(&aubManager); +HWTEST_F(AubMemoryOperationsHandlerTests, givenGfxAllocationWriteableWithGmmPresentWhenProcessingFlushResidencyThenPerformAllocationWriteMemoryAndReflectGmmFlags) { + auto mockAubManager = std::make_unique(); + auto *aubManager = mockAubManager.get(); + getMemoryOperationsHandler()->setAubManager(aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); allocPtr->storageInfo.cloningOfPageTables = true; auto executionEnvironment = std::unique_ptr(MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u)); executionEnvironment->rootDeviceEnvironments[0]->initGmm(); + executionEnvironment->rootDeviceEnvironments[0]->setHwInfoAndInitHelpers(&hardwareInfo); + executionEnvironment->initializeMemoryManager(); + MockGmm gmm(executionEnvironment->rootDeviceEnvironments[0]->getGmmHelper()); gmm.setCompressionEnabled(true); allocPtr->setDefaultGmm(&gmm); + auto mockAubCenter = std::make_unique(); + mockAubCenter->aubManager.reset(mockAubManager.release()); + + executionEnvironment->rootDeviceEnvironments[0]->aubCenter.reset(mockAubCenter.release()); + auto result = memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocPtr, 1)); EXPECT_EQ(result, MemoryOperationsStatus::success); - aubManager.writeMemory2Called = false; - aubManager.storeAllocationParams = true; + aubManager->writeMemory2Called = false; + aubManager->storeAllocationParams = true; - memoryOperationsInterface->processFlushResidency(device.get()); - EXPECT_TRUE(aubManager.writeMemory2Called); - EXPECT_EQ(1u, aubManager.storedAllocationParams.size()); - EXPECT_TRUE(aubManager.storedAllocationParams[0].additionalParams.compressionEnabled); - EXPECT_FALSE(aubManager.storedAllocationParams[0].additionalParams.uncached); + auto csr = std::make_unique>("", true, *executionEnvironment, device->getRootDeviceIndex(), device->deviceBitfield); + auto osContext = executionEnvironment->memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor()); + csr->setupContext(*osContext); + + memoryOperationsInterface->processFlushResidency(csr.get()); + + EXPECT_TRUE(aubManager->writeMemory2Called); + EXPECT_EQ(1u, aubManager->storedAllocationParams.size()); + EXPECT_TRUE(aubManager->storedAllocationParams[0].additionalParams.compressionEnabled); + EXPECT_FALSE(aubManager->storedAllocationParams[0].additionalParams.uncached); } TEST_F(AubMemoryOperationsHandlerTests, givenDeviceNullWhenProcessingFlushResidencyThenAllocationIsNotWriteable) { @@ -533,6 +589,8 @@ TEST_F(AubMemoryOperationsHandlerTests, givenDeviceNullWhenProcessingFlushReside aubManager.writeMemory2Called = false; - memoryOperationsInterface->processFlushResidency(nullptr); + auto csr = std::make_unique(*device->getExecutionEnvironment(), 0, device->deviceBitfield); + + memoryOperationsInterface->processFlushResidency(csr.get()); EXPECT_FALSE(aubManager.writeMemory2Called); } diff --git a/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.h b/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.h index b307772bee..3441f7b516 100644 --- a/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.h +++ b/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2023 Intel Corporation + * Copyright (C) 2019-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -36,7 +36,7 @@ class AubMemoryOperationsHandlerTests : public ::testing::Test { return residencyHandler.get(); } - MockGraphicsAllocation allocation; + MockGraphicsAllocation allocation{reinterpret_cast(0x1000), 0x1000u, MemoryConstants::pageSize}; GraphicsAllocation *allocPtr; DebugManagerStateRestore dbgRestore; HardwareInfo hardwareInfo = {};