diff --git a/level_zero/core/source/cmdqueue/cmdqueue.cpp b/level_zero/core/source/cmdqueue/cmdqueue.cpp index 8b0000e374..05b423fdbe 100644 --- a/level_zero/core/source/cmdqueue/cmdqueue.cpp +++ b/level_zero/core/source/cmdqueue/cmdqueue.cpp @@ -62,7 +62,7 @@ void CommandQueueImp::submitBatchBuffer(size_t offset, NEO::ResidencyContainer & NEO::QueueThrottle::HIGH, NEO::QueueSliceCount::defaultSliceCount, commandStream->getUsed(), commandStream, endingCmdPtr, false); - csr->submitBatchBuffer(batchBuffer, residencyContainer); + csr->submitBatchBuffer(batchBuffer, csr->getResidencyAllocations()); buffers.setCurrentFlushStamp(csr->obtainCurrentFlushStamp()); } diff --git a/level_zero/core/source/cmdqueue/cmdqueue_hw.h b/level_zero/core/source/cmdqueue/cmdqueue_hw.h index c17d49d76d..779b82a840 100644 --- a/level_zero/core/source/cmdqueue/cmdqueue_hw.h +++ b/level_zero/core/source/cmdqueue/cmdqueue_hw.h @@ -46,8 +46,7 @@ struct CommandQueueHw : public CommandQueueImp { size_t estimatePipelineSelect(); void programPipelineSelect(NEO::LinearStream &commandStream); - MOCKABLE_VIRTUAL void handleScratchSpace(NEO::ResidencyContainer &residency, - NEO::HeapContainer &heapContainer, + MOCKABLE_VIRTUAL void handleScratchSpace(NEO::HeapContainer &heapContainer, NEO::ScratchSpaceController *scratchController, bool &gsbaState, bool &frontEndState, uint32_t perThreadScratchSpaceSize); diff --git a/level_zero/core/source/cmdqueue/cmdqueue_hw.inl b/level_zero/core/source/cmdqueue/cmdqueue_hw.inl index 177bcc8d89..4a7c1f95ca 100644 --- a/level_zero/core/source/cmdqueue/cmdqueue_hw.inl +++ b/level_zero/core/source/cmdqueue/cmdqueue_hw.inl @@ -183,13 +183,12 @@ ze_result_t CommandQueueHw::executeCommandLists( spaceForResidency += residencyContainerSpaceForTagWrite; - residencyContainer.reserve(spaceForResidency); + csr->getResidencyAllocations().reserve(spaceForResidency); auto scratchSpaceController = csr->getScratchSpaceController(); bool gsbaStateDirty = false; bool frontEndStateDirty = false; - handleScratchSpace(residencyContainer, - heapContainer, + handleScratchSpace(heapContainer, scratchSpaceController, gsbaStateDirty, frontEndStateDirty, perThreadScratchSpaceSize); @@ -223,18 +222,18 @@ ze_result_t CommandQueueHw::executeCommandLists( const auto globalFenceAllocation = csr->getGlobalFenceAllocation(); if (globalFenceAllocation) { - residencyContainer.push_back(globalFenceAllocation); + csr->makeResident(*globalFenceAllocation); } const auto workPartitionAllocation = csr->getWorkPartitionAllocation(); if (workPartitionAllocation) { - residencyContainer.push_back(workPartitionAllocation); + csr->makeResident(*workPartitionAllocation); } if (NEO::DebugManager.flags.EnableSWTags.get()) { NEO::SWTagsManager *tagsManager = neoDevice->getRootDeviceEnvironment().tagsManager.get(); UNRECOVERABLE_IF(tagsManager == nullptr); - residencyContainer.push_back(tagsManager->getBXMLHeapAllocation()); - residencyContainer.push_back(tagsManager->getSWTagHeapAllocation()); + csr->makeResident(*tagsManager->getBXMLHeapAllocation()); + csr->makeResident(*tagsManager->getSWTagHeapAllocation()); tagsManager->insertBXMLHeapAddress(child); tagsManager->insertSWTagHeapAddress(child); } @@ -242,7 +241,7 @@ ze_result_t CommandQueueHw::executeCommandLists( csr->programHardwareContext(child); if (NEO::Debugger::isDebugEnabled(internalUsage) && device->getL0Debugger()) { - residencyContainer.push_back(device->getL0Debugger()->getSbaTrackingBuffer(csr->getOsContext().getContextId())); + csr->makeResident(*device->getL0Debugger()->getSbaTrackingBuffer(csr->getOsContext().getContextId())); } if (!isCopyOnlyCommandQueue) { @@ -281,17 +280,17 @@ ze_result_t CommandQueueHw::executeCommandLists( (neoDevice->getDebugger() != nullptr && NEO::Debugger::isDebugEnabled(internalUsage)); if (devicePreemption == NEO::PreemptionMode::MidThread) { - residencyContainer.push_back(csr->getPreemptionAllocation()); + csr->makeResident(*csr->getPreemptionAllocation()); } if (sipKernelUsed) { auto sipIsa = NEO::SipKernel::getSipKernel(*neoDevice).getSipAllocation(); - residencyContainer.push_back(sipIsa); + csr->makeResident(*sipIsa); } if (NEO::Debugger::isDebugEnabled(internalUsage) && neoDevice->getDebugger()) { UNRECOVERABLE_IF(device->getDebugSurface() == nullptr); - residencyContainer.push_back(device->getDebugSurface()); + csr->makeResident(*device->getDebugSurface()); } } @@ -355,9 +354,9 @@ ze_result_t CommandQueueHw::executeCommandLists( commandList->getPrintfFunctionContainer().end()); for (auto alloc : commandList->commandContainer.getResidencyContainer()) { - if (residencyContainer.end() == - std::find(residencyContainer.begin(), residencyContainer.end(), alloc)) { - residencyContainer.push_back(alloc); + if (csr->getResidencyAllocations().end() == + std::find(csr->getResidencyAllocations().begin(), csr->getResidencyAllocations().end(), alloc)) { + csr->makeResident(*alloc); if (performMigration) { if (alloc && @@ -385,7 +384,7 @@ ze_result_t CommandQueueHw::executeCommandLists( commandQueuePreemptionMode = statePreemption; if (hFence) { - residencyContainer.push_back(&fence->getAllocation()); + csr->makeResident(fence->getAllocation()); if (isCopyOnlyCommandQueue) { NEO::EncodeMiFlushDW::programMiFlushDw(child, fence->getGpuAddress(), Fence::STATE_SIGNALED, false, true); } else { @@ -401,7 +400,7 @@ ze_result_t CommandQueueHw::executeCommandLists( dispatchTaskCountWrite(child, true); - residencyContainer.push_back(csr->getTagAllocation()); + csr->makeResident(*csr->getTagAllocation()); void *endingCmd = nullptr; if (directSubmissionEnabled) { endingCmd = child.getSpace(0); @@ -417,17 +416,16 @@ ze_result_t CommandQueueHw::executeCommandLists( memset(paddingPtr, 0, padding); } - submitBatchBuffer(ptrDiff(child.getCpuBase(), commandStream->getCpuBase()), residencyContainer, endingCmd); + submitBatchBuffer(ptrDiff(child.getCpuBase(), commandStream->getCpuBase()), csr->getResidencyAllocations(), endingCmd); this->taskCount = csr->peekTaskCount(); - csr->makeSurfacePackNonResident(residencyContainer); + csr->makeSurfacePackNonResident(csr->getResidencyAllocations()); if (getSynchronousMode() == ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS) { this->synchronize(std::numeric_limits::max()); } - this->residencyContainer.clear(); this->heapContainer.clear(); return ZE_RESULT_SUCCESS; diff --git a/level_zero/core/source/cmdqueue/cmdqueue_hw_base.inl b/level_zero/core/source/cmdqueue/cmdqueue_hw_base.inl index 4be87424eb..d15904efdf 100644 --- a/level_zero/core/source/cmdqueue/cmdqueue_hw_base.inl +++ b/level_zero/core/source/cmdqueue/cmdqueue_hw_base.inl @@ -104,8 +104,7 @@ size_t CommandQueueHw::estimateStateBaseAddressCmdSize() { } template -void CommandQueueHw::handleScratchSpace(NEO::ResidencyContainer &residency, - NEO::HeapContainer &heapContainer, +void CommandQueueHw::handleScratchSpace(NEO::HeapContainer &heapContainer, NEO::ScratchSpaceController *scratchController, bool &gsbaState, bool &frontEndState, uint32_t perThreadScratchSpaceSize) { @@ -114,7 +113,7 @@ void CommandQueueHw::handleScratchSpace(NEO::ResidencyContainer & scratchController->setRequiredScratchSpace(nullptr, 0u, perThreadScratchSpaceSize, 0u, csr->peekTaskCount(), csr->getOsContext(), gsbaState, frontEndState); auto scratchAllocation = scratchController->getScratchSpaceAllocation(); - residency.push_back(scratchAllocation); + csr->makeResident(*scratchAllocation); } } diff --git a/level_zero/core/source/cmdqueue/cmdqueue_imp.h b/level_zero/core/source/cmdqueue/cmdqueue_imp.h index 2a43bcee85..35fd7bec36 100644 --- a/level_zero/core/source/cmdqueue/cmdqueue_imp.h +++ b/level_zero/core/source/cmdqueue/cmdqueue_imp.h @@ -96,7 +96,6 @@ struct CommandQueueImp : public CommandQueue { std::vector printfFunctionContainer; bool gpgpuEnabled = false; CommandBufferManager buffers; - NEO::ResidencyContainer residencyContainer; NEO::HeapContainer heapContainer; }; diff --git a/level_zero/core/test/unit_tests/mocks/mock_cmdqueue.h b/level_zero/core/test/unit_tests/mocks/mock_cmdqueue.h index 1b3635b99b..80a6cdee2e 100644 --- a/level_zero/core/test/unit_tests/mocks/mock_cmdqueue.h +++ b/level_zero/core/test/unit_tests/mocks/mock_cmdqueue.h @@ -25,6 +25,7 @@ struct WhiteBox<::L0::CommandQueue> : public ::L0::CommandQueueImp { using BaseClass::device; using BaseClass::preemptionCmdSyncProgramming; using BaseClass::printfFunctionContainer; + using BaseClass::submitBatchBuffer; using BaseClass::synchronizeByPollingForTaskCount; using CommandQueue::commandQueuePreemptionMode; using CommandQueue::internalUsage; @@ -85,6 +86,7 @@ struct MockCommandQueueHw : public L0::CommandQueueHw { using BaseClass::printfFunctionContainer; using L0::CommandQueue::internalUsage; using L0::CommandQueue::preemptionCmdSyncProgramming; + using L0::CommandQueueImp::csr; MockCommandQueueHw(L0::Device *device, NEO::CommandStreamReceiver *csr, const ze_command_queue_desc_t *desc) : L0::CommandQueueHw(device, csr, desc) { } diff --git a/level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue.cpp b/level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue.cpp index 2279ee64b9..ca2cda218b 100644 --- a/level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue.cpp @@ -217,6 +217,24 @@ HWTEST_F(CommandQueueCreate, given100CmdListsWhenExecutingThenCommandStreamIsNot commandQueue->destroy(); } +HWTEST_F(CommandQueueCreate, givenContainerWithAllocationsWhenResidencyContainerIsEmptyThenMakeResidentWasNotCalled) { + auto csr = std::make_unique(*neoDevice->getExecutionEnvironment(), 0, neoDevice->getDeviceBitfield()); + csr->setupContext(*neoDevice->getDefaultEngine().osContext); + const ze_command_queue_desc_t desc = {}; + ze_result_t returnValue; + auto commandQueue = whitebox_cast(CommandQueue::create(productFamily, + device, + csr.get(), + &desc, + false, + false, + returnValue)); + ResidencyContainer container; + commandQueue->submitBatchBuffer(0, container, nullptr); + EXPECT_EQ(csr->makeResidentCalledTimes, 0u); + commandQueue->destroy(); +} + TEST_F(CommandQueueCreate, whenCommandQueueCreatedThenExpectLinearStreamInitializedWithExpectedSize) { const ze_command_queue_desc_t desc = {}; ze_result_t returnValue; @@ -580,7 +598,19 @@ HWTEST_F(CommandQueueCommandsSingleTile, givenCommandQueueWhenExecutingCommandLi } HWTEST_F(CommandQueueCommandsMultiTile, givenCommandQueueOnMultiTileWhenExecutingCommandListsThenWorkPartitionAllocationIsMadeResident) { - MockCsrHw2 csr(*neoDevice->getExecutionEnvironment(), 0, neoDevice->getDeviceBitfield()); + class MyCsrMock : public MockCsrHw2 { + using MockCsrHw2::MockCsrHw2; + + public: + void makeResident(GraphicsAllocation &graphicsAllocation) override { + if (expectedGa == &graphicsAllocation) { + expectedGAWasMadeResident = true; + } + } + GraphicsAllocation *expectedGa = nullptr; + bool expectedGAWasMadeResident = false; + }; + MyCsrMock csr(*neoDevice->getExecutionEnvironment(), 0, neoDevice->getDeviceBitfield()); csr.initializeTagAllocation(); csr.createWorkPartitionAllocation(*neoDevice); csr.setupContext(*neoDevice->getDefaultEngine().osContext); @@ -597,12 +627,13 @@ HWTEST_F(CommandQueueCommandsMultiTile, givenCommandQueueOnMultiTileWhenExecutin std::unique_ptr commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::Compute, returnValue)); auto commandListHandle = commandList->toHandle(); + auto workPartitionAllocation = csr.getWorkPartitionAllocation(); + csr.expectedGa = workPartitionAllocation; auto status = commandQueue->executeCommandLists(1, &commandListHandle, nullptr, false); EXPECT_EQ(status, ZE_RESULT_SUCCESS); - auto workPartitionAllocation = csr.getWorkPartitionAllocation(); ASSERT_NE(nullptr, workPartitionAllocation); - EXPECT_TRUE(isAllocationInResidencyContainer(csr, workPartitionAllocation)); + EXPECT_TRUE(csr.expectedGAWasMadeResident); commandQueue->destroy(); } @@ -877,12 +908,11 @@ class MockCommandQueue : public L0::CommandQueueHw { MockCommandQueue(L0::Device *device, NEO::CommandStreamReceiver *csr, const ze_command_queue_desc_t *desc) : L0::CommandQueueHw(device, csr, desc) {} using BaseClass = ::L0::CommandQueueHw; + using BaseClass::csr; using BaseClass::heapContainer; - using BaseClass::residencyContainer; NEO::HeapContainer mockHeapContainer; - void handleScratchSpace(NEO::ResidencyContainer &residency, - NEO::HeapContainer &heapContainer, + void handleScratchSpace(NEO::HeapContainer &heapContainer, NEO::ScratchSpaceController *scratchController, bool &gsbaState, bool &frontEndState, uint32_t perThreadScratchSpaceSize) override { @@ -966,7 +996,7 @@ HWTEST2_F(ExecuteCommandListTests, givenExecuteCommandListWhenItReturnsThenConta commandQueue->executeCommandLists(1, &commandListHandle, nullptr, false); - EXPECT_EQ(0u, commandQueue->residencyContainer.size()); + EXPECT_EQ(0u, commandQueue->csr->getResidencyAllocations().size()); EXPECT_EQ(0u, commandQueue->heapContainer.size()); commandQueue->destroy(); diff --git a/opencl/test/unit_test/command_stream/command_stream_receiver_flush_task_2_tests.cpp b/opencl/test/unit_test/command_stream/command_stream_receiver_flush_task_2_tests.cpp index bd223bfffa..9eb8ec333d 100644 --- a/opencl/test/unit_test/command_stream/command_stream_receiver_flush_task_2_tests.cpp +++ b/opencl/test/unit_test/command_stream/command_stream_receiver_flush_task_2_tests.cpp @@ -552,7 +552,7 @@ struct MockScratchController : public ScratchSpaceController { OsContext &osContext, bool &stateBaseAddressDirty, bool &vfeStateDirty, - NEO::ResidencyContainer &residency) override { + NEO::CommandStreamReceiver *csr) override { } void reserveHeap(IndirectHeap::Type heapType, IndirectHeap *&indirectHeap) override{}; }; diff --git a/shared/source/command_stream/scratch_space_controller.h b/shared/source/command_stream/scratch_space_controller.h index 8893772186..4c8e0c0938 100644 --- a/shared/source/command_stream/scratch_space_controller.h +++ b/shared/source/command_stream/scratch_space_controller.h @@ -20,6 +20,7 @@ class InternalAllocationStorage; class MemoryManager; struct HardwareInfo; class OsContext; +class CommandStreamReceiver; namespace ScratchSpaceConstants { constexpr size_t scratchSpaceOffsetFor64Bit = 4096u; @@ -69,7 +70,7 @@ class ScratchSpaceController { OsContext &osContext, bool &stateBaseAddressDirty, bool &vfeStateDirty, - ResidencyContainer &residency) = 0; + CommandStreamReceiver *csr) = 0; protected: MemoryManager *getMemoryManager() const; diff --git a/shared/source/command_stream/scratch_space_controller_base.cpp b/shared/source/command_stream/scratch_space_controller_base.cpp index b9b2c1ef31..d90afda3db 100644 --- a/shared/source/command_stream/scratch_space_controller_base.cpp +++ b/shared/source/command_stream/scratch_space_controller_base.cpp @@ -98,6 +98,6 @@ void ScratchSpaceControllerBase::programBindlessSurfaceStateForScratch(BindlessH OsContext &osContext, bool &stateBaseAddressDirty, bool &vfeStateDirty, - ResidencyContainer &residency) { + NEO::CommandStreamReceiver *csr) { } } // namespace NEO diff --git a/shared/source/command_stream/scratch_space_controller_base.h b/shared/source/command_stream/scratch_space_controller_base.h index ebbd9ae5bc..26a548830a 100644 --- a/shared/source/command_stream/scratch_space_controller_base.h +++ b/shared/source/command_stream/scratch_space_controller_base.h @@ -42,7 +42,7 @@ class ScratchSpaceControllerBase : public ScratchSpaceController { OsContext &osContext, bool &stateBaseAddressDirty, bool &vfeStateDirty, - ResidencyContainer &residency) override; + NEO::CommandStreamReceiver *csr) override; protected: void createScratchSpaceAllocation(); diff --git a/shared/test/common/mocks/mock_command_stream_receiver.h b/shared/test/common/mocks/mock_command_stream_receiver.h index 1208884af8..0585699365 100644 --- a/shared/test/common/mocks/mock_command_stream_receiver.h +++ b/shared/test/common/mocks/mock_command_stream_receiver.h @@ -111,6 +111,9 @@ class MockCommandStreamReceiver : public CommandStreamReceiver { } GraphicsAllocation *getClearColorAllocation() override { return nullptr; } + void makeResident(GraphicsAllocation &gfxAllocation) override { + makeResidentCalledTimes++; + } void postInitFlagsSetup() override {} @@ -118,6 +121,7 @@ class MockCommandStreamReceiver : public CommandStreamReceiver { int *flushBatchedSubmissionsCallCounter = nullptr; uint32_t waitForCompletionWithTimeoutCalled = 0; uint32_t mockTagAddress = 0; + uint32_t makeResidentCalledTimes = 0; bool multiOsContextCapable = false; bool memoryCompressionEnabled = false; bool downloadAllocationsCalled = false; diff --git a/shared/test/unit_test/scratch_space_controler/scratch_space_controler_tests.cpp b/shared/test/unit_test/scratch_space_controler/scratch_space_controler_tests.cpp index 74b299fb31..4ed9e9e52b 100644 --- a/shared/test/unit_test/scratch_space_controler/scratch_space_controler_tests.cpp +++ b/shared/test/unit_test/scratch_space_controler/scratch_space_controler_tests.cpp @@ -37,8 +37,8 @@ class MockScratchSpaceControllerBase : public ScratchSpaceControllerBase { OsContext &osContext, bool &stateBaseAddressDirty, bool &vfeStateDirty, - NEO::ResidencyContainer &residency) override { - ScratchSpaceControllerBase::programBindlessSurfaceStateForScratch(heapsHelper, requiredPerThreadScratchSize, requiredPerThreadPrivateScratchSize, currentTaskCount, osContext, stateBaseAddressDirty, vfeStateDirty, residencyContainer); + NEO::CommandStreamReceiver *csr) override { + ScratchSpaceControllerBase::programBindlessSurfaceStateForScratch(heapsHelper, requiredPerThreadScratchSize, requiredPerThreadPrivateScratchSize, currentTaskCount, osContext, stateBaseAddressDirty, vfeStateDirty, csr); programBindlessSurfaceStateForScratchCalled = true; } ResidencyContainer residencyContainer; @@ -79,7 +79,7 @@ HWTEST_F(ScratchComtrolerTests, givenCommandQueueWhenProgramHeapBindlessCalledTh bool gsbaStateDirty = false; bool frontEndStateDirty = false; HeapContainer heapContainer; - scratchController->programBindlessSurfaceStateForScratch(nullptr, 0, 0, 0, *pDevice->getDefaultEngine().osContext, gsbaStateDirty, frontEndStateDirty, scratchController->residencyContainer); + scratchController->programBindlessSurfaceStateForScratch(nullptr, 0, 0, 0, *pDevice->getDefaultEngine().osContext, gsbaStateDirty, frontEndStateDirty, &csr); EXPECT_TRUE(static_cast(scratchController.get())->programBindlessSurfaceStateForScratchCalled); } \ No newline at end of file