From acb044dce3b28a0fab5fbe78b00be65160dd6bff Mon Sep 17 00:00:00 2001 From: "Mrozek, Michal" Date: Thu, 15 Feb 2018 08:29:57 +0100 Subject: [PATCH] Fix DC flush programming in non concurrent scenarios. -If out of order flag was disabled then pipe control was not having dc flush. -This could led to a batch buffer that doesn't end with dc flush. -This change adds differentiation between pipe controls that may be erased and pipe controls that are used as a part of epilogue command Change-Id: Ic9c970c75c89ff524a0e40506eff6dd097760145 --- .../command_stream_receiver_hw.inl | 26 +++++--- .../command_stream/submissions_aggregator.h | 3 +- .../command_queue/enqueue_kernel_tests.cpp | 13 ++-- .../command_stream_receiver_hw_tests.cpp | 62 ++++++++++++++----- .../linux/drm_command_stream_tests.cpp | 1 + .../windows/device_command_stream_tests.cpp | 1 + 6 files changed, 74 insertions(+), 32 deletions(-) diff --git a/runtime/command_stream/command_stream_receiver_hw.inl b/runtime/command_stream/command_stream_receiver_hw.inl index c8cfb4fbba..4a47bbc819 100644 --- a/runtime/command_stream/command_stream_receiver_hw.inl +++ b/runtime/command_stream/command_stream_receiver_hw.inl @@ -118,6 +118,7 @@ CompletionStamp CommandStreamReceiverHw::flushTask( auto levelClosed = false; void *currentPipeControlForNooping = nullptr; + void *epiloguePipeControlLocation = nullptr; Device *device = this->getMemoryManager()->device; if (dispatchFlags.blocking || dispatchFlags.dcFlush || dispatchFlags.guardCommandBufferWithPipeControl) { @@ -131,8 +132,10 @@ CompletionStamp CommandStreamReceiverHw::flushTask( } } + epiloguePipeControlLocation = ptrOffset(commandStreamTask.getBase(), commandStreamTask.getUsed()); + if (dispatchFlags.outOfOrderExecutionAllowed && !dispatchFlags.dcFlush) { - currentPipeControlForNooping = ptrOffset(commandStreamTask.getBase(), commandStreamTask.getUsed()); + currentPipeControlForNooping = epiloguePipeControlLocation; } //Some architectures (SKL) requires to have pipe control prior to pipe control with tag write, add it here @@ -343,7 +346,8 @@ CompletionStamp CommandStreamReceiverHw::flushTask( commandBuffer->batchBufferEndLocation = bbEndLocation; commandBuffer->taskCount = this->taskCount + 1; commandBuffer->flushStamp->replaceStampObject(dispatchFlags.flushStampReference); - commandBuffer->pipeControlLocation = currentPipeControlForNooping; + commandBuffer->pipeControlThatMayBeErasedLocation = currentPipeControlForNooping; + commandBuffer->epiloguePipeControlLocation = epiloguePipeControlLocation; this->submissionAggregator->recordCommandBuffer(commandBuffer); } } else { @@ -392,7 +396,8 @@ inline void CommandStreamReceiverHw::flushBatchedSubmissions() { ResidencyContainer surfacesForSubmit; ResourcePackage resourcePackage; auto pipeControlLocationSize = getRequiredPipeControlSize(); - void *currentPipeControl = nullptr; + void *currentPipeControlForNooping = nullptr; + void *epiloguePipeControlLocation = nullptr; while (!commandBufferList.peekIsEmpty()) { size_t totalUsedSize = 0u; @@ -405,15 +410,18 @@ inline void CommandStreamReceiverHw::flushBatchedSubmissions() { FlushStampUpdateHelper flushStampUpdateHelper; flushStampUpdateHelper.insert(primaryCmdBuffer->flushStamp->getStampReference()); - currentPipeControl = primaryCmdBuffer->pipeControlLocation; + currentPipeControlForNooping = primaryCmdBuffer->pipeControlThatMayBeErasedLocation; + epiloguePipeControlLocation = primaryCmdBuffer->epiloguePipeControlLocation; while (nextCommandBuffer && nextCommandBuffer->inspectionId == primaryCmdBuffer->inspectionId) { //noop pipe control - if (currentPipeControl) { - memset(currentPipeControl, 0, pipeControlLocationSize); + if (currentPipeControlForNooping) { + memset(currentPipeControlForNooping, 0, pipeControlLocationSize); } //obtain next candidate for nooping - currentPipeControl = nextCommandBuffer->pipeControlLocation; + currentPipeControlForNooping = nextCommandBuffer->pipeControlThatMayBeErasedLocation; + //track epilogue pipe control + epiloguePipeControlLocation = nextCommandBuffer->epiloguePipeControlLocation; flushStampUpdateHelper.insert(nextCommandBuffer->flushStamp->getStampReference()); auto nextCommandBufferAddress = nextCommandBuffer->batchBuffer.commandBufferAllocation->getUnderlyingBuffer(); @@ -430,9 +438,7 @@ inline void CommandStreamReceiverHw::flushBatchedSubmissions() { } //make sure we flush DC - if (currentPipeControl) { - ((PIPE_CONTROL *)currentPipeControl)->setDcFlushEnable(true); - } + ((PIPE_CONTROL *)epiloguePipeControlLocation)->setDcFlushEnable(true); auto flushStamp = this->flush(primaryCmdBuffer->batchBuffer, engineType, &surfacesForSubmit); diff --git a/runtime/command_stream/submissions_aggregator.h b/runtime/command_stream/submissions_aggregator.h index af22dd360e..d8b0c212ed 100644 --- a/runtime/command_stream/submissions_aggregator.h +++ b/runtime/command_stream/submissions_aggregator.h @@ -57,7 +57,8 @@ struct CommandBuffer : public IDNode { void *batchBufferEndLocation = nullptr; uint32_t inspectionId = 0; uint32_t taskCount = 0u; - void *pipeControlLocation = nullptr; + void *pipeControlThatMayBeErasedLocation = nullptr; + void *epiloguePipeControlLocation = nullptr; std::unique_ptr flushStamp; }; diff --git a/unit_tests/command_queue/enqueue_kernel_tests.cpp b/unit_tests/command_queue/enqueue_kernel_tests.cpp index 79d586a54f..600f48c739 100644 --- a/unit_tests/command_queue/enqueue_kernel_tests.cpp +++ b/unit_tests/command_queue/enqueue_kernel_tests.cpp @@ -1416,7 +1416,7 @@ HWTEST_F(EnqueueKernelTest, givenOutOfOrderCommandQueueWhenEnqueueKernelIsMadeTh EXPECT_FALSE(mockedSubmissionsAggregator->peekCmdBufferList().peekIsEmpty()); auto cmdBuffer = mockedSubmissionsAggregator->peekCmdBufferList().peekHead(); - EXPECT_NE(nullptr, cmdBuffer->pipeControlLocation); + EXPECT_NE(nullptr, cmdBuffer->pipeControlThatMayBeErasedLocation); clReleaseCommandQueue(ooq); } @@ -1438,7 +1438,7 @@ HWTEST_F(EnqueueKernelTest, givenInOrderCommandQueueWhenEnqueueKernelIsMadeThenP EXPECT_FALSE(mockedSubmissionsAggregator->peekCmdBufferList().peekIsEmpty()); auto cmdBuffer = mockedSubmissionsAggregator->peekCmdBufferList().peekHead(); - EXPECT_NE(nullptr, cmdBuffer->pipeControlLocation); + EXPECT_NE(nullptr, cmdBuffer->pipeControlThatMayBeErasedLocation); clReleaseCommandQueue(inOrderQueue); } @@ -1461,7 +1461,8 @@ HWTEST_F(EnqueueKernelTest, givenInOrderCommandQueueWhenEnqueueKernelThatHasShar EXPECT_FALSE(mockedSubmissionsAggregator->peekCmdBufferList().peekIsEmpty()); auto cmdBuffer = mockedSubmissionsAggregator->peekCmdBufferList().peekHead(); - EXPECT_EQ(nullptr, cmdBuffer->pipeControlLocation); + EXPECT_EQ(nullptr, cmdBuffer->pipeControlThatMayBeErasedLocation); + EXPECT_NE(nullptr, cmdBuffer->epiloguePipeControlLocation); clReleaseCommandQueue(inOrderQueue); } @@ -1485,7 +1486,8 @@ HWTEST_F(EnqueueKernelTest, givenInOrderCommandQueueWhenEnqueueKernelReturningEv EXPECT_FALSE(mockedSubmissionsAggregator->peekCmdBufferList().peekIsEmpty()); auto cmdBuffer = mockedSubmissionsAggregator->peekCmdBufferList().peekHead(); - EXPECT_EQ(nullptr, cmdBuffer->pipeControlLocation); + EXPECT_EQ(nullptr, cmdBuffer->pipeControlThatMayBeErasedLocation); + EXPECT_NE(nullptr, cmdBuffer->epiloguePipeControlLocation); clReleaseCommandQueue(inOrderQueue); clReleaseEvent(event); @@ -1510,7 +1512,8 @@ HWTEST_F(EnqueueKernelTest, givenOutOfOrderCommandQueueWhenEnqueueKernelReturnin EXPECT_FALSE(mockedSubmissionsAggregator->peekCmdBufferList().peekIsEmpty()); auto cmdBuffer = mockedSubmissionsAggregator->peekCmdBufferList().peekHead(); - EXPECT_NE(nullptr, cmdBuffer->pipeControlLocation); + EXPECT_NE(nullptr, cmdBuffer->pipeControlThatMayBeErasedLocation); + EXPECT_EQ(cmdBuffer->epiloguePipeControlLocation, cmdBuffer->pipeControlThatMayBeErasedLocation); clReleaseCommandQueue(inOrderQueue); clReleaseEvent(event); diff --git a/unit_tests/command_stream/command_stream_receiver_hw_tests.cpp b/unit_tests/command_stream/command_stream_receiver_hw_tests.cpp index 00c61a2cd4..eb2f060625 100644 --- a/unit_tests/command_stream/command_stream_receiver_hw_tests.cpp +++ b/unit_tests/command_stream/command_stream_receiver_hw_tests.cpp @@ -2856,14 +2856,14 @@ HWTEST_F(CommandStreamReceiverFlushTaskTests, givenCsrInBatchingModeWhenTwoTasks //validate if we recorded ppc positions auto firstCmdBuffer = mockedSubmissionsAggregator->peekCommandBuffers().peekHead(); - EXPECT_NE(nullptr, firstCmdBuffer->pipeControlLocation); + EXPECT_NE(nullptr, firstCmdBuffer->pipeControlThatMayBeErasedLocation); auto secondCmdBuffer = firstCmdBuffer->next; - EXPECT_NE(nullptr, secondCmdBuffer->pipeControlLocation); - EXPECT_NE(firstCmdBuffer->pipeControlLocation, secondCmdBuffer->pipeControlLocation); + EXPECT_NE(nullptr, secondCmdBuffer->pipeControlThatMayBeErasedLocation); + EXPECT_NE(firstCmdBuffer->pipeControlThatMayBeErasedLocation, secondCmdBuffer->pipeControlThatMayBeErasedLocation); - auto ppc = genCmdCast(firstCmdBuffer->pipeControlLocation); + auto ppc = genCmdCast(firstCmdBuffer->pipeControlThatMayBeErasedLocation); EXPECT_NE(nullptr, ppc); - auto ppc2 = genCmdCast(secondCmdBuffer->pipeControlLocation); + auto ppc2 = genCmdCast(secondCmdBuffer->pipeControlThatMayBeErasedLocation); EXPECT_NE(nullptr, ppc2); //flush needs to bump the taskLevel @@ -2952,6 +2952,36 @@ HWTEST_F(CommandStreamReceiverFlushTaskTests, givenCsrInBatchingModeWhenCommandA EXPECT_TRUE(pipeControl->getDcFlushEnable()); } +HWTEST_F(CommandStreamReceiverFlushTaskTests, givenCsrInBatchingModeWithOutOfOrderModeFisabledWhenCommandAreSubmittedThenDcFlushIsAdded) { + CommandQueueHw commandQueue(nullptr, pDevice, 0); + auto &commandStream = commandQueue.getCS(4096u); + + auto mockCsr = new MockCsrHw2(*platformDevices[0]); + pDevice->resetCommandStreamReceiver(mockCsr); + + mockCsr->overrideDispatchPolicy(CommandStreamReceiver::DispatchMode::BatchedDispatch); + + DispatchFlags dispatchFlags; + dispatchFlags.guardCommandBufferWithPipeControl = true; + dispatchFlags.outOfOrderExecutionAllowed = false; + + mockCsr->flushTask(commandStream, + 0, + dsh, + ih, + ioh, + ssh, + taskLevel, + dispatchFlags); + + parseCommands(commandStream); + auto itorPipeControl = find(cmdList.begin(), cmdList.end()); + auto pipeControl = genCmdCast(*itorPipeControl); + + mockCsr->flushBatchedSubmissions(); + EXPECT_TRUE(pipeControl->getDcFlushEnable()); +} + HWTEST_F(CommandStreamReceiverFlushTaskTests, givenCsrInBatchingModeWhenDcFlushIsRequiredThenPipeControlIsNotRegistredForNooping) { CommandQueueHw commandQueue(nullptr, pDevice, 0); auto &commandStream = commandQueue.getCS(4096u); @@ -2977,9 +3007,9 @@ HWTEST_F(CommandStreamReceiverFlushTaskTests, givenCsrInBatchingModeWhenDcFlushI taskLevel, dispatchFlags); - //validate if we recorded ppc positions auto cmdBuffer = mockedSubmissionsAggregator->peekCommandBuffers().peekHead(); - EXPECT_EQ(nullptr, cmdBuffer->pipeControlLocation); + EXPECT_EQ(nullptr, cmdBuffer->pipeControlThatMayBeErasedLocation); + EXPECT_NE(nullptr, cmdBuffer->epiloguePipeControlLocation); } HWTEST_F(CommandStreamReceiverFlushTaskTests, givenCsrInBatchingModeAndOoqFlagSetToFalseWhenTwoTasksArePassedWithTheSameLevelThenThereIsPipeControlBetweenThemAfterFlush) { @@ -3023,9 +3053,9 @@ HWTEST_F(CommandStreamReceiverFlushTaskTests, givenCsrInBatchingModeAndOoqFlagSe //validate if we recorded ppc positions auto firstCmdBuffer = mockedSubmissionsAggregator->peekCommandBuffers().peekHead(); - EXPECT_EQ(nullptr, firstCmdBuffer->pipeControlLocation); + EXPECT_EQ(nullptr, firstCmdBuffer->pipeControlThatMayBeErasedLocation); auto secondCmdBuffer = firstCmdBuffer->next; - EXPECT_EQ(nullptr, secondCmdBuffer->pipeControlLocation); + EXPECT_EQ(nullptr, secondCmdBuffer->pipeControlThatMayBeErasedLocation); mockCsr->flushBatchedSubmissions(); @@ -3079,9 +3109,9 @@ HWTEST_F(CommandStreamReceiverFlushTaskTests, givenCsrInBatchingModeWhenPipeCont //validate if we recorded ppc positions auto firstCmdBuffer = mockedSubmissionsAggregator->peekCommandBuffers().peekHead(); - auto ppc1Location = firstCmdBuffer->pipeControlLocation; + auto ppc1Location = firstCmdBuffer->pipeControlThatMayBeErasedLocation; - firstCmdBuffer->pipeControlLocation = nullptr; + firstCmdBuffer->pipeControlThatMayBeErasedLocation = nullptr; auto ppc = genCmdCast(ppc1Location); EXPECT_NE(nullptr, ppc); @@ -3153,12 +3183,12 @@ HWTEST_F(CommandStreamReceiverFlushTaskTests, givenCsrInBatchingModeWhenThreeTas auto secondCmdBuffer = firstCmdBuffer->next; auto thirdCmdBuffer = firstCmdBuffer->next->next; - EXPECT_NE(nullptr, thirdCmdBuffer->pipeControlLocation); - EXPECT_NE(firstCmdBuffer->pipeControlLocation, thirdCmdBuffer->pipeControlLocation); + EXPECT_NE(nullptr, thirdCmdBuffer->pipeControlThatMayBeErasedLocation); + EXPECT_NE(firstCmdBuffer->pipeControlThatMayBeErasedLocation, thirdCmdBuffer->pipeControlThatMayBeErasedLocation); - auto ppc = genCmdCast(firstCmdBuffer->pipeControlLocation); - auto ppc2 = genCmdCast(secondCmdBuffer->pipeControlLocation); - auto ppc3 = genCmdCast(thirdCmdBuffer->pipeControlLocation); + auto ppc = genCmdCast(firstCmdBuffer->pipeControlThatMayBeErasedLocation); + auto ppc2 = genCmdCast(secondCmdBuffer->pipeControlThatMayBeErasedLocation); + auto ppc3 = genCmdCast(thirdCmdBuffer->pipeControlThatMayBeErasedLocation); EXPECT_NE(nullptr, ppc2); EXPECT_NE(nullptr, ppc3); diff --git a/unit_tests/os_interface/linux/drm_command_stream_tests.cpp b/unit_tests/os_interface/linux/drm_command_stream_tests.cpp index e835f093fb..e8ff2755bd 100644 --- a/unit_tests/os_interface/linux/drm_command_stream_tests.cpp +++ b/unit_tests/os_interface/linux/drm_command_stream_tests.cpp @@ -1004,6 +1004,7 @@ TEST_F(DrmCommandStreamBatchingTests, givenRecordedCommandBufferWhenItIsSubmitte submittedCommandBuffer.getSpace(4); DispatchFlags dispatchFlags; + dispatchFlags.guardCommandBufferWithPipeControl = true; tCsr->flushTask(cs, 0u, cs, cs, cs, cs, 0u, dispatchFlags); auto &cmdBuffers = mockedSubmissionsAggregator->peekCommandBuffers(); diff --git a/unit_tests/os_interface/windows/device_command_stream_tests.cpp b/unit_tests/os_interface/windows/device_command_stream_tests.cpp index afd6ffca10..c2d6823cea 100644 --- a/unit_tests/os_interface/windows/device_command_stream_tests.cpp +++ b/unit_tests/os_interface/windows/device_command_stream_tests.cpp @@ -533,6 +533,7 @@ HWTEST_F(WddmCommandStreamMockGdiTest, givenRecordedCommandBufferWhenItIsSubmitt LinearStream cs(commandBuffer); DispatchFlags dispatchFlags; + dispatchFlags.guardCommandBufferWithPipeControl = true; dispatchFlags.requiresCoherency = true; mockCsr->flushTask(cs, 0u, cs, cs, cs, cs, 0u, dispatchFlags);