From ad2b8a0bbe5845197b35fe97758dbab55288634b Mon Sep 17 00:00:00 2001 From: "Dunajski, Bartosz" Date: Thu, 7 Nov 2019 09:15:53 +0100 Subject: [PATCH] Allow dispatching multiple blit commands with single flush Change-Id: I57ea32dfa8cfd89ec37fc2f1f82ee6f72598c18a Signed-off-by: Dunajski, Bartosz --- runtime/command_queue/command_queue_hw.h | 1 + runtime/command_queue/enqueue_common.h | 17 +- .../command_stream/command_stream_receiver.h | 2 +- .../command_stream_receiver_hw.h | 2 +- .../command_stream_receiver_hw_base.inl | 38 ++-- runtime/helpers/CMakeLists.txt | 1 + runtime/helpers/blit_commands_helper.cpp | 17 +- runtime/helpers/blit_commands_helper.h | 12 +- runtime/helpers/blit_commands_helper_base.inl | 18 +- runtime/helpers/enqueue_properties.h | 57 +++++ runtime/helpers/properties_helper.h | 46 ---- runtime/helpers/task_information.cpp | 6 +- runtime/helpers/task_information.h | 2 +- .../enqueue_command_without_kernel_tests.cpp | 19 +- .../aub_command_stream_receiver_1_tests.cpp | 6 +- .../command_stream_receiver_hw_tests.cpp | 203 +++++++++++++----- unit_tests/kernel/kernel_tests.cpp | 2 +- .../libult/ult_aub_command_stream_receiver.h | 4 +- .../libult/ult_command_stream_receiver.h | 4 +- unit_tests/mem_obj/buffer_tests.cpp | 12 +- unit_tests/mocks/mock_csr.h | 6 +- 21 files changed, 301 insertions(+), 174 deletions(-) create mode 100644 runtime/helpers/enqueue_properties.h diff --git a/runtime/command_queue/command_queue_hw.h b/runtime/command_queue/command_queue_hw.h index 7e510d7f27..2e8e282e4d 100644 --- a/runtime/command_queue/command_queue_hw.h +++ b/runtime/command_queue/command_queue_hw.h @@ -24,6 +24,7 @@ namespace NEO { class EventBuilder; +struct EnqueueProperties; template class CommandQueueHw : public CommandQueue { diff --git a/runtime/command_queue/enqueue_common.h b/runtime/command_queue/enqueue_common.h index 68752e6b61..a6f1f0453f 100644 --- a/runtime/command_queue/enqueue_common.h +++ b/runtime/command_queue/enqueue_common.h @@ -19,6 +19,7 @@ #include "runtime/gtpin/gtpin_notify.h" #include "runtime/helpers/array_count.h" #include "runtime/helpers/dispatch_info_builder.h" +#include "runtime/helpers/enqueue_properties.h" #include "runtime/helpers/hardware_commands_helper.h" #include "runtime/helpers/options.h" #include "runtime/helpers/task_information.h" @@ -192,7 +193,7 @@ void CommandQueueHw::enqueueHandler(Surface **surfacesForResidency, TimestampPacketContainer barrierTimestampPacketNode; EventsRequest eventsRequest(numEventsInWaitList, eventWaitList, event); CsrDependencies csrDeps; - BlitProperties blitProperties; + BlitPropertiesContainer blitPropertiesContainer; if (getGpgpuCommandStreamReceiver().peekTimestampPacketWriteEnabled()) { csrDeps.fillFromEventsRequest(eventsRequest, getGpgpuCommandStreamReceiver(), CsrDependencies::DependenciesType::OnCsr); @@ -226,8 +227,8 @@ void CommandQueueHw::enqueueHandler(Surface **surfacesForResidency, bool flushDependenciesForNonKernelCommand = false; if (blitEnqueue) { - blitProperties = processDispatchForBlitEnqueue(multiDispatchInfo, previousTimestampPacketNodes, barrierTimestampPacketNode, - eventsRequest, commandStream, commandType, blockQueue); + blitPropertiesContainer.push_back(processDispatchForBlitEnqueue(multiDispatchInfo, previousTimestampPacketNodes, barrierTimestampPacketNode, + eventsRequest, commandStream, commandType, blockQueue)); } else if (multiDispatchInfo.empty() == false) { processDispatchForKernels(multiDispatchInfo, printfHandler, eventBuilder.getEvent(), hwTimeStamps, blockQueue, devQueueHw, csrDeps, blockedCommandsData.get(), @@ -256,7 +257,7 @@ void CommandQueueHw::enqueueHandler(Surface **surfacesForResidency, CompletionStamp completionStamp = {Event::eventNotReady, taskLevel, 0}; const EnqueueProperties enqueueProperties(blitEnqueue, !multiDispatchInfo.empty(), isCacheFlushCommand(commandType), - flushDependenciesForNonKernelCommand, &blitProperties); + flushDependenciesForNonKernelCommand, &blitPropertiesContainer); if (!blockQueue) { csrDeps.makeResident(getGpgpuCommandStreamReceiver()); @@ -448,7 +449,7 @@ BlitProperties CommandQueueHw::processDispatchForBlitEnqueue(const Mu auto blitCommandStreamReceiver = getBcsCommandStreamReceiver(); auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(blitDirection, *blitCommandStreamReceiver, - multiDispatchInfo.peekBuiltinOpParams(), false); + multiDispatchInfo.peekBuiltinOpParams()); if (!queueBlocked) { blitProperties.csrDependencies.fillFromEventsRequest(eventsRequest, *blitCommandStreamReceiver, CsrDependencies::DependenciesType::All); @@ -777,7 +778,7 @@ void CommandQueueHw::enqueueBlocked( if (blockedCommandsData) { if (enqueueProperties.operation == EnqueueProperties::Operation::Blit) { - blockedCommandsData->blitProperties = *enqueueProperties.blitProperties; + blockedCommandsData->blitPropertiesContainer = *enqueueProperties.blitPropertiesContainer; blockedCommandsData->blitEnqueue = true; } @@ -861,8 +862,8 @@ CompletionStamp CommandQueueHw::enqueueCommandWithoutKernel( } if (enqueueProperties.operation == EnqueueProperties::Operation::Blit) { - UNRECOVERABLE_IF(!enqueueProperties.blitProperties); - this->bcsTaskCount = getBcsCommandStreamReceiver()->blitBuffer(*enqueueProperties.blitProperties); + UNRECOVERABLE_IF(!enqueueProperties.blitPropertiesContainer); + this->bcsTaskCount = getBcsCommandStreamReceiver()->blitBuffer(*enqueueProperties.blitPropertiesContainer, false); } DispatchFlags dispatchFlags( diff --git a/runtime/command_stream/command_stream_receiver.h b/runtime/command_stream/command_stream_receiver.h index 8f1fe68802..443064c410 100644 --- a/runtime/command_stream/command_stream_receiver.h +++ b/runtime/command_stream/command_stream_receiver.h @@ -176,7 +176,7 @@ class CommandStreamReceiver { this->latestSentTaskCount = latestSentTaskCount; } - virtual uint32_t blitBuffer(const BlitProperties &blitProperties) = 0; + virtual uint32_t blitBuffer(const BlitPropertiesContainer &blitPropertiesContainer, bool blocking) = 0; ScratchSpaceController *getScratchSpaceController() const { return scratchSpaceController.get(); diff --git a/runtime/command_stream/command_stream_receiver_hw.h b/runtime/command_stream/command_stream_receiver_hw.h index 3c6fc24c1c..bb9a67c585 100644 --- a/runtime/command_stream/command_stream_receiver_hw.h +++ b/runtime/command_stream/command_stream_receiver_hw.h @@ -74,7 +74,7 @@ class CommandStreamReceiverHw : public CommandStreamReceiver { return CommandStreamReceiverType::CSR_HW; } - uint32_t blitBuffer(const BlitProperties &blitProperties) override; + uint32_t blitBuffer(const BlitPropertiesContainer &blitPropertiesContainer, bool blocking) override; bool isMultiOsContextCapable() const override; diff --git a/runtime/command_stream/command_stream_receiver_hw_base.inl b/runtime/command_stream/command_stream_receiver_hw_base.inl index 470594a7eb..8d5fb2cd6f 100644 --- a/runtime/command_stream/command_stream_receiver_hw_base.inl +++ b/runtime/command_stream/command_stream_receiver_hw_base.inl @@ -797,40 +797,42 @@ bool CommandStreamReceiverHw::detectInitProgrammingFlagsRequired(cons } template -uint32_t CommandStreamReceiverHw::blitBuffer(const BlitProperties &blitProperties) { +uint32_t CommandStreamReceiverHw::blitBuffer(const BlitPropertiesContainer &blitPropertiesContainer, bool blocking) { using MI_BATCH_BUFFER_END = typename GfxFamily::MI_BATCH_BUFFER_END; using MI_FLUSH_DW = typename GfxFamily::MI_FLUSH_DW; auto lock = obtainUniqueOwnership(); - bool updateTimestampPacket = blitProperties.outputTimestampPacket != nullptr; - auto &commandStream = getCS(BlitCommandsHelper::estimateBlitCommandsSize(blitProperties.copySize, - blitProperties.csrDependencies, - updateTimestampPacket)); + + auto &commandStream = getCS(BlitCommandsHelper::estimateBlitCommandsSize(blitPropertiesContainer)); auto commandStreamStart = commandStream.getUsed(); auto newTaskCount = taskCount + 1; latestSentTaskCount = newTaskCount; - TimestampPacketHelper::programCsrDependencies(commandStream, blitProperties.csrDependencies); + for (auto &blitProperties : blitPropertiesContainer) { + TimestampPacketHelper::programCsrDependencies(commandStream, blitProperties.csrDependencies); - BlitCommandsHelper::dispatchBlitCommandsForBuffer(blitProperties, commandStream); + BlitCommandsHelper::dispatchBlitCommandsForBuffer(blitProperties, commandStream); + + if (blitProperties.outputTimestampPacket) { + UNRECOVERABLE_IF(blitProperties.outputTimestampPacket->peekNodes().size() != 1); + auto timestampPacketGpuAddress = blitProperties.outputTimestampPacket->peekNodes().at(0)->getGpuAddress() + offsetof(TimestampPacketStorage, packets[0].contextEnd); + HardwareCommandsHelper::programMiFlushDw(commandStream, timestampPacketGpuAddress, 0); + blitProperties.outputTimestampPacket->makeResident(*this); + } + + blitProperties.csrDependencies.makeResident(*this); + + makeResident(*blitProperties.srcAllocation); + makeResident(*blitProperties.dstAllocation); + } HardwareCommandsHelper::programMiFlushDw(commandStream, tagAllocation->getGpuAddress(), newTaskCount); - if (updateTimestampPacket) { - UNRECOVERABLE_IF(blitProperties.outputTimestampPacket->peekNodes().size() != 1); - auto timestampPacketGpuAddress = blitProperties.outputTimestampPacket->peekNodes().at(0)->getGpuAddress() + offsetof(TimestampPacketStorage, packets[0].contextEnd); - HardwareCommandsHelper::programMiFlushDw(commandStream, timestampPacketGpuAddress, 0); - blitProperties.outputTimestampPacket->makeResident(*this); - } - auto batchBufferEnd = reinterpret_cast(commandStream.getSpace(sizeof(MI_BATCH_BUFFER_END))); *batchBufferEnd = GfxFamily::cmdInitBatchBufferEnd; alignToCacheLine(commandStream); - blitProperties.csrDependencies.makeResident(*this); - makeResident(*blitProperties.srcAllocation); - makeResident(*blitProperties.dstAllocation); makeResident(*tagAllocation); BatchBuffer batchBuffer{commandStream.getGraphicsAllocation(), commandStreamStart, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, @@ -844,7 +846,7 @@ uint32_t CommandStreamReceiverHw::blitBuffer(const BlitProperties &bl auto flushStampToWait = flushStamp->peekStamp(); lock.unlock(); - if (blitProperties.blocking) { + if (blocking) { waitForTaskCountWithKmdNotifyFallback(newTaskCount, flushStampToWait, false, false); internalAllocationStorage->cleanAllocationList(newTaskCount, TEMPORARY_ALLOCATION); } diff --git a/runtime/helpers/CMakeLists.txt b/runtime/helpers/CMakeLists.txt index f7d42c4dd6..2ea367ebf2 100644 --- a/runtime/helpers/CMakeLists.txt +++ b/runtime/helpers/CMakeLists.txt @@ -32,6 +32,7 @@ set(RUNTIME_SRCS_HELPERS_BASE ${CMAKE_CURRENT_SOURCE_DIR}/engine_control.h ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/engine_node_helper.cpp ${CMAKE_CURRENT_SOURCE_DIR}/engine_node_helper.h + ${CMAKE_CURRENT_SOURCE_DIR}/enqueue_properties.h ${CMAKE_CURRENT_SOURCE_DIR}/error_mappers.h ${CMAKE_CURRENT_SOURCE_DIR}/flat_batch_buffer_helper.h ${CMAKE_CURRENT_SOURCE_DIR}/flat_batch_buffer_helper.cpp diff --git a/runtime/helpers/blit_commands_helper.cpp b/runtime/helpers/blit_commands_helper.cpp index 5177e7ef30..96434e27eb 100644 --- a/runtime/helpers/blit_commands_helper.cpp +++ b/runtime/helpers/blit_commands_helper.cpp @@ -20,7 +20,7 @@ BlitProperties BlitProperties::constructPropertiesForReadWriteBuffer(BlitterCons GraphicsAllocation *memObjAllocation, size_t memObjOffset, GraphicsAllocation *mapAllocation, void *hostPtr, size_t hostPtrOffset, - bool blocking, size_t copyOffset, uint64_t copySize) { + size_t copyOffset, uint64_t copySize) { GraphicsAllocation *hostAllocation = nullptr; @@ -36,16 +36,15 @@ BlitProperties BlitProperties::constructPropertiesForReadWriteBuffer(BlitterCons auto offset = copyOffset + memObjOffset; if (BlitterConstants::BlitDirection::HostPtrToBuffer == blitDirection) { - return {nullptr, blitDirection, {}, AuxTranslationDirection::None, memObjAllocation, hostAllocation, blocking, offset, hostPtrOffset, copySize}; + return {nullptr, blitDirection, {}, AuxTranslationDirection::None, memObjAllocation, hostAllocation, offset, hostPtrOffset, copySize}; } else { - return {nullptr, blitDirection, {}, AuxTranslationDirection::None, hostAllocation, memObjAllocation, blocking, hostPtrOffset, offset, copySize}; + return {nullptr, blitDirection, {}, AuxTranslationDirection::None, hostAllocation, memObjAllocation, hostPtrOffset, offset, copySize}; } } BlitProperties BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection blitDirection, CommandStreamReceiver &commandStreamReceiver, - const BuiltinOpParams &builtinOpParams, - bool blocking) { + const BuiltinOpParams &builtinOpParams) { GraphicsAllocation *gpuAllocation = nullptr; size_t copyOffset = 0; size_t memObjOffset = 0; @@ -89,20 +88,20 @@ BlitProperties BlitProperties::constructPropertiesForReadWriteBuffer(BlitterCons BlitterConstants::BlitDirection::BufferToHostPtr != blitDirection); return constructPropertiesForReadWriteBuffer(blitDirection, commandStreamReceiver, gpuAllocation, memObjOffset, - hostAllocation, hostPtr, hostPtrOffset, blocking, copyOffset, + hostAllocation, hostPtr, hostPtrOffset, copyOffset, builtinOpParams.size.x); } BlitProperties BlitProperties::constructPropertiesForCopyBuffer(GraphicsAllocation *dstAllocation, GraphicsAllocation *srcAllocation, - bool blocking, size_t dstOffset, size_t srcOffset, uint64_t copySize) { + size_t dstOffset, size_t srcOffset, uint64_t copySize) { - return {nullptr, BlitterConstants::BlitDirection::BufferToBuffer, {}, AuxTranslationDirection::None, dstAllocation, srcAllocation, blocking, dstOffset, srcOffset, copySize}; + return {nullptr, BlitterConstants::BlitDirection::BufferToBuffer, {}, AuxTranslationDirection::None, dstAllocation, srcAllocation, dstOffset, srcOffset, copySize}; } BlitProperties BlitProperties::constructPropertiesForAuxTranslation(AuxTranslationDirection auxTranslationDirection, GraphicsAllocation *allocation) { auto allocationSize = allocation->getUnderlyingBufferSize(); - return {nullptr, BlitterConstants::BlitDirection::BufferToBuffer, {}, auxTranslationDirection, allocation, allocation, false, 0, 0, allocationSize}; + return {nullptr, BlitterConstants::BlitDirection::BufferToBuffer, {}, auxTranslationDirection, allocation, allocation, 0, 0, allocationSize}; } BlitterConstants::BlitDirection BlitProperties::obtainBlitDirection(uint32_t commandType) { diff --git a/runtime/helpers/blit_commands_helper.h b/runtime/helpers/blit_commands_helper.h index df2950c16f..d02dd70f16 100644 --- a/runtime/helpers/blit_commands_helper.h +++ b/runtime/helpers/blit_commands_helper.h @@ -7,6 +7,7 @@ #pragma once #include "core/memory_manager/memory_constants.h" +#include "core/utilities/stackvec.h" #include "runtime/helpers/csr_deps.h" #include "runtime/helpers/properties_helper.h" @@ -26,15 +27,14 @@ struct BlitProperties { GraphicsAllocation *memObjAllocation, size_t memObjOFfset, GraphicsAllocation *mapAllocation, void *hostPtr, size_t hostPtrOffset, - bool blocking, size_t copyOffset, uint64_t copySize); + size_t copyOffset, uint64_t copySize); static BlitProperties constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection blitDirection, CommandStreamReceiver &commandStreamReceiver, - const BuiltinOpParams &builtinOpParams, - bool blocking); + const BuiltinOpParams &builtinOpParams); static BlitProperties constructPropertiesForCopyBuffer(GraphicsAllocation *dstAllocation, GraphicsAllocation *srcAllocation, - bool blocking, size_t dstOffset, size_t srcOffset, uint64_t copySize); + size_t dstOffset, size_t srcOffset, uint64_t copySize); static BlitProperties constructPropertiesForAuxTranslation(AuxTranslationDirection auxTranslationDirection, GraphicsAllocation *allocation); @@ -48,15 +48,17 @@ struct BlitProperties { GraphicsAllocation *dstAllocation = nullptr; GraphicsAllocation *srcAllocation = nullptr; - bool blocking = false; size_t dstOffset = 0; size_t srcOffset = 0; uint64_t copySize = 0; }; +using BlitPropertiesContainer = StackVec; + template struct BlitCommandsHelper { static size_t estimateBlitCommandsSize(uint64_t copySize, const CsrDependencies &csrDependencies, bool updateTimestampPacket); + static size_t estimateBlitCommandsSize(const BlitPropertiesContainer &blitPropertiesContainer); static void dispatchBlitCommandsForBuffer(const BlitProperties &blitProperties, LinearStream &linearStream); static void appendBlitCommandsForBuffer(const BlitProperties &blitProperties, typename GfxFamily::XY_COPY_BLT &blitCmd); }; diff --git a/runtime/helpers/blit_commands_helper_base.inl b/runtime/helpers/blit_commands_helper_base.inl index 67ee383053..bf2cacaf3e 100644 --- a/runtime/helpers/blit_commands_helper_base.inl +++ b/runtime/helpers/blit_commands_helper_base.inl @@ -30,11 +30,19 @@ size_t BlitCommandsHelper::estimateBlitCommandsSize(uint64_t copySize numberOfBlits++; } - size_t size = TimestampPacketHelper::getRequiredCmdStreamSize(csrDependencies) + - (sizeof(typename GfxFamily::XY_COPY_BLT) * numberOfBlits) + - sizeof(typename GfxFamily::MI_FLUSH_DW) + - (sizeof(typename GfxFamily::MI_FLUSH_DW) * static_cast(updateTimestampPacket)) + - sizeof(typename GfxFamily::MI_BATCH_BUFFER_END); + return TimestampPacketHelper::getRequiredCmdStreamSize(csrDependencies) + + (sizeof(typename GfxFamily::XY_COPY_BLT) * numberOfBlits) + + (sizeof(typename GfxFamily::MI_FLUSH_DW) * static_cast(updateTimestampPacket)); +} + +template +size_t BlitCommandsHelper::estimateBlitCommandsSize(const BlitPropertiesContainer &blitPropertiesContainer) { + size_t size = 0; + for (auto &blitProperties : blitPropertiesContainer) { + size += BlitCommandsHelper::estimateBlitCommandsSize(blitProperties.copySize, blitProperties.csrDependencies, + blitProperties.outputTimestampPacket != nullptr); + } + size += sizeof(typename GfxFamily::MI_FLUSH_DW) + sizeof(typename GfxFamily::MI_BATCH_BUFFER_END); return alignUp(size, MemoryConstants::cacheLineSize); } diff --git a/runtime/helpers/enqueue_properties.h b/runtime/helpers/enqueue_properties.h new file mode 100644 index 0000000000..99f856160b --- /dev/null +++ b/runtime/helpers/enqueue_properties.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2018-2019 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once +#include "runtime/helpers/blit_commands_helper.h" + +namespace NEO { + +struct EnqueueProperties { + enum class Operation { + Blit, + ExplicitCacheFlush, + EnqueueWithoutSubmission, + DependencyResolveOnGpu, + GpuKernel, + }; + + EnqueueProperties() = delete; + EnqueueProperties(bool blitEnqueue, bool hasKernels, bool isCacheFlushCmd, bool flushDependenciesOnly, + const BlitPropertiesContainer *blitPropertiesContainer) { + if (blitEnqueue) { + operation = Operation::Blit; + this->blitPropertiesContainer = blitPropertiesContainer; + return; + } + + if (hasKernels) { + operation = Operation::GpuKernel; + return; + } + + if (isCacheFlushCmd) { + operation = Operation::ExplicitCacheFlush; + return; + } + + if (flushDependenciesOnly) { + operation = Operation::DependencyResolveOnGpu; + return; + } + + operation = Operation::EnqueueWithoutSubmission; + } + + bool isFlushWithoutKernelRequired() const { + return (operation == Operation::Blit) || (operation == Operation::ExplicitCacheFlush) || + (operation == Operation::DependencyResolveOnGpu); + } + + const BlitPropertiesContainer *blitPropertiesContainer = nullptr; + Operation operation = Operation::EnqueueWithoutSubmission; +}; +} // namespace NEO diff --git a/runtime/helpers/properties_helper.h b/runtime/helpers/properties_helper.h index 1c5042e366..d445b8cdd1 100644 --- a/runtime/helpers/properties_helper.h +++ b/runtime/helpers/properties_helper.h @@ -15,7 +15,6 @@ namespace NEO { class MemObj; class Buffer; -struct BlitProperties; enum QueueThrottle : uint32_t { LOW, @@ -83,49 +82,4 @@ struct MapInfo { uint32_t mipLevel = 0; bool readOnly = false; }; - -struct EnqueueProperties { - enum class Operation { - Blit, - ExplicitCacheFlush, - EnqueueWithoutSubmission, - DependencyResolveOnGpu, - GpuKernel, - }; - - EnqueueProperties() = delete; - EnqueueProperties(bool blitEnqueue, bool hasKernels, bool isCacheFlushCmd, bool flushDependenciesOnly, - const BlitProperties *blitProperties) { - if (blitEnqueue) { - operation = Operation::Blit; - this->blitProperties = blitProperties; - return; - } - - if (hasKernels) { - operation = Operation::GpuKernel; - return; - } - - if (isCacheFlushCmd) { - operation = Operation::ExplicitCacheFlush; - return; - } - - if (flushDependenciesOnly) { - operation = Operation::DependencyResolveOnGpu; - return; - } - - operation = Operation::EnqueueWithoutSubmission; - } - - bool isFlushWithoutKernelRequired() const { - return (operation == Operation::Blit) || (operation == Operation::ExplicitCacheFlush) || - (operation == Operation::DependencyResolveOnGpu); - } - - const BlitProperties *blitProperties = nullptr; - Operation operation = Operation::EnqueueWithoutSubmission; -}; } // namespace NEO diff --git a/runtime/helpers/task_information.cpp b/runtime/helpers/task_information.cpp index 582d87e2be..3eb70f9a6f 100644 --- a/runtime/helpers/task_information.cpp +++ b/runtime/helpers/task_information.cpp @@ -18,6 +18,7 @@ #include "runtime/device_queue/device_queue.h" #include "runtime/gtpin/gtpin_notify.h" #include "runtime/helpers/csr_deps.h" +#include "runtime/helpers/enqueue_properties.h" #include "runtime/helpers/task_information.inl" #include "runtime/mem_obj/mem_obj.h" #include "runtime/memory_manager/internal_allocation_storage.h" @@ -254,13 +255,14 @@ CompletionStamp &CommandComputeKernel::submit(uint32_t taskLevel, bool terminate void CommandWithoutKernel::dispatchBlitOperation() { auto bcsCsr = commandQueue.getBcsCommandStreamReceiver(); - auto &blitProperties = kernelOperation->blitProperties; + UNRECOVERABLE_IF(kernelOperation->blitPropertiesContainer.size() != 1); + auto &blitProperties = *kernelOperation->blitPropertiesContainer.begin(); blitProperties.csrDependencies.fillFromEventsRequest(eventsRequest, *bcsCsr, CsrDependencies::DependenciesType::All); blitProperties.csrDependencies.push_back(previousTimestampPacketNodes.get()); blitProperties.csrDependencies.push_back(barrierTimestampPacketNodes.get()); blitProperties.outputTimestampPacket = currentTimestampPacketNodes.get(); - auto bcsTaskCount = bcsCsr->blitBuffer(blitProperties); + auto bcsTaskCount = bcsCsr->blitBuffer(kernelOperation->blitPropertiesContainer, false); commandQueue.updateBcsTaskCount(bcsTaskCount); } diff --git a/runtime/helpers/task_information.h b/runtime/helpers/task_information.h index 0cca2483ab..da260d5b0a 100644 --- a/runtime/helpers/task_information.h +++ b/runtime/helpers/task_information.h @@ -75,7 +75,7 @@ struct KernelOperation { IndirectHeapUniquePtrT ioh{nullptr, resourceCleaner}; IndirectHeapUniquePtrT ssh{nullptr, resourceCleaner}; - BlitProperties blitProperties; + BlitPropertiesContainer blitPropertiesContainer; bool blitEnqueue = false; size_t surfaceStateHeapSizeEM = 0; }; diff --git a/unit_tests/command_queue/enqueue_command_without_kernel_tests.cpp b/unit_tests/command_queue/enqueue_command_without_kernel_tests.cpp index 25c34923e6..53c9d3bf5f 100644 --- a/unit_tests/command_queue/enqueue_command_without_kernel_tests.cpp +++ b/unit_tests/command_queue/enqueue_command_without_kernel_tests.cpp @@ -7,6 +7,7 @@ #include "runtime/event/event_builder.h" #include "runtime/event/user_event.h" +#include "runtime/helpers/enqueue_properties.h" #include "runtime/helpers/timestamp_packet.h" #include "runtime/memory_manager/surface.h" #include "runtime/os_interface/os_context.h" @@ -95,7 +96,9 @@ HWTEST_F(EnqueueHandlerTest, givenBlitPropertyWhenEnqueueIsBlockedThenRegisterBl BlitProperties blitProperties; blitProperties.srcAllocation = reinterpret_cast(0x12345); blitProperties.dstAllocation = reinterpret_cast(0x56789); - const EnqueueProperties enqueuePropertiesForBlitEnqueue(true, false, false, false, &blitProperties); + BlitPropertiesContainer blitPropertiesContainer; + blitPropertiesContainer.push_back(blitProperties); + const EnqueueProperties enqueuePropertiesForBlitEnqueue(true, false, false, false, &blitPropertiesContainer); auto blockedCommandsData = std::unique_ptr(blockedCommandsDataForBlitEnqueue); Surface *surfaces[] = {nullptr}; @@ -103,8 +106,8 @@ HWTEST_F(EnqueueHandlerTest, givenBlitPropertyWhenEnqueueIsBlockedThenRegisterBl barrierTimestampPacketNodes, blockedCommandsData, enqueuePropertiesForBlitEnqueue, eventsRequest, eventBuilder, std::unique_ptr(nullptr)); EXPECT_TRUE(blockedCommandsDataForBlitEnqueue->blitEnqueue); - EXPECT_EQ(blitProperties.srcAllocation, blockedCommandsDataForBlitEnqueue->blitProperties.srcAllocation); - EXPECT_EQ(blitProperties.dstAllocation, blockedCommandsDataForBlitEnqueue->blitProperties.dstAllocation); + EXPECT_EQ(blitProperties.srcAllocation, blockedCommandsDataForBlitEnqueue->blitPropertiesContainer.begin()->srcAllocation); + EXPECT_EQ(blitProperties.dstAllocation, blockedCommandsDataForBlitEnqueue->blitPropertiesContainer.begin()->dstAllocation); } HWTEST_F(DispatchFlagsTests, whenEnqueueCommandWithoutKernelThenPassCorrectDispatchFlags) { @@ -159,7 +162,10 @@ HWTEST_F(DispatchFlagsTests, givenBlitEnqueueWhenDispatchingCommandsWithoutKerne BlitProperties blitProperties = mockCmdQ->processDispatchForBlitEnqueue(multiDispatchInfo, previousTimestampPacketNodes, barrierTimestampPacketNodes, eventsRequest, mockCmdQ->getCS(0), 0, false); - EnqueueProperties enqueueProperties(true, false, false, false, &blitProperties); + BlitPropertiesContainer blitPropertiesContainer; + blitPropertiesContainer.push_back(blitProperties); + + EnqueueProperties enqueueProperties(true, false, false, false, &blitPropertiesContainer); mockCmdQ->enqueueCommandWithoutKernel(nullptr, 0, mockCmdQ->getCS(0), 0, blocking, enqueueProperties, &previousTimestampPacketNodes, barrierTimestampPacketNodes, eventsRequest, eventBuilder, 0); @@ -195,8 +201,9 @@ HWTEST_F(DispatchFlagsTests, givenN1EnabledWhenDispatchingWithoutKernelTheAllowO mockCmdQ->obtainNewTimestampPacketNodes(1, previousTimestampPacketNodes, true); BlitProperties blitProperties = mockCmdQ->processDispatchForBlitEnqueue(multiDispatchInfo, previousTimestampPacketNodes, barrierTimestampPacketNodes, eventsRequest, mockCmdQ->getCS(0), 0, false); - EnqueueProperties enqueueProperties(true, false, false, false, &blitProperties); - enqueueProperties.blitProperties = &blitProperties; + BlitPropertiesContainer blitPropertiesContainer; + blitPropertiesContainer.push_back(blitProperties); + EnqueueProperties enqueueProperties(true, false, false, false, &blitPropertiesContainer); mockCsr->nTo1SubmissionModelEnabled = false; mockCmdQ->enqueueCommandWithoutKernel(nullptr, 0, mockCmdQ->getCS(0), 0, blocked, enqueueProperties, &previousTimestampPacketNodes, diff --git a/unit_tests/command_stream/aub_command_stream_receiver_1_tests.cpp b/unit_tests/command_stream/aub_command_stream_receiver_1_tests.cpp index fdcc7133f9..0b541ffdd4 100644 --- a/unit_tests/command_stream/aub_command_stream_receiver_1_tests.cpp +++ b/unit_tests/command_stream/aub_command_stream_receiver_1_tests.cpp @@ -1159,8 +1159,10 @@ HWTEST_F(AubCommandStreamReceiverTests, WhenBlitBufferIsCalledThenCounterIsCorre EXPECT_EQ(0u, aubCsr->blitBufferCalled); MockGraphicsAllocation allocation(reinterpret_cast(0x1000), 0); - BlitProperties blitProperties = BlitProperties::constructPropertiesForCopyBuffer(&allocation, &allocation, true, 0, 0, 0); - aubCsr->blitBuffer(blitProperties); + BlitProperties blitProperties = BlitProperties::constructPropertiesForCopyBuffer(&allocation, &allocation, 0, 0, 0); + BlitPropertiesContainer blitPropertiesContainer; + blitPropertiesContainer.push_back(blitProperties); + aubCsr->blitBuffer(blitPropertiesContainer, true); EXPECT_EQ(1u, aubCsr->blitBufferCalled); } diff --git a/unit_tests/command_stream/command_stream_receiver_hw_tests.cpp b/unit_tests/command_stream/command_stream_receiver_hw_tests.cpp index 9feaa9eebf..5ee8361e7b 100644 --- a/unit_tests/command_stream/command_stream_receiver_hw_tests.cpp +++ b/unit_tests/command_stream/command_stream_receiver_hw_tests.cpp @@ -310,6 +310,13 @@ struct BcsTests : public CommandStreamReceiverHwTest { CommandStreamReceiverHwTest::TearDown(); } + uint32_t blitBuffer(CommandStreamReceiver *bcsCsr, const BlitProperties &blitProperties, bool blocking) { + BlitPropertiesContainer container; + container.push_back(blitProperties); + + return bcsCsr->blitBuffer(container, blocking); + } + TimestampPacketContainer timestampPacketContainer; CsrDependencies csrDependencies; std::unique_ptr context; @@ -322,10 +329,8 @@ HWTEST_F(BcsTests, givenBltSizeWhenEstimatingCommandSizeThenAddAllRequiredComman uint32_t alignedNumberOfBlts = 3; uint32_t notAlignedNumberOfBlts = 4; - size_t expectedSize = sizeof(typename FamilyType::MI_FLUSH_DW) + sizeof(typename FamilyType::MI_BATCH_BUFFER_END); - - auto expectedAlignedSize = alignUp(expectedSize + (sizeof(typename FamilyType::XY_COPY_BLT) * alignedNumberOfBlts), MemoryConstants::cacheLineSize); - auto expectedNotAlignedSize = alignUp(expectedSize + (sizeof(typename FamilyType::XY_COPY_BLT) * notAlignedNumberOfBlts), MemoryConstants::cacheLineSize); + auto expectedAlignedSize = sizeof(typename FamilyType::XY_COPY_BLT) * alignedNumberOfBlts; + auto expectedNotAlignedSize = sizeof(typename FamilyType::XY_COPY_BLT) * notAlignedNumberOfBlts; auto alignedEstimatedSize = BlitCommandsHelper::estimateBlitCommandsSize(alignedBltSize, csrDependencies, false); auto notAlignedEstimatedSize = BlitCommandsHelper::estimateBlitCommandsSize(notAlignedBltSize, csrDependencies, false); @@ -334,12 +339,38 @@ HWTEST_F(BcsTests, givenBltSizeWhenEstimatingCommandSizeThenAddAllRequiredComman EXPECT_EQ(expectedNotAlignedSize, notAlignedEstimatedSize); } -HWTEST_F(BcsTests, givenTimestampPacketWriteRequestWhenEstimatingSizeForCommandsThenAddMiFlushDw) { - size_t expectedBaseSize = sizeof(typename FamilyType::XY_COPY_BLT) + sizeof(typename FamilyType::MI_FLUSH_DW) + - sizeof(typename FamilyType::MI_BATCH_BUFFER_END); +HWTEST_F(BcsTests, givenBlitPropertiesContainerWhenExstimatingCommandsSizeThenCalculateForAllAttachedProperites) { + const auto max2DBlitSize = BlitterConstants::maxBlitWidth * BlitterConstants::maxBlitHeight; + const uint32_t numberOfBlts = 3; + const uint64_t bltSize = (3 * max2DBlitSize); + const uint32_t numberOfBlitOperations = 4; - auto expectedSizeWithTimestampPacketWrite = alignUp(expectedBaseSize + sizeof(typename FamilyType::MI_FLUSH_DW), MemoryConstants::cacheLineSize); - auto expectedSizeWithoutTimestampPacketWrite = alignUp(expectedBaseSize, MemoryConstants::cacheLineSize); + auto baseSize = sizeof(typename FamilyType::MI_FLUSH_DW) + sizeof(typename FamilyType::MI_BATCH_BUFFER_END); + auto expectedBlitInstructionsSize = sizeof(typename FamilyType::XY_COPY_BLT) * numberOfBlts; + + auto expectedAlignedSize = baseSize; + + BlitPropertiesContainer blitPropertiesContainer; + for (uint32_t i = 0; i < numberOfBlitOperations; i++) { + BlitProperties blitProperties; + blitProperties.copySize = bltSize; + blitPropertiesContainer.push_back(blitProperties); + + expectedAlignedSize += expectedBlitInstructionsSize; + } + + expectedAlignedSize = alignUp(expectedAlignedSize, MemoryConstants::cacheLineSize); + + auto alignedEstimatedSize = BlitCommandsHelper::estimateBlitCommandsSize(blitPropertiesContainer); + + EXPECT_EQ(expectedAlignedSize, alignedEstimatedSize); +} + +HWTEST_F(BcsTests, givenTimestampPacketWriteRequestWhenEstimatingSizeForCommandsThenAddMiFlushDw) { + size_t expectedBaseSize = sizeof(typename FamilyType::XY_COPY_BLT); + + auto expectedSizeWithTimestampPacketWrite = expectedBaseSize + sizeof(typename FamilyType::MI_FLUSH_DW); + auto expectedSizeWithoutTimestampPacketWrite = expectedBaseSize; auto estimatedSizeWithTimestampPacketWrite = BlitCommandsHelper::estimateBlitCommandsSize(1, csrDependencies, true); auto estimatedSizeWithoutTimestampPacketWrite = BlitCommandsHelper::estimateBlitCommandsSize(1, csrDependencies, false); @@ -358,15 +389,12 @@ HWTEST_F(BcsTests, givenBltSizeAndCsrDependenciesWhenEstimatingCommandSizeThenAd csrDependencies.push_back(×tamp0); csrDependencies.push_back(×tamp1); - size_t expectedSize = sizeof(typename FamilyType::MI_FLUSH_DW) + sizeof(typename FamilyType::MI_BATCH_BUFFER_END) + - (sizeof(typename FamilyType::XY_COPY_BLT) * numberOfBlts) + + size_t expectedSize = (sizeof(typename FamilyType::XY_COPY_BLT) * numberOfBlts) + TimestampPacketHelper::getRequiredCmdStreamSize(csrDependencies); - auto expectedAlignedSize = alignUp(expectedSize, MemoryConstants::cacheLineSize); - auto estimatedSize = BlitCommandsHelper::estimateBlitCommandsSize(1, csrDependencies, false); - EXPECT_EQ(expectedAlignedSize, estimatedSize); + EXPECT_EQ(expectedSize, estimatedSize); } HWTEST_F(BcsTests, givenBltSizeWithLeftoverWhenDispatchedThenProgramAllRequiredCommands) { @@ -389,9 +417,9 @@ HWTEST_F(BcsTests, givenBltSizeWithLeftoverWhenDispatchedThenProgramAllRequiredC EXPECT_EQ(0u, csr.recursiveLockCounter.load()); auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, csr, buffer->getGraphicsAllocation(), 0, nullptr, hostPtr, - 0, true, 0, bltSize); + 0, 0, bltSize); - csr.blitBuffer(blitProperties); + blitBuffer(&csr, blitProperties, true); EXPECT_EQ(newTaskCount, csr.taskCount); EXPECT_EQ(newTaskCount, csr.latestFlushedTaskCount); EXPECT_EQ(newTaskCount, csr.latestSentTaskCount); @@ -449,14 +477,14 @@ HWTEST_F(BcsTests, givenCsrDependenciesWhenProgrammingCommandStreamThenAddSemaph auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, csr, buffer->getGraphicsAllocation(), 0, nullptr, hostPtr, - 0, true, 0, 1); + 0, 0, 1); MockTimestampPacketContainer timestamp0(*csr.getTimestampPacketAllocator(), numberNodesPerContainer); MockTimestampPacketContainer timestamp1(*csr.getTimestampPacketAllocator(), numberNodesPerContainer); blitProperties.csrDependencies.push_back(×tamp0); blitProperties.csrDependencies.push_back(×tamp1); - csr.blitBuffer(blitProperties); + blitBuffer(&csr, blitProperties, true); HardwareParse hwParser; hwParser.parseCommands(csr.commandStream); @@ -486,27 +514,89 @@ HWTEST_F(BcsTests, givenCsrDependenciesWhenProgrammingCommandStreamThenAddSemaph EXPECT_TRUE(dependenciesFound); } +HWTEST_F(BcsTests, givenMultipleBlitPropertiesWhenDispatchingThenProgramCommandsInCorrectOrder) { + auto &csr = pDevice->getUltCommandStreamReceiver(); + + cl_int retVal = CL_SUCCESS; + auto buffer1 = clUniquePtr(Buffer::create(context.get(), CL_MEM_READ_WRITE, 1, nullptr, retVal)); + auto buffer2 = clUniquePtr(Buffer::create(context.get(), CL_MEM_READ_WRITE, 1, nullptr, retVal)); + void *hostPtr1 = reinterpret_cast(0x12340000); + void *hostPtr2 = reinterpret_cast(0x12340000); + + auto blitProperties1 = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, + csr, buffer1->getGraphicsAllocation(), 0, nullptr, hostPtr1, + 0, 0, 1); + auto blitProperties2 = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, + csr, buffer2->getGraphicsAllocation(), 0, nullptr, hostPtr2, + 0, 0, 1); + + MockTimestampPacketContainer timestamp1(*csr.getTimestampPacketAllocator(), 1); + MockTimestampPacketContainer timestamp2(*csr.getTimestampPacketAllocator(), 1); + blitProperties1.csrDependencies.push_back(×tamp1); + blitProperties2.csrDependencies.push_back(×tamp2); + + BlitPropertiesContainer blitPropertiesContainer; + blitPropertiesContainer.push_back(blitProperties1); + blitPropertiesContainer.push_back(blitProperties2); + + csr.blitBuffer(blitPropertiesContainer, true); + + HardwareParse hwParser; + hwParser.parseCommands(csr.commandStream); + auto &cmdList = hwParser.cmdList; + + uint32_t xyCopyBltCmdFound = 0; + uint32_t dependenciesFound = 0; + + for (auto cmdIterator = cmdList.begin(); cmdIterator != cmdList.end(); cmdIterator++) { + if (genCmdCast(*cmdIterator)) { + xyCopyBltCmdFound++; + EXPECT_EQ(xyCopyBltCmdFound, dependenciesFound); + + continue; + } + auto miSemaphore = genCmdCast(*cmdIterator); + if (miSemaphore) { + dependenciesFound++; + EXPECT_EQ(xyCopyBltCmdFound, dependenciesFound - 1); + } + } + EXPECT_EQ(2u, xyCopyBltCmdFound); + EXPECT_EQ(2u, dependenciesFound); +} + HWTEST_F(BcsTests, givenInputAllocationsWhenBlitDispatchedThenMakeAllAllocationsResident) { auto &csr = pDevice->getUltCommandStreamReceiver(); csr.storeMakeResidentAllocations = true; cl_int retVal = CL_SUCCESS; - auto buffer = clUniquePtr(Buffer::create(context.get(), CL_MEM_READ_WRITE, 1, nullptr, retVal)); - void *hostPtr = reinterpret_cast(0x12340000); + auto buffer1 = clUniquePtr(Buffer::create(context.get(), CL_MEM_READ_WRITE, 1, nullptr, retVal)); + auto buffer2 = clUniquePtr(Buffer::create(context.get(), CL_MEM_READ_WRITE, 1, nullptr, retVal)); + void *hostPtr1 = reinterpret_cast(0x12340000); + void *hostPtr2 = reinterpret_cast(0x43210000); EXPECT_EQ(0u, csr.makeSurfacePackNonResidentCalled); - auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, - csr, buffer->getGraphicsAllocation(), 0, nullptr, hostPtr, 0, - true, 0, 1); + auto blitProperties1 = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, + csr, buffer1->getGraphicsAllocation(), 0, nullptr, hostPtr1, + 0, 0, 1); - csr.blitBuffer(blitProperties); + auto blitProperties2 = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, + csr, buffer2->getGraphicsAllocation(), 0, nullptr, hostPtr2, + 0, 0, 1); - EXPECT_TRUE(csr.isMadeResident(buffer->getGraphicsAllocation())); + BlitPropertiesContainer blitPropertiesContainer; + blitPropertiesContainer.push_back(blitProperties1); + blitPropertiesContainer.push_back(blitProperties2); + + csr.blitBuffer(blitPropertiesContainer, false); + + EXPECT_TRUE(csr.isMadeResident(buffer1->getGraphicsAllocation())); + EXPECT_TRUE(csr.isMadeResident(buffer2->getGraphicsAllocation())); EXPECT_TRUE(csr.isMadeResident(csr.getTagAllocation())); EXPECT_EQ(1u, csr.makeSurfacePackNonResidentCalled); - EXPECT_EQ(3u, csr.makeResidentAllocations.size()); + EXPECT_EQ(5u, csr.makeResidentAllocations.size()); } HWTEST_F(BcsTests, givenBufferWhenBlitCalledThenFlushCommandBuffer) { @@ -526,9 +616,9 @@ HWTEST_F(BcsTests, givenBufferWhenBlitCalledThenFlushCommandBuffer) { auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, csr, buffer->getGraphicsAllocation(), 0, nullptr, hostPtr, - 0, true, 0, 1); + 0, 0, 1); - csr.blitBuffer(blitProperties); + blitBuffer(&csr, blitProperties, true); EXPECT_EQ(commandStream.getGraphicsAllocation(), csr.latestFlushedBatchBuffer.commandBufferAllocation); EXPECT_EQ(commandStreamOffset, csr.latestFlushedBatchBuffer.startOffset); @@ -575,14 +665,13 @@ HWTEST_F(BcsTests, whenBlitFromHostPtrCalledThenCallWaitWithKmdFallback) { auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, *myMockCsr, buffer->getGraphicsAllocation(), 0, nullptr, - hostPtr, 0, false, 0, 1); + hostPtr, 0, 0, 1); - myMockCsr->blitBuffer(blitProperties); + blitBuffer(myMockCsr.get(), blitProperties, false); EXPECT_EQ(0u, myMockCsr->waitForTaskCountWithKmdNotifyFallbackCalled); - blitProperties.blocking = true; - myMockCsr->blitBuffer(blitProperties); + blitBuffer(myMockCsr.get(), blitProperties, true); EXPECT_EQ(1u, myMockCsr->waitForTaskCountWithKmdNotifyFallbackCalled); EXPECT_EQ(myMockCsr->taskCount, myMockCsr->taskCountToWaitPassed); @@ -606,14 +695,13 @@ HWTEST_F(BcsTests, whenBlitFromHostPtrCalledThenCleanTemporaryAllocations) { auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, bcsCsr, buffer->getGraphicsAllocation(), 0, nullptr, hostPtr, - 0, false, 0, 1); + 0, 0, 1); - bcsCsr.blitBuffer(blitProperties); + blitBuffer(&bcsCsr, blitProperties, false); EXPECT_EQ(0u, mockInternalAllocationsStorage->cleanAllocationsCalled); - blitProperties.blocking = true; - bcsCsr.blitBuffer(blitProperties); + blitBuffer(&bcsCsr, blitProperties, true); EXPECT_EQ(1u, mockInternalAllocationsStorage->cleanAllocationsCalled); EXPECT_EQ(bcsCsr.taskCount, mockInternalAllocationsStorage->lastCleanAllocationsTaskCount); @@ -639,9 +727,9 @@ HWTEST_F(BcsTests, givenBufferWhenBlitOperationCalledThenProgramCorrectGpuAddres HardwareParse hwParser; auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, csr, subBuffer1->getGraphicsAllocation(), subBuffer1Offset, - nullptr, hostPtr, hostPtrOffset, true, 0, 1); + nullptr, hostPtr, hostPtrOffset, 0, 1); - csr.blitBuffer(blitProperties); + blitBuffer(&csr, blitProperties, true); hwParser.parseCommands(csr.commandStream); @@ -658,9 +746,9 @@ HWTEST_F(BcsTests, givenBufferWhenBlitOperationCalledThenProgramCorrectGpuAddres auto offset = csr.commandStream.getUsed(); auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::BufferToHostPtr, csr, subBuffer1->getGraphicsAllocation(), subBuffer1Offset, - nullptr, hostPtr, hostPtrOffset, true, 0, 1); + nullptr, hostPtr, hostPtrOffset, 0, 1); - csr.blitBuffer(blitProperties); + blitBuffer(&csr, blitProperties, true); hwParser.parseCommands(csr.commandStream, offset); @@ -676,10 +764,9 @@ HWTEST_F(BcsTests, givenBufferWhenBlitOperationCalledThenProgramCorrectGpuAddres HardwareParse hwParser; auto offset = csr.commandStream.getUsed(); auto blitProperties = BlitProperties::constructPropertiesForCopyBuffer(buffer1->getGraphicsAllocation(), - buffer2->getGraphicsAllocation(), - true, 0, 0, 1); + buffer2->getGraphicsAllocation(), 0, 0, 1); - csr.blitBuffer(blitProperties); + blitBuffer(&csr, blitProperties, true); hwParser.parseCommands(csr.commandStream, offset); @@ -710,9 +797,9 @@ HWTEST_F(BcsTests, givenMapAllocationWhenDispatchReadWriteOperationThenSetValidG HardwareParse hwParser; auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, csr, buffer->getGraphicsAllocation(), 0, - mapAllocation, mapPtr, hostPtrOffset, true, 0, 1); + mapAllocation, mapPtr, hostPtrOffset, 0, 1); - csr.blitBuffer(blitProperties); + blitBuffer(&csr, blitProperties, true); hwParser.parseCommands(csr.commandStream); @@ -730,9 +817,9 @@ HWTEST_F(BcsTests, givenMapAllocationWhenDispatchReadWriteOperationThenSetValidG auto offset = csr.commandStream.getUsed(); auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::BufferToHostPtr, csr, buffer->getGraphicsAllocation(), 0, - mapAllocation, mapPtr, hostPtrOffset, true, 0, 1); + mapAllocation, mapPtr, hostPtrOffset, 0, 1); - csr.blitBuffer(blitProperties); + blitBuffer(&csr, blitProperties, true); hwParser.parseCommands(csr.commandStream, offset); @@ -769,7 +856,7 @@ HWTEST_F(BcsTests, givenMapAllocationInBuiltinOpParamsWhenConstructingThenUseItA builtinOpParams.transferAllocation = mapAllocation; auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, - csr, builtinOpParams, true); + csr, builtinOpParams); EXPECT_EQ(mapAllocation, blitProperties.srcAllocation); } { @@ -781,7 +868,7 @@ HWTEST_F(BcsTests, givenMapAllocationInBuiltinOpParamsWhenConstructingThenUseItA builtinOpParams.transferAllocation = mapAllocation; auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::BufferToHostPtr, - csr, builtinOpParams, true); + csr, builtinOpParams); EXPECT_EQ(mapAllocation, blitProperties.dstAllocation); } @@ -810,7 +897,7 @@ HWTEST_F(BcsTests, givenNonZeroCopySvmAllocationWhenConstructingBlitPropertiesFo builtinOpParams.size.x = 1; auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, - csr, builtinOpParams, true); + csr, builtinOpParams); EXPECT_EQ(svmData->cpuAllocation, blitProperties.srcAllocation); EXPECT_EQ(svmData->gpuAllocation, blitProperties.dstAllocation); } @@ -823,7 +910,7 @@ HWTEST_F(BcsTests, givenNonZeroCopySvmAllocationWhenConstructingBlitPropertiesFo builtinOpParams.size.x = 1; auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::BufferToHostPtr, - csr, builtinOpParams, true); + csr, builtinOpParams); EXPECT_EQ(svmData->cpuAllocation, blitProperties.dstAllocation); EXPECT_EQ(svmData->gpuAllocation, blitProperties.srcAllocation); } @@ -848,9 +935,9 @@ HWTEST_F(BcsTests, givenBufferWithOffsetWhenBlitOperationCalledThenProgramCorrec auto offset = csr.commandStream.getUsed(); auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, csr, buffer1->getGraphicsAllocation(), 0, nullptr, - hostPtr, 0, true, buffer1Offset, 1); + hostPtr, 0, buffer1Offset, 1); - csr.blitBuffer(blitProperties); + blitBuffer(&csr, blitProperties, true); hwParser.parseCommands(csr.commandStream, offset); @@ -867,9 +954,9 @@ HWTEST_F(BcsTests, givenBufferWithOffsetWhenBlitOperationCalledThenProgramCorrec auto offset = csr.commandStream.getUsed(); auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::BufferToHostPtr, csr, buffer1->getGraphicsAllocation(), 0, nullptr, - hostPtr, 0, true, buffer1Offset, 1); + hostPtr, 0, buffer1Offset, 1); - csr.blitBuffer(blitProperties); + blitBuffer(&csr, blitProperties, true); hwParser.parseCommands(csr.commandStream, offset); @@ -887,9 +974,9 @@ HWTEST_F(BcsTests, givenBufferWithOffsetWhenBlitOperationCalledThenProgramCorrec auto offset = csr.commandStream.getUsed(); auto blitProperties = BlitProperties::constructPropertiesForCopyBuffer(buffer1->getGraphicsAllocation(), buffer2->getGraphicsAllocation(), - true, buffer1Offset, buffer2Offset, 1); + buffer1Offset, buffer2Offset, 1); - csr.blitBuffer(blitProperties); + blitBuffer(&csr, blitProperties, true); hwParser.parseCommands(csr.commandStream, offset); @@ -916,7 +1003,7 @@ HWTEST_F(BcsTests, givenAuxTranslationRequestWhenBlitCalledThenProgramCommandCor buffer->getGraphicsAllocation()); auto offset = csr.commandStream.getUsed(); - csr.blitBuffer(blitProperties); + blitBuffer(&csr, blitProperties, false); HardwareParse hwParser; hwParser.parseCommands(csr.commandStream, offset); diff --git a/unit_tests/kernel/kernel_tests.cpp b/unit_tests/kernel/kernel_tests.cpp index e974b77194..cd6952e06b 100644 --- a/unit_tests/kernel/kernel_tests.cpp +++ b/unit_tests/kernel/kernel_tests.cpp @@ -490,7 +490,7 @@ class CommandStreamReceiverMock : public CommandStreamReceiver { void waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait, bool quickKmdSleep, bool forcePowerSavingMode) override { } - uint32_t blitBuffer(const BlitProperties &blitProperties) override { return taskCount; }; + uint32_t blitBuffer(const BlitPropertiesContainer &blitPropertiesContainer, bool blocking) override { return taskCount; }; CompletionStamp flushTask( LinearStream &commandStream, diff --git a/unit_tests/libult/ult_aub_command_stream_receiver.h b/unit_tests/libult/ult_aub_command_stream_receiver.h index cff2dd1ec8..a60837239b 100644 --- a/unit_tests/libult/ult_aub_command_stream_receiver.h +++ b/unit_tests/libult/ult_aub_command_stream_receiver.h @@ -38,9 +38,9 @@ class UltAubCommandStreamReceiver : public AUBCommandStreamReceiverHw return csr; } - uint32_t blitBuffer(const BlitProperties &blitProperties) override { + uint32_t blitBuffer(const BlitPropertiesContainer &blitPropertiesContainer, bool blocking) override { blitBufferCalled++; - return BaseClass::blitBuffer(blitProperties); + return BaseClass::blitBuffer(blitPropertiesContainer, blocking); } uint32_t blitBufferCalled = 0; diff --git a/unit_tests/libult/ult_command_stream_receiver.h b/unit_tests/libult/ult_command_stream_receiver.h index 49f0fa2537..9a0256124b 100644 --- a/unit_tests/libult/ult_command_stream_receiver.h +++ b/unit_tests/libult/ult_command_stream_receiver.h @@ -174,9 +174,9 @@ class UltCommandStreamReceiver : public CommandStreamReceiverHw, publ return CommandStreamReceiverHw::obtainUniqueOwnership(); } - uint32_t blitBuffer(const BlitProperties &blitProperties) override { + uint32_t blitBuffer(const BlitPropertiesContainer &blitPropertiesContainer, bool blocking) override { blitBufferCalled++; - return CommandStreamReceiverHw::blitBuffer(blitProperties); + return CommandStreamReceiverHw::blitBuffer(blitPropertiesContainer, blocking); } bool createPerDssBackedBuffer(Device &device) override { diff --git a/unit_tests/mem_obj/buffer_tests.cpp b/unit_tests/mem_obj/buffer_tests.cpp index 31c51f2bf4..2490ad9be6 100644 --- a/unit_tests/mem_obj/buffer_tests.cpp +++ b/unit_tests/mem_obj/buffer_tests.cpp @@ -700,8 +700,12 @@ struct BcsBufferTests : public ::testing::Test { BlitOperationResult blitMemoryToAllocation(MemObj &memObj, GraphicsAllocation *memory, void *hostPtr, size_t size) const override { auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, *bcsCsr, memory, 0, nullptr, - hostPtr, 0, true, 0, size); - bcsCsr->blitBuffer(blitProperties); + hostPtr, 0, 0, size); + + BlitPropertiesContainer container; + container.push_back(blitProperties); + bcsCsr->blitBuffer(container, true); + return BlitOperationResult::Success; } std::unique_ptr bcsOsContext; @@ -1094,9 +1098,9 @@ HWTEST_TEMPLATED_F(BcsBufferTests, givenOutputTimestampPacketWhenBlitCalledThenP for (auto &cmd : hwParser.cmdList) { if (auto miFlushDwCmd = genCmdCast(cmd)) { EXPECT_TRUE(blitCmdFound); - EXPECT_EQ(miFlushDwCmdsCount == 1, + EXPECT_EQ(miFlushDwCmdsCount == 0, timestampPacketGpuWriteAddress == miFlushDwCmd->getDestinationAddress()); - EXPECT_EQ(miFlushDwCmdsCount == 1, + EXPECT_EQ(miFlushDwCmdsCount == 0, 0u == miFlushDwCmd->getImmediateData()); miFlushDwCmdsCount++; } else if (genCmdCast(cmd)) { diff --git a/unit_tests/mocks/mock_csr.h b/unit_tests/mocks/mock_csr.h index 8eaabea103..480897a054 100644 --- a/unit_tests/mocks/mock_csr.h +++ b/unit_tests/mocks/mock_csr.h @@ -215,9 +215,9 @@ class MockCsrHw2 : public CommandStreamReceiverHw { return completionStamp; } - uint32_t blitBuffer(const BlitProperties &blitProperties) override { + uint32_t blitBuffer(const BlitPropertiesContainer &blitPropertiesContainer, bool blocking) override { if (!skipBlitCalls) { - return CommandStreamReceiverHw::blitBuffer(blitProperties); + return CommandStreamReceiverHw::blitBuffer(blitPropertiesContainer, blocking); } return taskCount; } @@ -288,7 +288,7 @@ class MockCommandStreamReceiver : public CommandStreamReceiver { void waitForTaskCountWithKmdNotifyFallback(uint32_t taskCountToWait, FlushStamp flushStampToWait, bool quickKmdSleep, bool forcePowerSavingMode) override { } - uint32_t blitBuffer(const BlitProperties &blitProperties) override { return taskCount; }; + uint32_t blitBuffer(const BlitPropertiesContainer &blitPropertiesContainer, bool blocking) override { return taskCount; }; void setOSInterface(OSInterface *osInterface);