From 27654c9282990c58698e727138d9f940e62ef7e6 Mon Sep 17 00:00:00 2001 From: "Dunajski, Bartosz" Date: Mon, 1 Jul 2019 11:08:58 +0200 Subject: [PATCH] Buffer-to-Buffer blit operations support Change-Id: I76c9fae83fa2a31bd6108999c7f77f4a47c47f1b Signed-off-by: Dunajski, Bartosz Related-To: NEO-3020 --- .../command_queue/command_queue_hw_base.inl | 4 +-- runtime/helpers/blit_commands_helper.cpp | 26 ++++++++------- runtime/helpers/blit_commands_helper.h | 7 ++-- runtime/mem_obj/buffer.cpp | 2 +- runtime/memory_manager/memory_constants.h | 7 ++-- .../command_stream_receiver_hw_tests.cpp | 33 +++++++++---------- 6 files changed, 43 insertions(+), 36 deletions(-) diff --git a/runtime/command_queue/command_queue_hw_base.inl b/runtime/command_queue/command_queue_hw_base.inl index 3fe0f0b2f8..30d1c70509 100644 --- a/runtime/command_queue/command_queue_hw_base.inl +++ b/runtime/command_queue/command_queue_hw_base.inl @@ -108,8 +108,8 @@ cl_int CommandQueueHw::enqueueReadWriteBufferWithBlitTransfer(cl_command EventsRequest eventsRequest(numEventsInWaitList, eventWaitList, event); TimestampPacketContainer previousTimestampPacketNodes; - auto copyDirection = (CL_COMMAND_WRITE_BUFFER == commandType) ? BlitterConstants::BlitWithHostPtrDirection::FromHostPtr - : BlitterConstants::BlitWithHostPtrDirection::ToHostPtr; + auto copyDirection = (CL_COMMAND_WRITE_BUFFER == commandType) ? BlitterConstants::BlitDirection::HostPtrToBuffer + : BlitterConstants::BlitDirection::BufferToHostPtr; auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(copyDirection, *blitCommandStreamReceiver, buffer->getGraphicsAllocation(), ptr, blocking, offset, size); diff --git a/runtime/helpers/blit_commands_helper.cpp b/runtime/helpers/blit_commands_helper.cpp index 5aaf32c268..b5c8e34269 100644 --- a/runtime/helpers/blit_commands_helper.cpp +++ b/runtime/helpers/blit_commands_helper.cpp @@ -13,23 +13,27 @@ #include "CL/cl.h" namespace NEO { -BlitProperties BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection copyDirection, +BlitProperties BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection blitDirection, CommandStreamReceiver &commandStreamReceiver, GraphicsAllocation *memObjAllocation, void *hostPtr, bool blocking, size_t offset, uint64_t copySize) { - GraphicsAllocation *hostPtrAllocation = nullptr; - if (hostPtr) { - HostPtrSurface hostPtrSurface(hostPtr, static_cast(copySize), true); - bool success = commandStreamReceiver.createAllocationForHostSurface(hostPtrSurface, false); - UNRECOVERABLE_IF(!success); - hostPtrAllocation = hostPtrSurface.getAllocation(); - } - if (BlitterConstants::BlitWithHostPtrDirection::FromHostPtr == copyDirection) { - return {nullptr, copyDirection, {}, memObjAllocation, hostPtrAllocation, hostPtr, blocking, offset, 0, copySize}; + HostPtrSurface hostPtrSurface(hostPtr, static_cast(copySize), true); + bool success = commandStreamReceiver.createAllocationForHostSurface(hostPtrSurface, false); + UNRECOVERABLE_IF(!success); + auto hostPtrAllocation = hostPtrSurface.getAllocation(); + + if (BlitterConstants::BlitDirection::HostPtrToBuffer == blitDirection) { + return {nullptr, blitDirection, {}, memObjAllocation, hostPtrAllocation, hostPtr, blocking, offset, 0, copySize}; } else { - return {nullptr, copyDirection, {}, hostPtrAllocation, memObjAllocation, hostPtr, blocking, 0, offset, copySize}; + return {nullptr, blitDirection, {}, hostPtrAllocation, memObjAllocation, hostPtr, blocking, 0, offset, copySize}; } } +BlitProperties BlitProperties::constructPropertiesForCopyBuffer(GraphicsAllocation *dstAllocation, GraphicsAllocation *srcAllocation, + bool blocking, size_t dstOffset, size_t srcOffset, uint64_t copySize) { + + return {nullptr, BlitterConstants::BlitDirection::BufferToBuffer, {}, dstAllocation, srcAllocation, nullptr, blocking, dstOffset, srcOffset, copySize}; +} + } // namespace NEO diff --git a/runtime/helpers/blit_commands_helper.h b/runtime/helpers/blit_commands_helper.h index 5e709e848f..a4c2dcae1a 100644 --- a/runtime/helpers/blit_commands_helper.h +++ b/runtime/helpers/blit_commands_helper.h @@ -20,13 +20,16 @@ class TimestampPacketContainer; struct BlitProperties { BlitProperties() = delete; - static BlitProperties constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection copyDirection, + static BlitProperties constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection blitDirection, CommandStreamReceiver &commandStreamReceiver, GraphicsAllocation *memObjAllocation, void *hostPtr, bool blocking, size_t offset, uint64_t copySize); + static BlitProperties constructPropertiesForCopyBuffer(GraphicsAllocation *dstAllocation, GraphicsAllocation *srcAllocation, + bool blocking, size_t dstOffset, size_t srcOffset, uint64_t copySize); + TimestampPacketContainer *outputTimestampPacket = nullptr; - BlitterConstants::BlitWithHostPtrDirection copyDirection; + BlitterConstants::BlitDirection blitDirection; CsrDependencies csrDependencies; GraphicsAllocation *dstAllocation = nullptr; diff --git a/runtime/mem_obj/buffer.cpp b/runtime/mem_obj/buffer.cpp index 7c310e5137..30f5d43771 100644 --- a/runtime/mem_obj/buffer.cpp +++ b/runtime/mem_obj/buffer.cpp @@ -286,7 +286,7 @@ Buffer *Buffer::create(Context *context, if (gpuCopyRequired) { auto blitCommandStreamReceiver = context->getCommandStreamReceiverForBlitOperation(*pBuffer); if (blitCommandStreamReceiver) { - auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection::FromHostPtr, + auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, *blitCommandStreamReceiver, memory, hostPtr, true, 0, size); blitCommandStreamReceiver->blitBuffer(blitProperties); diff --git a/runtime/memory_manager/memory_constants.h b/runtime/memory_manager/memory_constants.h index 29964ff8ad..8f66730c12 100644 --- a/runtime/memory_manager/memory_constants.h +++ b/runtime/memory_manager/memory_constants.h @@ -51,8 +51,9 @@ static const uint64_t maxSvmAddress = is64bit ? maxNBitValue<47> : maxNBitValue< namespace BlitterConstants { static constexpr uint64_t maxBlitWidth = 0x7FC0; // 0x7FFF aligned to cacheline size static constexpr uint64_t maxBlitHeight = 0x7FFF; -enum class BlitWithHostPtrDirection { - ToHostPtr, - FromHostPtr +enum class BlitDirection { + BufferToHostPtr, + HostPtrToBuffer, + BufferToBuffer }; } // namespace BlitterConstants diff --git a/unit_tests/command_stream/command_stream_receiver_hw_tests.cpp b/unit_tests/command_stream/command_stream_receiver_hw_tests.cpp index 7d369b3e3a..171ca4c02e 100644 --- a/unit_tests/command_stream/command_stream_receiver_hw_tests.cpp +++ b/unit_tests/command_stream/command_stream_receiver_hw_tests.cpp @@ -365,7 +365,7 @@ HWTEST_F(BcsTests, givenBltSizeWithLeftoverWhenDispatchedThenProgramAllRequiredC uint32_t newTaskCount = 19; csr.taskCount = newTaskCount - 1; EXPECT_EQ(0u, csr.recursiveLockCounter.load()); - auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection::FromHostPtr, + auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, csr, buffer->getGraphicsAllocation(), hostPtr, true, 0, bltSize); csr.blitBuffer(blitProperties); @@ -424,7 +424,7 @@ HWTEST_F(BcsTests, givenCsrDependenciesWhenProgrammingCommandStreamThenAddSemaph uint32_t numberOfDependencyContainers = 2; size_t numberNodesPerContainer = 5; - auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection::FromHostPtr, + auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, csr, buffer->getGraphicsAllocation(), hostPtr, true, 0, 1); MockTimestampPacketContainer timestamp0(*csr.getTimestampPacketAllocator(), numberNodesPerContainer); @@ -472,7 +472,7 @@ HWTEST_F(BcsTests, givenInputAllocationsWhenBlitDispatchedThenMakeAllAllocations EXPECT_EQ(0u, csr.makeSurfacePackNonResidentCalled); - auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection::FromHostPtr, + auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, csr, buffer->getGraphicsAllocation(), hostPtr, true, 0, 1); csr.blitBuffer(blitProperties); @@ -500,7 +500,7 @@ HWTEST_F(BcsTests, givenBufferWhenBlitCalledThenFlushCommandBuffer) { uint32_t newTaskCount = 17; csr.taskCount = newTaskCount - 1; - auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection::FromHostPtr, + auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, csr, buffer->getGraphicsAllocation(), hostPtr, true, 0, 1); csr.blitBuffer(blitProperties); @@ -548,7 +548,7 @@ HWTEST_F(BcsTests, whenBlitFromHostPtrCalledThenCallWaitWithKmdFallback) { auto buffer = clUniquePtr(Buffer::create(context.get(), CL_MEM_READ_WRITE, 1, nullptr, retVal)); void *hostPtr = reinterpret_cast(0x12340000); - auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection::FromHostPtr, + auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, *myMockCsr, buffer->getGraphicsAllocation(), hostPtr, false, 0, 1); myMockCsr->blitBuffer(blitProperties); @@ -578,7 +578,7 @@ HWTEST_F(BcsTests, whenBlitFromHostPtrCalledThenCleanTemporaryAllocations) { EXPECT_EQ(0u, mockInternalAllocationsStorage->cleanAllocationsCalled); - auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection::FromHostPtr, + auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, bcsCsr, buffer->getGraphicsAllocation(), hostPtr, false, 0, 1); bcsCsr.blitBuffer(blitProperties); @@ -604,7 +604,7 @@ HWTEST_F(BcsTests, givenBufferWhenBlitOperationCalledThenProgramCorrectGpuAddres { // from hostPtr HardwareParse hwParser; - auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection::FromHostPtr, + auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, csr, buffer1->getGraphicsAllocation(), hostPtr, true, 0, 1); csr.blitBuffer(blitProperties); @@ -622,7 +622,7 @@ HWTEST_F(BcsTests, givenBufferWhenBlitOperationCalledThenProgramCorrectGpuAddres // to hostPtr HardwareParse hwParser; auto offset = csr.commandStream.getUsed(); - auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection::ToHostPtr, + auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::BufferToHostPtr, csr, buffer1->getGraphicsAllocation(), hostPtr, true, 0, 1); csr.blitBuffer(blitProperties); @@ -640,9 +640,9 @@ HWTEST_F(BcsTests, givenBufferWhenBlitOperationCalledThenProgramCorrectGpuAddres // Buffer to Buffer HardwareParse hwParser; auto offset = csr.commandStream.getUsed(); - auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection::FromHostPtr, - csr, buffer1->getGraphicsAllocation(), nullptr, true, 0, 1); - blitProperties.srcAllocation = buffer2->getGraphicsAllocation(); + auto blitProperties = BlitProperties::constructPropertiesForCopyBuffer(buffer1->getGraphicsAllocation(), + buffer2->getGraphicsAllocation(), + true, 0, 0, 1); csr.blitBuffer(blitProperties); @@ -670,7 +670,7 @@ HWTEST_F(BcsTests, givenBufferWithOffsetWhenBlitOperationCalledThenProgramCorrec // from hostPtr HardwareParse hwParser; auto offset = csr.commandStream.getUsed(); - auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection::FromHostPtr, + auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::HostPtrToBuffer, csr, buffer1->getGraphicsAllocation(), hostPtr, true, buffer1Offset, 1); csr.blitBuffer(blitProperties); @@ -688,7 +688,7 @@ HWTEST_F(BcsTests, givenBufferWithOffsetWhenBlitOperationCalledThenProgramCorrec // to hostPtr HardwareParse hwParser; auto offset = csr.commandStream.getUsed(); - auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection::ToHostPtr, + auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitDirection::BufferToHostPtr, csr, buffer1->getGraphicsAllocation(), hostPtr, true, buffer1Offset, 1); csr.blitBuffer(blitProperties); @@ -707,10 +707,9 @@ HWTEST_F(BcsTests, givenBufferWithOffsetWhenBlitOperationCalledThenProgramCorrec // Buffer to Buffer HardwareParse hwParser; auto offset = csr.commandStream.getUsed(); - auto blitProperties = BlitProperties::constructPropertiesForReadWriteBuffer(BlitterConstants::BlitWithHostPtrDirection::FromHostPtr, - csr, buffer1->getGraphicsAllocation(), nullptr, true, buffer1Offset, 1); - blitProperties.srcAllocation = buffer2->getGraphicsAllocation(); - blitProperties.srcOffset = buffer2Offset; + auto blitProperties = BlitProperties::constructPropertiesForCopyBuffer(buffer1->getGraphicsAllocation(), + buffer2->getGraphicsAllocation(), + true, buffer1Offset, buffer2Offset, 1); csr.blitBuffer(blitProperties);