From 89195ca4a3ed4692ca15fa5c0482c84042572ebb Mon Sep 17 00:00:00 2001 From: Maciej Plewka Date: Thu, 21 May 2020 12:36:23 +0200 Subject: [PATCH] Add support for copy images by bliter Change-Id: I0082fafb3363c6c6eb6973e5e35fa4d3ba1b6465 Signed-off-by: Maciej Plewka --- level_zero/core/source/cmdlist/cmdlist_hw.h | 8 + level_zero/core/source/cmdlist/cmdlist_hw.inl | 109 +++++++++--- .../sources/cmdlist/test_cmdlist.cpp | 85 +++++++++ .../command_stream_receiver_hw_gen12lp.cpp | 104 +++++++++++ shared/source/helpers/blit_commands_helper.h | 10 ++ .../helpers/blit_commands_helper_base.inl | 38 +++- .../helpers/blit_commands_helper_bdw_plus.inl | 19 ++ shared/test/unit_test/helpers/CMakeLists.txt | 1 + .../helpers/blit_commands_helper_tests.cpp | 36 ++++ .../blit_commands_helper_tests_gen12lp.cpp | 162 ++++++++++++++++++ 10 files changed, 548 insertions(+), 24 deletions(-) create mode 100644 shared/test/unit_test/helpers/blit_commands_helper_tests_gen12lp.cpp diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.h b/level_zero/core/source/cmdlist/cmdlist_hw.h index 3d97d10fd4..a53784dfb0 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.h +++ b/level_zero/core/source/cmdlist/cmdlist_hw.h @@ -162,6 +162,14 @@ struct CommandListCoreFamily : CommandListImp { size_t patternSize, size_t size, ze_event_handle_t hEvent); + MOCKABLE_VIRTUAL ze_result_t appendCopyImageBlit(NEO::GraphicsAllocation *src, + NEO::GraphicsAllocation *dst, + Vec3 srcOffsets, Vec3 dstOffsets, + size_t srcRowPitch, size_t srcSlicePitch, + size_t dstRowPitch, size_t dstSlicePitch, + size_t bytesPerPixel, Vec3 copySize, + Vec3 srcSize, Vec3 dstSize); + ze_result_t appendLaunchKernelWithParams(ze_kernel_handle_t hKernel, const ze_group_count_t *pThreadGroupDimensions, ze_event_handle_t hEvent, uint32_t numWaitEvents, diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.inl b/level_zero/core/source/cmdlist/cmdlist_hw.inl index 9536266a96..f05f5ca1d8 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.inl +++ b/level_zero/core/source/cmdlist/cmdlist_hw.inl @@ -247,15 +247,18 @@ ze_result_t CommandListCoreFamily::appendImageCopyFromMemory(ze_i auto image = Image::fromHandle(hDstImage); auto bytesPerPixel = static_cast(image->getImageInfo().surfaceFormat->ImageElementSizeInBytes); - ze_image_region_t tmpRegion; + Vec3 imgSize = {static_cast(image->getImageInfo().imgDesc.imageWidth), + static_cast(image->getImageInfo().imgDesc.imageHeight), + static_cast(image->getImageInfo().imgDesc.imageDepth)}; + ze_image_region_t tmpRegion; if (pDstRegion == nullptr) { tmpRegion = {0, 0, 0, - static_cast(image->getImageInfo().imgDesc.imageWidth), - static_cast(image->getImageInfo().imgDesc.imageHeight), - static_cast(image->getImageInfo().imgDesc.imageDepth)}; + imgSize.x, + imgSize.y, + imgSize.z}; pDstRegion = &tmpRegion; } @@ -263,6 +266,16 @@ ze_result_t CommandListCoreFamily::appendImageCopyFromMemory(ze_i auto allocationStruct = getAlignedAllocation(this->device, srcPtr, bufferSize); + auto rowPitch = pDstRegion->width * bytesPerPixel; + auto slicePitch = + image->getImageInfo().imgDesc.imageType == NEO::ImageType::Image1DArray ? 1 : pDstRegion->height * rowPitch; + + if (isCopyOnlyCmdList) { + return appendCopyImageBlit(allocationStruct.alloc, image->getAllocation(), + {0, 0, 0}, {pDstRegion->originX, pDstRegion->originY, pDstRegion->originZ}, rowPitch, slicePitch, + rowPitch, slicePitch, bytesPerPixel, {pDstRegion->width, pDstRegion->height, pDstRegion->depth}, {pDstRegion->width, pDstRegion->height, pDstRegion->depth}, imgSize); + } + Kernel *builtinKernel = nullptr; switch (bytesPerPixel) { @@ -297,13 +310,9 @@ ze_result_t CommandListCoreFamily::appendImageCopyFromMemory(ze_i 0}; builtinKernel->setArgumentValue(3u, sizeof(origin), &origin); - auto srcRowPitch = pDstRegion->width * bytesPerPixel; - auto srcSlicePitch = - (image->getImageInfo().imgDesc.imageType == NEO::ImageType::Image1DArray ? 1 : pDstRegion->height) * srcRowPitch; - uint32_t pitch[] = { - srcRowPitch, - srcSlicePitch}; + rowPitch, + slicePitch}; builtinKernel->setArgumentValue(4u, sizeof(pitch), &pitch); uint32_t groupSizeX = pDstRegion->width; @@ -343,15 +352,19 @@ ze_result_t CommandListCoreFamily::appendImageCopyToMemory(void * auto image = Image::fromHandle(hSrcImage); auto bytesPerPixel = static_cast(image->getImageInfo().surfaceFormat->ImageElementSizeInBytes); - ze_image_region_t tmpRegion; + Vec3 imgSize = {static_cast(image->getImageInfo().imgDesc.imageWidth), + static_cast(image->getImageInfo().imgDesc.imageHeight), + static_cast(image->getImageInfo().imgDesc.imageDepth)}; + + ze_image_region_t tmpRegion; if (pSrcRegion == nullptr) { tmpRegion = {0, 0, 0, - static_cast(image->getImageInfo().imgDesc.imageWidth), - static_cast(image->getImageInfo().imgDesc.imageHeight), - static_cast(image->getImageInfo().imgDesc.imageDepth)}; + imgSize.x, + imgSize.y, + imgSize.z}; pSrcRegion = &tmpRegion; } @@ -359,6 +372,16 @@ ze_result_t CommandListCoreFamily::appendImageCopyToMemory(void * auto allocationStruct = getAlignedAllocation(this->device, dstPtr, bufferSize); + auto rowPitch = pSrcRegion->width * bytesPerPixel; + auto slicePitch = + (image->getImageInfo().imgDesc.imageType == NEO::ImageType::Image1DArray ? 1 : pSrcRegion->height) * rowPitch; + + if (isCopyOnlyCmdList) { + return appendCopyImageBlit(image->getAllocation(), allocationStruct.alloc, + {pSrcRegion->originX, pSrcRegion->originY, pSrcRegion->originZ}, {0, 0, 0}, rowPitch, slicePitch, + rowPitch, slicePitch, bytesPerPixel, {pSrcRegion->width, pSrcRegion->height, pSrcRegion->depth}, imgSize, {pSrcRegion->width, pSrcRegion->height, pSrcRegion->depth}); + } + Kernel *builtinKernel = nullptr; switch (bytesPerPixel) { @@ -394,13 +417,9 @@ ze_result_t CommandListCoreFamily::appendImageCopyToMemory(void * builtinKernel->setArgumentValue(3u, sizeof(size_t), &allocationStruct.offset); - auto srcRowPitch = pSrcRegion->width * bytesPerPixel; - auto srcSlicePitch = - (image->getImageInfo().imgDesc.imageType == NEO::ImageType::Image1DArray ? 1 : pSrcRegion->height) * srcRowPitch; - uint32_t pitch[] = { - srcRowPitch, - srcSlicePitch}; + rowPitch, + slicePitch}; builtinKernel->setArgumentValue(4u, sizeof(pitch), &pitch); uint32_t groupSizeX = pSrcRegion->width; @@ -445,8 +464,6 @@ ze_result_t CommandListCoreFamily::appendImageCopyRegion(ze_image ze_event_handle_t hEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents) { - - auto kernel = device->getBuiltinFunctionsLib()->getImageFunction(ImageBuiltin::CopyImageRegion); auto dstImage = L0::Image::fromHandle(hDstImage); auto srcImage = L0::Image::fromHandle(hSrcImage); cl_int4 srcOffset, dstOffset; @@ -487,6 +504,32 @@ ze_result_t CommandListCoreFamily::appendImageCopyRegion(ze_image uint32_t groupSizeY = srcRegion.height; uint32_t groupSizeZ = srcRegion.depth; + if (isCopyOnlyCmdList) { + auto bytesPerPixel = static_cast(srcImage->getImageInfo().surfaceFormat->ImageElementSizeInBytes); + + Vec3 srcImgSize = {static_cast(srcImage->getImageInfo().imgDesc.imageWidth), + static_cast(srcImage->getImageInfo().imgDesc.imageHeight), + static_cast(srcImage->getImageInfo().imgDesc.imageDepth)}; + + Vec3 dstImgSize = {static_cast(dstImage->getImageInfo().imgDesc.imageWidth), + static_cast(dstImage->getImageInfo().imgDesc.imageHeight), + static_cast(dstImage->getImageInfo().imgDesc.imageDepth)}; + + auto srcRowPitch = srcRegion.width * bytesPerPixel; + auto srcSlicePitch = + (srcImage->getImageInfo().imgDesc.imageType == NEO::ImageType::Image1DArray ? 1 : srcRegion.height) * srcRowPitch; + + auto dstRowPitch = dstRegion.width * bytesPerPixel; + auto dstSlicePitch = + (dstImage->getImageInfo().imgDesc.imageType == NEO::ImageType::Image1DArray ? 1 : dstRegion.height) * dstRowPitch; + + return appendCopyImageBlit(srcImage->getAllocation(), dstImage->getAllocation(), + {srcRegion.originX, srcRegion.originY, srcRegion.originZ}, {dstRegion.originX, dstRegion.originY, dstRegion.originZ}, srcRowPitch, srcSlicePitch, + dstRowPitch, dstSlicePitch, bytesPerPixel, {srcRegion.width, srcRegion.height, srcRegion.depth}, srcImgSize, dstImgSize); + } + + auto kernel = device->getBuiltinFunctionsLib()->getImageFunction(ImageBuiltin::CopyImageRegion); + if (kernel->suggestGroupSize(groupSizeX, groupSizeY, groupSizeZ, &groupSizeX, &groupSizeY, &groupSizeZ) != ZE_RESULT_SUCCESS) { DEBUG_BREAK_IF(true); @@ -615,6 +658,28 @@ ze_result_t CommandListCoreFamily::appendMemoryCopyBlitRegion(NEO return ZE_RESULT_SUCCESS; } +template +ze_result_t CommandListCoreFamily::appendCopyImageBlit(NEO::GraphicsAllocation *src, + NEO::GraphicsAllocation *dst, + Vec3 srcOffsets, Vec3 dstOffsets, + size_t srcRowPitch, size_t srcSlicePitch, + size_t dstRowPitch, size_t dstSlicePitch, + size_t bytesPerPixel, Vec3 copySize, + Vec3 srcSize, Vec3 dstSize) { + using GfxFamily = typename NEO::GfxFamilyMapper::GfxFamily; + + auto blitProperties = NEO::BlitProperties::constructPropertiesForCopyBuffer(dst, src, + dstOffsets, srcOffsets, copySize, srcRowPitch, srcSlicePitch, + dstRowPitch, dstSlicePitch); + blitProperties.bytesPerPixel = bytesPerPixel; + blitProperties.srcSize = srcSize; + blitProperties.dstSize = dstSize; + commandContainer.addToResidencyContainer(dst); + commandContainer.addToResidencyContainer(src); + NEO::BlitCommandsHelper::dispatchBlitCommandsForImages(blitProperties, *commandContainer.getCommandStream(), *device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[device->getRootDeviceIndex()]); + return ZE_RESULT_SUCCESS; +} + template ze_result_t CommandListCoreFamily::appendPageFaultCopy(NEO::GraphicsAllocation *dstptr, NEO::GraphicsAllocation *srcptr, diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist.cpp index 558adb1719..32f346eab3 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist.cpp @@ -14,6 +14,7 @@ #include "level_zero/core/source/cmdlist/cmdlist_hw.h" #include "level_zero/core/source/driver/driver_handle_imp.h" +#include "level_zero/core/source/image/image_hw.h" #include "level_zero/core/test/unit_tests/fixtures/device_fixture.h" #include "level_zero/core/test/unit_tests/mocks/mock_event.h" #include "level_zero/core/test/unit_tests/mocks/mock_kernel.h" @@ -308,12 +309,23 @@ class MockCommandList : public WhiteBox<::L0::CommandListCoreFamily srcOffsets, Vec3 dstOffsets, + size_t srcRowPitch, size_t srcSlicePitch, + size_t dstRowPitch, size_t dstSlicePitch, + size_t bytesPerPixel, Vec3 copySize, + Vec3 srcSize, Vec3 dstSize) override { + appendCopyImageBlitCalledTimes++; + return ZE_RESULT_SUCCESS; + } uint32_t appendMemoryCopyKernelWithGACalledTimes = 0; uint32_t appendMemoryCopyBlitCalledTimes = 0; uint32_t appendMemoryCopyBlitRegionCalledTimes = 0; uint32_t appendMemoryCopyKernel2dCalledTimes = 0; uint32_t appendMemoryCopyKernel3dCalledTimes = 0; uint32_t appendBlitFillCalledTimes = 0; + uint32_t appendCopyImageBlitCalledTimes = 0; }; using Platforms = IsAtLeastProduct; @@ -664,5 +676,78 @@ HWTEST2_F(CommandListCreate, givenCopyOnlyCommandListWhenAppenBlitFillThenCopyBl auto itor = find(cmdList.begin(), cmdList.end()); EXPECT_NE(cmdList.end(), itor); } + +using ImageSupport = IsWithinProducts; + +HWTEST2_F(CommandListCreate, givenCopyCommandListWhenCopyFromMemoryToImageThenBlitImageCopyCalled, ImageSupport) { + MockCommandList cmdList; + cmdList.initialize(device, true); + MockDriverHandle driverHandle; + device->setDriverHandle(&driverHandle); + + void *srcPtr = reinterpret_cast(0x1234); + + ze_image_desc_t zeDesc = {}; + auto imageHW = std::make_unique>>(); + imageHW->initialize(device, &zeDesc); + + ze_image_region_t dstRegion = {4, 4, 4, 2, 2, 2}; + cmdList.appendImageCopyFromMemory(imageHW->toHandle(), srcPtr, &dstRegion, nullptr, 0, nullptr); + EXPECT_GT(cmdList.appendCopyImageBlitCalledTimes, 0u); +} + +HWTEST2_F(CommandListCreate, givenCopyCommandListWhenCopyFromImageToMemoryThenBlitImageCopyCalled, ImageSupport) { + MockCommandList cmdList; + cmdList.initialize(device, true); + MockDriverHandle driverHandle; + device->setDriverHandle(&driverHandle); + + void *dstPtr = reinterpret_cast(0x1234); + + ze_image_desc_t zeDesc = {}; + auto imageHW = std::make_unique>>(); + imageHW->initialize(device, &zeDesc); + + ze_image_region_t srcRegion = {4, 4, 4, 2, 2, 2}; + cmdList.appendImageCopyToMemory(dstPtr, imageHW->toHandle(), &srcRegion, nullptr, 0, nullptr); + EXPECT_GT(cmdList.appendCopyImageBlitCalledTimes, 0u); +} + +HWTEST2_F(CommandListCreate, givenCopyCommandListWhenCopyFromImageToImageThenBlitImageCopyCalled, ImageSupport) { + MockCommandList cmdList; + cmdList.initialize(device, true); + MockDriverHandle driverHandle; + device->setDriverHandle(&driverHandle); + + ze_image_desc_t zeDesc = {}; + auto imageHWSrc = std::make_unique>>(); + auto imageHWDst = std::make_unique>>(); + imageHWSrc->initialize(device, &zeDesc); + imageHWDst->initialize(device, &zeDesc); + + ze_image_region_t srcRegion = {4, 4, 4, 2, 2, 2}; + ze_image_region_t dstRegion = {4, 4, 4, 2, 2, 2}; + cmdList.appendImageCopyRegion(imageHWDst->toHandle(), imageHWSrc->toHandle(), &dstRegion, &srcRegion, nullptr, 0, nullptr); + EXPECT_GT(cmdList.appendCopyImageBlitCalledTimes, 0u); +} + +HWTEST2_F(CommandListCreate, givenCopyCommandListWhenCopyFromImagBlitThenCommandAddedToStream, ImageSupport) { + using GfxFamily = typename NEO::GfxFamilyMapper::GfxFamily; + using XY_COPY_BLT = typename GfxFamily::XY_COPY_BLT; + std::unique_ptr commandList(CommandList::create(productFamily, device, true)); + ze_image_desc_t zeDesc = {}; + auto imageHWSrc = std::make_unique>>(); + auto imageHWDst = std::make_unique>>(); + imageHWSrc->initialize(device, &zeDesc); + imageHWDst->initialize(device, &zeDesc); + + commandList->appendImageCopyRegion(imageHWDst->toHandle(), imageHWSrc->toHandle(), nullptr, nullptr, nullptr, 0, nullptr); + GenCmdList cmdList; + ASSERT_TRUE(FamilyType::PARSE::parseCommandBuffer( + cmdList, ptrOffset(commandList->commandContainer.getCommandStream()->getCpuBase(), 0), commandList->commandContainer.getCommandStream()->getUsed())); + auto itor = find(cmdList.begin(), cmdList.end()); + EXPECT_NE(cmdList.end(), itor); +} + } // namespace ult } // namespace L0 diff --git a/shared/source/gen12lp/command_stream_receiver_hw_gen12lp.cpp b/shared/source/gen12lp/command_stream_receiver_hw_gen12lp.cpp index 692d84fc03..738b3eae12 100644 --- a/shared/source/gen12lp/command_stream_receiver_hw_gen12lp.cpp +++ b/shared/source/gen12lp/command_stream_receiver_hw_gen12lp.cpp @@ -59,6 +59,110 @@ void populateFactoryTable>() { commandStreamReceiverFactory[gfxCore] = DeviceCommandStreamReceiver::create; } +template <> +void BlitCommandsHelper::appendColorDepth(const BlitProperties &blitProperites, typename Family::XY_COPY_BLT &blitCmd) { + using XY_COPY_BLT = typename Family::XY_COPY_BLT; + switch (blitProperites.bytesPerPixel) { + default: + UNRECOVERABLE_IF(true); + case 1: + blitCmd.setColorDepth(XY_COPY_BLT::COLOR_DEPTH::COLOR_DEPTH_8_BIT_COLOR); + break; + case 2: + blitCmd.setColorDepth(XY_COPY_BLT::COLOR_DEPTH::COLOR_DEPTH_16_BIT_COLOR); + break; + case 4: + blitCmd.setColorDepth(XY_COPY_BLT::COLOR_DEPTH::COLOR_DEPTH_32_BIT_COLOR); + break; + case 8: + blitCmd.setColorDepth(XY_COPY_BLT::COLOR_DEPTH::COLOR_DEPTH_64_BIT_COLOR); + break; + case 16: + blitCmd.setColorDepth(XY_COPY_BLT::COLOR_DEPTH::COLOR_DEPTH_128_BIT_COLOR); + break; + } +} + +template <> +void BlitCommandsHelper::appendTilingType(const GMM_TILE_TYPE srcTilingType, const GMM_TILE_TYPE dstTilingType, typename Family::XY_COPY_BLT &blitCmd) { + using XY_COPY_BLT = typename Family::XY_COPY_BLT; + if (srcTilingType == GMM_TILED_Y) { + blitCmd.setSourceTiling(XY_COPY_BLT::SOURCE_TILING::SOURCE_TILING_YMAJOR); + } + if (dstTilingType == GMM_TILED_Y) { + blitCmd.setDestinationTiling(XY_COPY_BLT::DESTINATION_TILING::DESTINATION_TILING_YMAJOR); + } +} + +template <> +void BlitCommandsHelper::appendSurfaceType(const BlitProperties &blitProperties, typename Family::XY_COPY_BLT &blitCmd) { +} + +template <> +void BlitCommandsHelper::appendSliceOffsets(const BlitProperties &blitProperties, typename Family::XY_COPY_BLT &blitCmd, uint32_t sliceIndex) { + using XY_COPY_BLT = typename Family::XY_COPY_BLT; + auto dstAllocation = blitProperties.dstAllocation; + auto srcAllocation = blitProperties.srcAllocation; + + size_t srcOffset = blitProperties.srcSlicePitch * (sliceIndex + static_cast(blitProperties.srcOffset.z)); + size_t dstOffset = blitProperties.dstSlicePitch * (sliceIndex + static_cast(blitProperties.dstOffset.z)); + + blitCmd.setSourceBaseAddress(ptrOffset(srcAllocation->getGpuAddress(), srcOffset)); + blitCmd.setDestinationBaseAddress(ptrOffset(dstAllocation->getGpuAddress(), dstOffset)); +} + +template <> +void BlitCommandsHelper::appendBlitCommandsForImages(BlitProperties &blitProperties, typename Family::XY_COPY_BLT &blitCmd) { + uint32_t dstPitch = static_cast(blitProperties.dstRowPitch); + uint32_t srcPitch = static_cast(blitProperties.srcRowPitch); + uint32_t dstQPitch = static_cast(blitProperties.dstSize.y); + uint32_t srcQPitch = static_cast(blitProperties.srcSize.y); + auto dstAllocation = blitProperties.dstAllocation; + auto srcAllocation = blitProperties.srcAllocation; + GMM_TILE_TYPE dstTileType = GMM_NOT_TILED; + GMM_TILE_TYPE srcTileType = GMM_NOT_TILED; + + constexpr uint32_t TILED_Y_PITCH_ALIGNMENT = 128; + constexpr uint32_t NON_TILED_PITCH_ALIGNMENT = 16; + + if (dstAllocation->getDefaultGmm()) { + auto dstResInfo = dstAllocation->getDefaultGmm()->gmmResourceInfo->getResourceFlags()->Info; + dstPitch = static_cast(dstAllocation->getDefaultGmm()->gmmResourceInfo->getRenderPitch()); + dstQPitch = static_cast(dstAllocation->getDefaultGmm()->gmmResourceInfo->getQPitch()); + if (dstResInfo.TiledY) { + dstTileType = GMM_TILED_Y; + dstPitch = alignUp(dstPitch, TILED_Y_PITCH_ALIGNMENT); + } else { + dstPitch = alignUp(dstPitch, NON_TILED_PITCH_ALIGNMENT); + } + } + if (srcAllocation->getDefaultGmm()) { + auto srcResInfo = srcAllocation->getDefaultGmm()->gmmResourceInfo->getResourceFlags()->Info; + srcPitch = static_cast(srcAllocation->getDefaultGmm()->gmmResourceInfo->getRenderPitch()); + srcQPitch = static_cast(srcAllocation->getDefaultGmm()->gmmResourceInfo->getQPitch()); + if (srcResInfo.TiledY) { + srcTileType = GMM_TILED_Y; + srcPitch = alignUp(srcPitch, TILED_Y_PITCH_ALIGNMENT); + } else { + srcPitch = alignUp(srcPitch, NON_TILED_PITCH_ALIGNMENT); + } + } + blitProperties.srcSlicePitch = srcPitch * srcQPitch; + blitProperties.dstSlicePitch = dstPitch * dstQPitch; + if (dstTileType != GMM_NOT_TILED) { + blitCmd.setDestinationPitch(dstPitch / 4); + } else { + blitCmd.setDestinationPitch(dstPitch); + } + if (srcTileType != GMM_NOT_TILED) { + blitCmd.setSourcePitch(srcPitch / 4); + } else { + blitCmd.setSourcePitch(srcPitch); + } + + appendTilingType(srcTileType, dstTileType, blitCmd); +} + template class CommandStreamReceiverHw; template struct BlitCommandsHelper; diff --git a/shared/source/helpers/blit_commands_helper.h b/shared/source/helpers/blit_commands_helper.h index a0994890e2..5f67956b2d 100644 --- a/shared/source/helpers/blit_commands_helper.h +++ b/shared/source/helpers/blit_commands_helper.h @@ -7,6 +7,7 @@ #pragma once #include "shared/source/command_stream/csr_deps.h" +#include "shared/source/gmm_helper/gmm.h" #include "shared/source/helpers/aux_translation.h" #include "shared/source/helpers/constants.h" #include "shared/source/helpers/vec.h" @@ -70,6 +71,9 @@ struct BlitProperties { size_t dstSlicePitch = 0; size_t srcRowPitch = 0; size_t srcSlicePitch = 0; + size_t bytesPerPixel = 0; + Vec3 dstSize = 0; + Vec3 srcSize = 0; }; template @@ -84,11 +88,17 @@ struct BlitCommandsHelper { static uint64_t calculateBlitCommandDestinationBaseAddress(const BlitProperties &blitProperties, uint64_t offset, uint64_t row, uint64_t slice); static uint64_t calculateBlitCommandSourceBaseAddress(const BlitProperties &blitProperties, uint64_t offset, uint64_t row, uint64_t slice); static void dispatchBlitCommandsForBuffer(const BlitProperties &blitProperties, LinearStream &linearStream, const RootDeviceEnvironment &rootDeviceEnvironment); + static void dispatchBlitCommandsForImages(BlitProperties &blitProperties, LinearStream &linearStream, const RootDeviceEnvironment &rootDeviceEnvironment); static void dispatchBlitMemoryColorFill(NEO::GraphicsAllocation *dstAlloc, uint32_t *pattern, size_t patternSize, LinearStream &linearStream, size_t size, const RootDeviceEnvironment &rootDeviceEnvironment); template static void dispatchBlitMemoryFill(NEO::GraphicsAllocation *dstAlloc, uint32_t *pattern, LinearStream &linearStream, size_t size, const RootDeviceEnvironment &rootDeviceEnvironment, COLOR_DEPTH depth); static void appendBlitCommandsForBuffer(const BlitProperties &blitProperties, typename GfxFamily::XY_COPY_BLT &blitCmd, const RootDeviceEnvironment &rootDeviceEnvironment); + static void appendBlitCommandsForImages(BlitProperties &blitProperties, typename GfxFamily::XY_COPY_BLT &blitCmd); + static void appendColorDepth(const BlitProperties &blitProperties, typename GfxFamily::XY_COPY_BLT &blitCmd); static void appendBlitCommandsForFillBuffer(NEO::GraphicsAllocation *dstAlloc, typename GfxFamily::XY_COLOR_BLT &blitCmd, const RootDeviceEnvironment &rootDeviceEnvironment); + static void appendSurfaceType(const BlitProperties &blitProperties, typename GfxFamily::XY_COPY_BLT &blitCmd); static void appendTilingEnable(typename GfxFamily::XY_COLOR_BLT &blitCmd); + static void appendTilingType(const GMM_TILE_TYPE srcTilingType, const GMM_TILE_TYPE dstTilingType, typename GfxFamily::XY_COPY_BLT &blitCmd); + static void appendSliceOffsets(const BlitProperties &blitProperties, typename GfxFamily::XY_COPY_BLT &blitCmd, uint32_t sliceIndex); }; } // namespace NEO diff --git a/shared/source/helpers/blit_commands_helper_base.inl b/shared/source/helpers/blit_commands_helper_base.inl index 49bf9927b0..45456acb45 100644 --- a/shared/source/helpers/blit_commands_helper_base.inl +++ b/shared/source/helpers/blit_commands_helper_base.inl @@ -6,6 +6,9 @@ */ #include "shared/source/command_container/command_encoder.h" +#include "shared/source/gmm_helper/gmm.h" +#include "shared/source/gmm_helper/gmm_helper.h" +#include "shared/source/gmm_helper/resource_info.h" #include "shared/source/helpers/blit_commands_helper.h" #include "shared/source/helpers/hw_helper.h" #include "shared/source/helpers/timestamp_packet.h" @@ -178,8 +181,6 @@ void BlitCommandsHelper::dispatchBlitMemoryFill(NEO::GraphicsAllocati blitCmd.setFillColor(pattern); blitCmd.setColorDepth(depth); - appendBlitCommandsForFillBuffer(dstAlloc, blitCmd, rootDeviceEnvironment); - uint64_t offset = 0; uint64_t sizeToFill = size; while (sizeToFill != 0) { @@ -200,6 +201,9 @@ void BlitCommandsHelper::dispatchBlitMemoryFill(NEO::GraphicsAllocati tmpCmd.setTransferWidth(static_cast(width)); tmpCmd.setTransferHeight(static_cast(height)); tmpCmd.setDestinationPitch(static_cast(width)); + + appendBlitCommandsForFillBuffer(dstAlloc, tmpCmd, rootDeviceEnvironment); + auto cmd = linearStream.getSpaceForCmd(); *cmd = tmpCmd; auto blitSize = width * height; @@ -208,4 +212,34 @@ void BlitCommandsHelper::dispatchBlitMemoryFill(NEO::GraphicsAllocati } } +template +void BlitCommandsHelper::dispatchBlitCommandsForImages(BlitProperties &blitProperties, LinearStream &linearStream, const RootDeviceEnvironment &rootDeviceEnvironment) { + auto dstAllocation = blitProperties.dstAllocation; + auto srcAllocation = blitProperties.srcAllocation; + + UNRECOVERABLE_IF(blitProperties.copySize.x > BlitterConstants::maxBlitWidth || blitProperties.copySize.y > BlitterConstants::maxBlitWidth); + auto bltCmd = GfxFamily::cmdInitXyCopyBlt; + + bltCmd.setSourceBaseAddress(srcAllocation->getGpuAddress()); + bltCmd.setDestinationBaseAddress(dstAllocation->getGpuAddress()); + + bltCmd.setDestinationX1CoordinateLeft(static_cast(blitProperties.dstOffset.x)); + bltCmd.setDestinationY1CoordinateTop(static_cast(blitProperties.dstOffset.y)); + bltCmd.setTransferWidth(static_cast(blitProperties.dstOffset.x + blitProperties.copySize.x)); + bltCmd.setTransferHeight(static_cast(blitProperties.dstOffset.y + blitProperties.copySize.y)); + + bltCmd.setSourceX1CoordinateLeft(static_cast(blitProperties.srcOffset.x)); + bltCmd.setSourceY1CoordinateTop(static_cast(blitProperties.srcOffset.y)); + + appendBlitCommandsForBuffer(blitProperties, bltCmd, rootDeviceEnvironment); + appendBlitCommandsForImages(blitProperties, bltCmd); + appendColorDepth(blitProperties, bltCmd); + appendSurfaceType(blitProperties, bltCmd); + for (uint32_t i = 0; i < blitProperties.copySize.z; i++) { + appendSliceOffsets(blitProperties, bltCmd, i); + auto cmd = linearStream.getSpaceForCmd(); + *cmd = bltCmd; + } +} + } // namespace NEO diff --git a/shared/source/helpers/blit_commands_helper_bdw_plus.inl b/shared/source/helpers/blit_commands_helper_bdw_plus.inl index 7b916afddf..756a79a06f 100644 --- a/shared/source/helpers/blit_commands_helper_bdw_plus.inl +++ b/shared/source/helpers/blit_commands_helper_bdw_plus.inl @@ -12,6 +12,11 @@ namespace NEO { template void BlitCommandsHelper::appendBlitCommandsForBuffer(const BlitProperties &blitProperties, typename GfxFamily::XY_COPY_BLT &blitCmd, const RootDeviceEnvironment &rootDeviceEnvironment) {} +template +void BlitCommandsHelper::appendBlitCommandsForImages(BlitProperties &blitProperties, typename GfxFamily::XY_COPY_BLT &blitCmd) { + appendTilingType(GMM_NOT_TILED, GMM_NOT_TILED, blitCmd); +} + template void BlitCommandsHelper::appendBlitCommandsForFillBuffer(NEO::GraphicsAllocation *dstAlloc, typename GfxFamily::XY_COLOR_BLT &blitCmd, const RootDeviceEnvironment &rootDeviceEnvironment) { using XY_COLOR_BLT = typename GfxFamily::XY_COLOR_BLT; @@ -34,10 +39,24 @@ void BlitCommandsHelper::dispatchBlitMemoryColorFill(NEO::GraphicsAll } } +template +void BlitCommandsHelper::appendSurfaceType(const BlitProperties &blitProperties, typename GfxFamily::XY_COPY_BLT &blitCmd) { +} template void BlitCommandsHelper::appendTilingEnable(typename GfxFamily::XY_COLOR_BLT &blitCmd) { using XY_COLOR_BLT = typename GfxFamily::XY_COLOR_BLT; blitCmd.setDestTilingEnable(XY_COLOR_BLT::DEST_TILING_ENABLE::DEST_TILING_ENABLE_TILING_ENABLED); } +template +void BlitCommandsHelper::appendTilingType(const GMM_TILE_TYPE srcTilingType, const GMM_TILE_TYPE dstTilingType, typename GfxFamily::XY_COPY_BLT &blitCmd) { +} +template +void BlitCommandsHelper::appendColorDepth(const BlitProperties &blitProperites, typename GfxFamily::XY_COPY_BLT &blitCmd) { +} + +template +void BlitCommandsHelper::appendSliceOffsets(const BlitProperties &blitProperties, typename GfxFamily::XY_COPY_BLT &blitCmd, uint32_t sliceIndex) { +} + } // namespace NEO diff --git a/shared/test/unit_test/helpers/CMakeLists.txt b/shared/test/unit_test/helpers/CMakeLists.txt index c80358d56a..c3ac8c42e6 100644 --- a/shared/test/unit_test/helpers/CMakeLists.txt +++ b/shared/test/unit_test/helpers/CMakeLists.txt @@ -7,6 +7,7 @@ set(NEO_CORE_HELPERS_TESTS ${CMAKE_CURRENT_SOURCE_DIR}/blit_commands_helper_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/blit_commands_helper_tests.inl + ${CMAKE_CURRENT_SOURCE_DIR}/blit_commands_helper_tests_gen12lp.cpp ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt ${CMAKE_CURRENT_SOURCE_DIR}/debug_manager_state_restore.h ${CMAKE_CURRENT_SOURCE_DIR}/default_hw_info.h diff --git a/shared/test/unit_test/helpers/blit_commands_helper_tests.cpp b/shared/test/unit_test/helpers/blit_commands_helper_tests.cpp index 09d302a992..6da7b0d871 100644 --- a/shared/test/unit_test/helpers/blit_commands_helper_tests.cpp +++ b/shared/test/unit_test/helpers/blit_commands_helper_tests.cpp @@ -241,3 +241,39 @@ HWTEST2_F(BlitColorTests, givenCommandStreamAndPaternSizeEqualFourWhenCallToDisp GivenLinearStreamWhenCallDispatchBlitMemoryColorFillThenCorrectDepthIsProgrammed test(pDevice); test.TestBodyImpl(patternSize, expecttedDepth); } + +using ImageSupport = IsWithinProducts; + +HWTEST2_F(BlitTests, givenMemoryAndImageWhenDispatchCopyImageCallThenCommandAddedToStream, BlitPlatforms) { + using XY_COPY_BLT = typename FamilyType::XY_COPY_BLT; + MockGraphicsAllocation srcAlloc; + MockGraphicsAllocation dstAlloc; + + Vec3 dstOffsets = {0, 0, 0}; + Vec3 srcOffsets = {0, 0, 0}; + + Vec3 copySize = {0x100, 0x40, 0x1}; + Vec3 srcSize = {0x100, 0x40, 0x1}; + Vec3 dstSize = {0x100, 0x40, 0x1}; + + uint32_t srcRowPitch = srcSize.x; + uint32_t srcSlicePitch = srcSize.y; + uint32_t dstRowPitch = dstSize.x; + uint32_t dstSlicePitch = dstSize.y; + + auto blitProperties = NEO::BlitProperties::constructPropertiesForCopyBuffer(&dstAlloc, &srcAlloc, + dstOffsets, srcOffsets, copySize, srcRowPitch, srcSlicePitch, + dstRowPitch, dstSlicePitch); + + uint32_t streamBuffer[100] = {}; + LinearStream stream(streamBuffer, sizeof(streamBuffer)); + blitProperties.bytesPerPixel = 4; + blitProperties.srcSize = srcSize; + blitProperties.dstSize = dstSize; + NEO::BlitCommandsHelper::dispatchBlitCommandsForImages(blitProperties, stream, *pDevice->getExecutionEnvironment()->rootDeviceEnvironments[pDevice->getRootDeviceIndex()]); + GenCmdList cmdList; + ASSERT_TRUE(FamilyType::PARSE::parseCommandBuffer( + cmdList, ptrOffset(stream.getCpuBase(), 0), stream.getUsed())); + auto itor = find(cmdList.begin(), cmdList.end()); + EXPECT_NE(cmdList.end(), itor); +} \ No newline at end of file diff --git a/shared/test/unit_test/helpers/blit_commands_helper_tests_gen12lp.cpp b/shared/test/unit_test/helpers/blit_commands_helper_tests_gen12lp.cpp new file mode 100644 index 0000000000..194ae61ee3 --- /dev/null +++ b/shared/test/unit_test/helpers/blit_commands_helper_tests_gen12lp.cpp @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/test/unit_test/helpers/blit_commands_helper_tests.inl" +#include "shared/test/unit_test/helpers/debug_manager_state_restore.h" + +#include "opencl/test/unit_test/mocks/mock_gmm.h" +#include "opencl/test/unit_test/mocks/mock_graphics_allocation.h" + +#include "gtest/gtest.h" + +using namespace NEO; + +using BlitTests = Test; + +HWTEST2_F(BlitTests, givenOneBytePerPixelWhenAppendColorDepthThenCorrectDepthIsSet, IsGen12LP) { + using XY_COPY_BLT = typename FamilyType::XY_COPY_BLT; + auto bltCmd = FamilyType::cmdInitXyCopyBlt; + BlitProperties properties = {}; + properties.bytesPerPixel = 1; + BlitCommandsHelper::appendColorDepth(properties, bltCmd); + EXPECT_EQ(bltCmd.getColorDepth(), XY_COPY_BLT::COLOR_DEPTH::COLOR_DEPTH_8_BIT_COLOR); +} + +HWTEST2_F(BlitTests, givenTwoBytePerPixelWhenAppendColorDepthThenCorrectDepthIsSet, IsGen12LP) { + using XY_COPY_BLT = typename FamilyType::XY_COPY_BLT; + auto bltCmd = FamilyType::cmdInitXyCopyBlt; + BlitProperties properties = {}; + properties.bytesPerPixel = 2; + BlitCommandsHelper::appendColorDepth(properties, bltCmd); + EXPECT_EQ(bltCmd.getColorDepth(), XY_COPY_BLT::COLOR_DEPTH::COLOR_DEPTH_16_BIT_COLOR); +} + +HWTEST2_F(BlitTests, givenFourBytePerPixelWhenAppendColorDepthThenCorrectDepthIsSet, IsGen12LP) { + using XY_COPY_BLT = typename FamilyType::XY_COPY_BLT; + auto bltCmd = FamilyType::cmdInitXyCopyBlt; + BlitProperties properties = {}; + properties.bytesPerPixel = 4; + BlitCommandsHelper::appendColorDepth(properties, bltCmd); + EXPECT_EQ(bltCmd.getColorDepth(), XY_COPY_BLT::COLOR_DEPTH::COLOR_DEPTH_32_BIT_COLOR); +} + +HWTEST2_F(BlitTests, givenEightBytePerPixelWhenAppendColorDepthThenCorrectDepthIsSet, IsGen12LP) { + using XY_COPY_BLT = typename FamilyType::XY_COPY_BLT; + auto bltCmd = FamilyType::cmdInitXyCopyBlt; + BlitProperties properties = {}; + properties.bytesPerPixel = 8; + BlitCommandsHelper::appendColorDepth(properties, bltCmd); + EXPECT_EQ(bltCmd.getColorDepth(), XY_COPY_BLT::COLOR_DEPTH::COLOR_DEPTH_64_BIT_COLOR); +} + +HWTEST2_F(BlitTests, givenSixteenBytePerPixelWhenAppendColorDepthThenCorrectDepthIsSet, IsGen12LP) { + using XY_COPY_BLT = typename FamilyType::XY_COPY_BLT; + auto bltCmd = FamilyType::cmdInitXyCopyBlt; + BlitProperties properties = {}; + properties.bytesPerPixel = 16; + BlitCommandsHelper::appendColorDepth(properties, bltCmd); + EXPECT_EQ(bltCmd.getColorDepth(), XY_COPY_BLT::COLOR_DEPTH::COLOR_DEPTH_128_BIT_COLOR); +} + +HWTEST2_F(BlitTests, givenIncorrectBytePerPixelWhenAppendColorDepthThenAbortIsThrown, IsGen12LP) { + using XY_COPY_BLT = typename FamilyType::XY_COPY_BLT; + auto bltCmd = FamilyType::cmdInitXyCopyBlt; + BlitProperties properties = {}; + properties.bytesPerPixel = 48; + EXPECT_THROW(BlitCommandsHelper::appendColorDepth(properties, bltCmd), std::exception); +} + +HWTEST2_F(BlitTests, givenNotTiledSrcAndDestinationWhenAppendTilingTypeThenCorrectTilingIsSet, IsGen12LP) { + using XY_COPY_BLT = typename FamilyType::XY_COPY_BLT; + auto bltCmd = FamilyType::cmdInitXyCopyBlt; + BlitProperties properties = {}; + BlitCommandsHelper::appendTilingType(GMM_NOT_TILED, GMM_NOT_TILED, bltCmd); + EXPECT_EQ(bltCmd.getSourceTiling(), XY_COPY_BLT::SOURCE_TILING::SOURCE_TILING_LINEAR); + EXPECT_EQ(bltCmd.getDestinationTiling(), XY_COPY_BLT::DESTINATION_TILING::DESTINATION_TILING_LINEAR); +} +HWTEST2_F(BlitTests, givenTiledSrcAndDestinationAppendTilingTypeThenCorrectTilingIsSet, IsGen12LP) { + using XY_COPY_BLT = typename FamilyType::XY_COPY_BLT; + auto bltCmd = FamilyType::cmdInitXyCopyBlt; + BlitProperties properties = {}; + BlitCommandsHelper::appendTilingType(GMM_TILED_Y, GMM_TILED_Y, bltCmd); + EXPECT_EQ(bltCmd.getSourceTiling(), XY_COPY_BLT::SOURCE_TILING::SOURCE_TILING_YMAJOR); + EXPECT_EQ(bltCmd.getDestinationTiling(), XY_COPY_BLT::DESTINATION_TILING::DESTINATION_TILING_YMAJOR); +} + +HWTEST2_F(BlitTests, givenTiledSrcAndDestinationWhenAppendImageCommandsThenCorrectTiledIsSet, IsGen12LP) { + using XY_COPY_BLT = typename FamilyType::XY_COPY_BLT; + auto gmm = std::make_unique(); + auto flags = gmm->gmmResourceInfo->getResourceFlags(); + flags->Info.TiledY = true; + MockGraphicsAllocation mockAllocationSrc(0, GraphicsAllocation::AllocationType::INTERNAL_HOST_MEMORY, + reinterpret_cast(0x1234), 0x1000, 0, sizeof(uint32_t), + MemoryPool::System4KBPages); + MockGraphicsAllocation mockAllocationDst(0, GraphicsAllocation::AllocationType::INTERNAL_HOST_MEMORY, + reinterpret_cast(0x1234), 0x1000, 0, sizeof(uint32_t), + MemoryPool::System4KBPages); + mockAllocationSrc.setGmm(gmm.get(), 0); + mockAllocationDst.setGmm(gmm.get(), 0); + auto bltCmd = FamilyType::cmdInitXyCopyBlt; + BlitProperties properties = {}; + properties.srcAllocation = &mockAllocationSrc; + properties.dstAllocation = &mockAllocationDst; + BlitCommandsHelper::appendBlitCommandsForImages(properties, bltCmd); + EXPECT_EQ(bltCmd.getSourceTiling(), XY_COPY_BLT::SOURCE_TILING::SOURCE_TILING_YMAJOR); + EXPECT_EQ(bltCmd.getDestinationTiling(), XY_COPY_BLT::DESTINATION_TILING::DESTINATION_TILING_YMAJOR); +} + +HWTEST2_F(BlitTests, givenNotTiledSrcAndDestinationWhenAppendImageCommandsThenCorrectTiledIsSet, IsGen12LP) { + using XY_COPY_BLT = typename FamilyType::XY_COPY_BLT; + auto gmm = std::make_unique(); + auto flags = gmm->gmmResourceInfo->getResourceFlags(); + flags->Info.TiledY = false; + MockGraphicsAllocation mockAllocationSrc(0, GraphicsAllocation::AllocationType::INTERNAL_HOST_MEMORY, + reinterpret_cast(0x1234), 0x1000, 0, sizeof(uint32_t), + MemoryPool::System4KBPages); + MockGraphicsAllocation mockAllocationDst(0, GraphicsAllocation::AllocationType::INTERNAL_HOST_MEMORY, + reinterpret_cast(0x1234), 0x1000, 0, sizeof(uint32_t), + MemoryPool::System4KBPages); + mockAllocationSrc.setGmm(gmm.get(), 0); + mockAllocationDst.setGmm(gmm.get(), 0); + auto bltCmd = FamilyType::cmdInitXyCopyBlt; + BlitProperties properties = {}; + properties.srcAllocation = &mockAllocationSrc; + properties.dstAllocation = &mockAllocationDst; + BlitCommandsHelper::appendBlitCommandsForImages(properties, bltCmd); + EXPECT_EQ(bltCmd.getSourceTiling(), XY_COPY_BLT::SOURCE_TILING::SOURCE_TILING_LINEAR); + EXPECT_EQ(bltCmd.getDestinationTiling(), XY_COPY_BLT::DESTINATION_TILING::DESTINATION_TILING_LINEAR); +} + +HWTEST2_F(BlitTests, givenSrcAndDestinationImagesWhenAppendSliceOffsetsThenAdressAreCorectOffseted, IsGen12LP) { + using XY_COPY_BLT = typename FamilyType::XY_COPY_BLT; + auto gmm = std::make_unique(); + auto flags = gmm->gmmResourceInfo->getResourceFlags(); + flags->Info.TiledY = false; + MockGraphicsAllocation mockAllocationSrc(0, GraphicsAllocation::AllocationType::INTERNAL_HOST_MEMORY, + reinterpret_cast(0x1234), 0x1000, 0, sizeof(uint32_t), + MemoryPool::System4KBPages); + MockGraphicsAllocation mockAllocationDst(0, GraphicsAllocation::AllocationType::INTERNAL_HOST_MEMORY, + reinterpret_cast(0x1234), 0x1000, 0, sizeof(uint32_t), + MemoryPool::System4KBPages); + mockAllocationSrc.setGmm(gmm.get(), 0); + mockAllocationDst.setGmm(gmm.get(), 0); + auto bltCmd = FamilyType::cmdInitXyCopyBlt; + BlitProperties properties = {}; + properties.srcAllocation = &mockAllocationSrc; + properties.dstAllocation = &mockAllocationDst; + properties.dstSlicePitch = 0x1000; + properties.srcSlicePitch = 0x2000; + properties.srcOffset = {0x10, 0x10, 0x10}; + properties.dstOffset = {0x20, 0x20, 0x20}; + uint32_t index = 7; + BlitCommandsHelper::appendSliceOffsets(properties, bltCmd, index); + auto expectesSrcOffset = (index + properties.srcOffset.z) * properties.srcSlicePitch; + auto expectesDstOffset = (index + properties.dstOffset.z) * properties.dstSlicePitch; + EXPECT_EQ(bltCmd.getSourceBaseAddress(), ptrOffset(mockAllocationSrc.getGpuAddress(), expectesSrcOffset)); + EXPECT_EQ(bltCmd.getDestinationBaseAddress(), ptrOffset(mockAllocationDst.getGpuAddress(), expectesDstOffset)); +} \ No newline at end of file