diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.inl b/level_zero/core/source/cmdlist/cmdlist_hw.inl index e34ab73718..d0895c8b87 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.inl +++ b/level_zero/core/source/cmdlist/cmdlist_hw.inl @@ -17,6 +17,7 @@ #include "shared/source/gmm_helper/gmm_helper.h" #include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/helpers/definitions/mi_flush_args.h" #include "shared/source/helpers/heap_helper.h" #include "shared/source/helpers/hw_helper.h" diff --git a/level_zero/core/source/printf_handler/printf_handler.cpp b/level_zero/core/source/printf_handler/printf_handler.cpp index 850759f1bd..f713c9a366 100644 --- a/level_zero/core/source/printf_handler/printf_handler.cpp +++ b/level_zero/core/source/printf_handler/printf_handler.cpp @@ -9,7 +9,7 @@ #include "shared/source/command_stream/command_stream_receiver.h" #include "shared/source/execution_environment/execution_environment.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/helpers/engine_node_helper.h" #include "shared/source/memory_manager/allocation_properties.h" #include "shared/source/memory_manager/memory_manager.h" diff --git a/level_zero/core/test/unit_tests/sources/context/test_context.cpp b/level_zero/core/test/unit_tests/sources/context/test_context.cpp index 7ae9a33807..9baa2f53d5 100644 --- a/level_zero/core/test/unit_tests/sources/context/test_context.cpp +++ b/level_zero/core/test/unit_tests/sources/context/test_context.cpp @@ -7,7 +7,7 @@ #include "shared/source/built_ins/sip.h" #include "shared/source/gmm_helper/gmm.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/test/common/mocks/mock_command_stream_receiver.h" #include "shared/test/common/mocks/mock_compilers.h" #include "shared/test/common/mocks/mock_cpu_page_fault_manager.h" diff --git a/level_zero/core/test/unit_tests/sources/module/test_module.cpp b/level_zero/core/test/unit_tests/sources/module/test_module.cpp index 204114e3ec..5f090d3419 100644 --- a/level_zero/core/test/unit_tests/sources/module/test_module.cpp +++ b/level_zero/core/test/unit_tests/sources/module/test_module.cpp @@ -16,6 +16,7 @@ #include "shared/source/gmm_helper/gmm_helper.h" #include "shared/source/helpers/addressing_mode_helper.h" #include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_helper.h" #include "shared/source/helpers/compiler_hw_info_config.h" #include "shared/source/kernel/implicit_args.h" #include "shared/source/os_interface/os_inc_base.h" diff --git a/opencl/source/helpers/cl_blit_properties.h b/opencl/source/helpers/cl_blit_properties.h index 268d8f06f2..7867e97ef8 100644 --- a/opencl/source/helpers/cl_blit_properties.h +++ b/opencl/source/helpers/cl_blit_properties.h @@ -7,7 +7,7 @@ #pragma once #include "shared/source/command_stream/command_stream_receiver.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "opencl/source/built_ins/builtins_dispatch_builder.h" #include "opencl/source/mem_obj/image.h" diff --git a/opencl/source/helpers/enqueue_properties.h b/opencl/source/helpers/enqueue_properties.h index 3b11f8ffd3..980b702d8c 100644 --- a/opencl/source/helpers/enqueue_properties.h +++ b/opencl/source/helpers/enqueue_properties.h @@ -1,12 +1,12 @@ /* - * Copyright (C) 2019-2021 Intel Corporation + * Copyright (C) 2019-2023 Intel Corporation * * SPDX-License-Identifier: MIT * */ #pragma once -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties_container.h" namespace NEO { diff --git a/opencl/source/helpers/task_information.cpp b/opencl/source/helpers/task_information.cpp index 92609fdbc6..e91ff96ec8 100644 --- a/opencl/source/helpers/task_information.cpp +++ b/opencl/source/helpers/task_information.cpp @@ -12,6 +12,7 @@ #include "shared/source/command_stream/linear_stream.h" #include "shared/source/command_stream/preemption.h" #include "shared/source/command_stream/wait_status.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/helpers/flush_stamp.h" #include "shared/source/memory_manager/internal_allocation_storage.h" #include "shared/source/memory_manager/surface.h" diff --git a/opencl/source/helpers/task_information.h b/opencl/source/helpers/task_information.h index 0eb3934b56..68f378c693 100644 --- a/opencl/source/helpers/task_information.h +++ b/opencl/source/helpers/task_information.h @@ -7,7 +7,7 @@ #pragma once #include "shared/source/command_stream/linear_stream.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/helpers/completion_stamp.h" #include "shared/source/helpers/map_operation_type.h" #include "shared/source/helpers/timestamp_packet.h" diff --git a/opencl/source/mem_obj/buffer.cpp b/opencl/source/mem_obj/buffer.cpp index 5150244ab2..a8cacd76ad 100644 --- a/opencl/source/mem_obj/buffer.cpp +++ b/opencl/source/mem_obj/buffer.cpp @@ -13,6 +13,7 @@ #include "shared/source/execution_environment/root_device_environment.h" #include "shared/source/gmm_helper/gmm_helper.h" #include "shared/source/helpers/aligned_memory.h" +#include "shared/source/helpers/blit_helper.h" #include "shared/source/helpers/local_memory_access_modes.h" #include "shared/source/helpers/memory_properties_helpers.h" #include "shared/source/memory_manager/host_ptr_manager.h" diff --git a/opencl/source/program/printf_handler.cpp b/opencl/source/program/printf_handler.cpp index 360ec19fa5..dfc09fdfc2 100644 --- a/opencl/source/program/printf_handler.cpp +++ b/opencl/source/program/printf_handler.cpp @@ -11,7 +11,7 @@ #include "shared/source/compiler_interface/compiler_cache.h" #include "shared/source/device/device.h" #include "shared/source/helpers/aligned_memory.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/helpers/hw_helper.h" #include "shared/source/helpers/ptr_math.h" #include "shared/source/kernel/implicit_args.h" diff --git a/opencl/test/unit_test/aub_tests/command_stream/copy_engine_aub_tests_xehp_and_later.h b/opencl/test/unit_test/aub_tests/command_stream/copy_engine_aub_tests_xehp_and_later.h index 6dc878df94..9d07e1c085 100644 --- a/opencl/test/unit_test/aub_tests/command_stream/copy_engine_aub_tests_xehp_and_later.h +++ b/opencl/test/unit_test/aub_tests/command_stream/copy_engine_aub_tests_xehp_and_later.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 Intel Corporation + * Copyright (C) 2022-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -10,6 +10,7 @@ #include "shared/source/command_stream/aub_command_stream_receiver_hw.h" #include "shared/source/gmm_helper/gmm.h" #include "shared/source/gmm_helper/gmm_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/helpers/timestamp_packet.h" #include "shared/source/memory_manager/surface.h" #include "shared/source/os_interface/hw_info_config.h" diff --git a/opencl/test/unit_test/command_stream/command_stream_receiver_hw_1_tests.cpp b/opencl/test/unit_test/command_stream/command_stream_receiver_hw_1_tests.cpp index 12caf89da5..9724d7ec7a 100644 --- a/opencl/test/unit_test/command_stream/command_stream_receiver_hw_1_tests.cpp +++ b/opencl/test/unit_test/command_stream/command_stream_receiver_hw_1_tests.cpp @@ -9,6 +9,7 @@ #include "shared/source/command_stream/scratch_space_controller.h" #include "shared/source/command_stream/scratch_space_controller_base.h" #include "shared/source/command_stream/wait_status.h" +#include "shared/source/helpers/blit_commands_helper.h" #include "shared/source/helpers/constants.h" #include "shared/source/helpers/logical_state_helper.h" #include "shared/source/os_interface/hw_info_config.h" @@ -748,6 +749,8 @@ HWTEST_F(BcsTests, givenBlitPropertiesContainerWhenEstimatingCommandsSizeThenCal BlitPropertiesContainer blitPropertiesContainer; for (uint32_t i = 0; i < numberOfBlitOperations; i++) { BlitProperties blitProperties; + blitProperties.blitDirection = BlitterConstants::BlitDirection::BufferToHostPtr; + blitProperties.auxTranslationDirection = AuxTranslationDirection::None; blitProperties.copySize = {bltSize, 1, 1}; blitPropertiesContainer.push_back(blitProperties); @@ -786,6 +789,8 @@ HWTEST_F(BcsTests, givenBlitPropertiesContainerWhenDirectsubmissionEnabledEstima BlitPropertiesContainer blitPropertiesContainer; for (uint32_t i = 0; i < numberOfBlitOperations; i++) { BlitProperties blitProperties; + blitProperties.blitDirection = BlitterConstants::BlitDirection::BufferToHostPtr; + blitProperties.auxTranslationDirection = AuxTranslationDirection::None; blitProperties.copySize = {bltSize, 1, 1}; blitPropertiesContainer.push_back(blitProperties); @@ -823,6 +828,8 @@ HWTEST_F(BcsTests, givenBlitPropertiesContainerWhenEstimatingCommandsSizeForWrit BlitPropertiesContainer blitPropertiesContainer; for (uint32_t i = 0; i < numberOfBlitOperations; i++) { BlitProperties blitProperties; + blitProperties.blitDirection = BlitterConstants::BlitDirection::BufferToHostPtr; + blitProperties.auxTranslationDirection = AuxTranslationDirection::None; blitProperties.copySize = bltSize; blitPropertiesContainer.push_back(blitProperties); @@ -860,6 +867,8 @@ HWTEST_F(BcsTests, givenBlitPropertiesContainerWhenDirectSubmissionEnabledEstima BlitPropertiesContainer blitPropertiesContainer; for (uint32_t i = 0; i < numberOfBlitOperations; i++) { BlitProperties blitProperties; + blitProperties.blitDirection = BlitterConstants::BlitDirection::BufferToHostPtr; + blitProperties.auxTranslationDirection = AuxTranslationDirection::None; blitProperties.copySize = bltSize; blitPropertiesContainer.push_back(blitProperties); diff --git a/opencl/test/unit_test/command_stream/command_stream_receiver_hw_2_tests.cpp b/opencl/test/unit_test/command_stream/command_stream_receiver_hw_2_tests.cpp index 3381431fdb..b0bb0304ce 100644 --- a/opencl/test/unit_test/command_stream/command_stream_receiver_hw_2_tests.cpp +++ b/opencl/test/unit_test/command_stream/command_stream_receiver_hw_2_tests.cpp @@ -8,6 +8,7 @@ #include "shared/source/command_stream/submission_status.h" #include "shared/source/command_stream/wait_status.h" #include "shared/source/direct_submission/dispatchers/blitter_dispatcher.h" +#include "shared/source/helpers/blit_commands_helper.h" #include "shared/source/helpers/flush_stamp.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" #include "shared/test/common/helpers/engine_descriptor_helper.h" diff --git a/opencl/test/unit_test/gen12lp/buffer_tests_gen12lp.inl b/opencl/test/unit_test/gen12lp/buffer_tests_gen12lp.inl index 97f5cf57bd..e52e34a4d3 100644 --- a/opencl/test/unit_test/gen12lp/buffer_tests_gen12lp.inl +++ b/opencl/test/unit_test/gen12lp/buffer_tests_gen12lp.inl @@ -7,6 +7,7 @@ #include "shared/source/device/device.h" #include "shared/source/gmm_helper/gmm_helper.h" +#include "shared/source/gmm_helper/gmm_lib.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" #include "shared/test/common/mocks/mock_memory_manager.h" #include "shared/test/common/test_macros/hw_test.h" diff --git a/opencl/test/unit_test/mocks/mock_context.cpp b/opencl/test/unit_test/mocks/mock_context.cpp index 7e91408674..433422edba 100644 --- a/opencl/test/unit_test/mocks/mock_context.cpp +++ b/opencl/test/unit_test/mocks/mock_context.cpp @@ -10,6 +10,7 @@ #include "shared/source/built_ins/built_ins.h" #include "shared/source/compiler_interface/compiler_interface.h" #include "shared/source/compiler_interface/external_functions.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/memory_manager/deferred_deleter.h" #include "shared/source/memory_manager/unified_memory_manager.h" #include "shared/source/os_interface/os_context.h" diff --git a/opencl/test/unit_test/mocks/mock_context.h b/opencl/test/unit_test/mocks/mock_context.h index c60547a342..5cd570eeb6 100644 --- a/opencl/test/unit_test/mocks/mock_context.h +++ b/opencl/test/unit_test/mocks/mock_context.h @@ -6,7 +6,7 @@ */ #pragma once -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_helper.h" #include "shared/test/common/helpers/variable_backup.h" #include "opencl/source/context/context.h" diff --git a/shared/source/command_stream/command_stream_receiver_hw_base.inl b/shared/source/command_stream/command_stream_receiver_hw_base.inl index 99034b38c1..952c47b214 100644 --- a/shared/source/command_stream/command_stream_receiver_hw_base.inl +++ b/shared/source/command_stream/command_stream_receiver_hw_base.inl @@ -25,6 +25,7 @@ #include "shared/source/execution_environment/root_device_environment.h" #include "shared/source/gmm_helper/page_table_mngr.h" #include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/helpers/cache_policy.h" #include "shared/source/helpers/definitions/mi_flush_args.h" #include "shared/source/helpers/engine_node_helper.h" diff --git a/shared/source/helpers/CMakeLists.txt b/shared/source/helpers/CMakeLists.txt index 85b5d0476d..6428724e07 100644 --- a/shared/source/helpers/CMakeLists.txt +++ b/shared/source/helpers/CMakeLists.txt @@ -25,8 +25,9 @@ set(NEO_CORE_HELPERS ${CMAKE_CURRENT_SOURCE_DIR}/bit_helpers.h ${CMAKE_CURRENT_SOURCE_DIR}/blit_commands_helper_base.inl ${CMAKE_CURRENT_SOURCE_DIR}/blit_commands_helper_bdw_and_later.inl - ${CMAKE_CURRENT_SOURCE_DIR}/blit_commands_helper.cpp ${CMAKE_CURRENT_SOURCE_DIR}/blit_commands_helper.h + ${CMAKE_CURRENT_SOURCE_DIR}/blit_helper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/blit_properties.cpp ${CMAKE_CURRENT_SOURCE_DIR}/cache_flush.inl ${CMAKE_CURRENT_SOURCE_DIR}/cache_policy.cpp ${CMAKE_CURRENT_SOURCE_DIR}/cache_policy.h diff --git a/shared/source/helpers/blit_commands_helper.h b/shared/source/helpers/blit_commands_helper.h index 39f4310ff9..b487e26224 100644 --- a/shared/source/helpers/blit_commands_helper.h +++ b/shared/source/helpers/blit_commands_helper.h @@ -6,115 +6,21 @@ */ #pragma once -#include "shared/source/command_stream/csr_deps.h" #include "shared/source/gmm_helper/gmm_lib.h" -#include "shared/source/helpers/aux_translation.h" #include "shared/source/helpers/blit_properties_container.h" -#include "shared/source/helpers/common_types.h" -#include "shared/source/helpers/constants.h" #include "shared/source/helpers/vec.h" #include -#include namespace NEO { -class CommandStreamReceiver; -class Device; + +class CsrDependencies; class GraphicsAllocation; class LinearStream; - -struct RootDeviceEnvironment; - -template -class TagNode; - -template -class TimestampPackets; - class TagNodeBase; - -struct BlitProperties; +enum class DebugPauseState : uint32_t; struct HardwareInfo; -struct TimestampPacketDependencies; - -struct BlitProperties { - static BlitProperties constructPropertiesForReadWrite(BlitterConstants::BlitDirection blitDirection, - CommandStreamReceiver &commandStreamReceiver, - GraphicsAllocation *memObjAllocation, - GraphicsAllocation *preallocatedHostAllocation, - const void *hostPtr, uint64_t memObjGpuVa, - uint64_t hostAllocGpuVa, const Vec3 &hostPtrOffset, - const Vec3 ©Offset, Vec3 copySize, - size_t hostRowPitch, size_t hostSlicePitch, - size_t gpuRowPitch, size_t gpuSlicePitch); - - static BlitProperties constructPropertiesForCopy(GraphicsAllocation *dstAllocation, GraphicsAllocation *srcAllocation, - const Vec3 &dstOffset, const Vec3 &srcOffset, Vec3 copySize, - size_t srcRowPitch, size_t srcSlicePitch, - size_t dstRowPitch, size_t dstSlicePitch, GraphicsAllocation *clearColorAllocation); - - static BlitProperties constructPropertiesForAuxTranslation(AuxTranslationDirection auxTranslationDirection, - GraphicsAllocation *allocation, GraphicsAllocation *clearColorAllocation); - - static void setupDependenciesForAuxTranslation(BlitPropertiesContainer &blitPropertiesContainer, TimestampPacketDependencies ×tampPacketDependencies, - TimestampPacketContainer &kernelTimestamps, const CsrDependencies &depsFromEvents, - CommandStreamReceiver &gpguCsr, CommandStreamReceiver &bcsCsr); - - TagNodeBase *outputTimestampPacket = nullptr; - BlitterConstants::BlitDirection blitDirection = BlitterConstants::BlitDirection::BufferToHostPtr; - CsrDependencies csrDependencies; - AuxTranslationDirection auxTranslationDirection = AuxTranslationDirection::None; - - GraphicsAllocation *dstAllocation = nullptr; - GraphicsAllocation *srcAllocation = nullptr; - GraphicsAllocation *clearColorAllocation = nullptr; - uint64_t dstGpuAddress = 0; - uint64_t srcGpuAddress = 0; - - Vec3 copySize = 0; - Vec3 dstOffset = 0; - Vec3 srcOffset = 0; - bool isSystemMemoryPoolUsed = false; - - size_t dstRowPitch = 0; - size_t dstSlicePitch = 0; - size_t srcRowPitch = 0; - size_t srcSlicePitch = 0; - Vec3 dstSize = 0; - Vec3 srcSize = 0; - size_t bytesPerPixel = 1; - GMM_YUV_PLANE_ENUM dstPlane = GMM_YUV_PLANE_ENUM::GMM_NO_PLANE; - GMM_YUV_PLANE_ENUM srcPlane = GMM_YUV_PLANE_ENUM::GMM_NO_PLANE; - - bool isImageOperation() const { - return blitDirection == BlitterConstants::BlitDirection::HostPtrToImage || - blitDirection == BlitterConstants::BlitDirection::ImageToHostPtr || - blitDirection == BlitterConstants::BlitDirection::ImageToImage; - } -}; - -enum class BlitOperationResult { - Unsupported, - Fail, - Success, - GpuHang -}; - -namespace BlitHelperFunctions { -using BlitMemoryToAllocationFunc = std::function &size)>; -extern BlitMemoryToAllocationFunc blitMemoryToAllocation; -} // namespace BlitHelperFunctions - -struct BlitHelper { - static BlitOperationResult blitMemoryToAllocation(const Device &device, GraphicsAllocation *memory, size_t offset, const void *hostPtr, - const Vec3 &size); - static BlitOperationResult blitMemoryToAllocationBanks(const Device &device, GraphicsAllocation *memory, size_t offset, const void *hostPtr, - const Vec3 &size, DeviceBitfield memoryBanks); -}; +struct RootDeviceEnvironment; template struct BlitCommandsHelper { diff --git a/shared/source/helpers/blit_commands_helper_base.inl b/shared/source/helpers/blit_commands_helper_base.inl index bef912ec91..9fc8a7b68a 100644 --- a/shared/source/helpers/blit_commands_helper_base.inl +++ b/shared/source/helpers/blit_commands_helper_base.inl @@ -9,7 +9,7 @@ #include "shared/source/gmm_helper/gmm.h" #include "shared/source/gmm_helper/gmm_helper.h" #include "shared/source/gmm_helper/resource_info.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/helpers/hw_helper.h" #include "shared/source/helpers/register_offsets.h" #include "shared/source/helpers/timestamp_packet.h" diff --git a/shared/source/helpers/blit_helper.cpp b/shared/source/helpers/blit_helper.cpp new file mode 100644 index 0000000000..d7c8847a40 --- /dev/null +++ b/shared/source/helpers/blit_helper.cpp @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/helpers/blit_helper.h" + +#include "shared/source/command_stream/command_stream_receiver.h" +#include "shared/source/device/device.h" +#include "shared/source/helpers/blit_properties.h" +#include "shared/source/helpers/hw_helper.h" +#include "shared/source/memory_manager/graphics_allocation.h" + +namespace NEO { + +namespace BlitHelperFunctions { +BlitMemoryToAllocationFunc blitMemoryToAllocation = BlitHelper::blitMemoryToAllocation; +} // namespace BlitHelperFunctions + +BlitOperationResult BlitHelper::blitMemoryToAllocation(const Device &device, GraphicsAllocation *memory, size_t offset, const void *hostPtr, + const Vec3 &size) { + auto memoryBanks = memory->storageInfo.getMemoryBanks(); + return blitMemoryToAllocationBanks(device, memory, offset, hostPtr, size, memoryBanks); +} + +BlitOperationResult BlitHelper::blitMemoryToAllocationBanks(const Device &device, GraphicsAllocation *memory, size_t offset, const void *hostPtr, + const Vec3 &size, DeviceBitfield memoryBanks) { + const auto &hwInfo = device.getHardwareInfo(); + if (!hwInfo.capabilityTable.blitterOperationsSupported) { + return BlitOperationResult::Unsupported; + } + auto &gfxCoreHelper = device.getGfxCoreHelper(); + + UNRECOVERABLE_IF(memoryBanks.none()); + + auto pRootDevice = device.getRootDevice(); + + for (uint8_t tileId = 0u; tileId < 4u; tileId++) { + if (!memoryBanks.test(tileId)) { + continue; + } + + UNRECOVERABLE_IF(!pRootDevice->getDeviceBitfield().test(tileId)); + auto pDeviceForBlit = pRootDevice->getNearestGenericSubDevice(tileId); + auto &selectorCopyEngine = pDeviceForBlit->getSelectorCopyEngine(); + auto deviceBitfield = pDeviceForBlit->getDeviceBitfield(); + auto internalUsage = true; + auto bcsEngineType = EngineHelpers::getBcsEngineType(pDeviceForBlit->getRootDeviceEnvironment(), deviceBitfield, selectorCopyEngine, internalUsage); + auto bcsEngineUsage = gfxCoreHelper.preferInternalBcsEngine() ? EngineUsage::Internal : EngineUsage::Regular; + auto bcsEngine = pDeviceForBlit->tryGetEngine(bcsEngineType, bcsEngineUsage); + if (!bcsEngine) { + return BlitOperationResult::Unsupported; + } + + bcsEngine->commandStreamReceiver->initializeResources(); + bcsEngine->commandStreamReceiver->initDirectSubmission(); + BlitPropertiesContainer blitPropertiesContainer; + blitPropertiesContainer.push_back( + BlitProperties::constructPropertiesForReadWrite(BlitterConstants::BlitDirection::HostPtrToBuffer, + *bcsEngine->commandStreamReceiver, memory, nullptr, + hostPtr, + (memory->getGpuAddress() + offset), + 0, 0, 0, size, 0, 0, 0, 0)); + + const auto newTaskCount = bcsEngine->commandStreamReceiver->flushBcsTask(blitPropertiesContainer, true, false, *pDeviceForBlit); + if (newTaskCount == CompletionStamp::gpuHang) { + return BlitOperationResult::GpuHang; + } + } + + return BlitOperationResult::Success; +} + +} // namespace NEO diff --git a/shared/source/helpers/blit_helper.h b/shared/source/helpers/blit_helper.h new file mode 100644 index 0000000000..60434b8fd8 --- /dev/null +++ b/shared/source/helpers/blit_helper.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once +#include "shared/source/helpers/common_types.h" +#include "shared/source/helpers/vec.h" + +#include + +namespace NEO { + +class GraphicsAllocation; + +enum class BlitOperationResult { + Unsupported, + Fail, + Success, + GpuHang +}; + +namespace BlitHelperFunctions { +using BlitMemoryToAllocationFunc = std::function &size)>; +extern BlitMemoryToAllocationFunc blitMemoryToAllocation; +} // namespace BlitHelperFunctions + +struct BlitHelper { + static BlitOperationResult blitMemoryToAllocation(const Device &device, GraphicsAllocation *memory, size_t offset, const void *hostPtr, + const Vec3 &size); + static BlitOperationResult blitMemoryToAllocationBanks(const Device &device, GraphicsAllocation *memory, size_t offset, const void *hostPtr, + const Vec3 &size, DeviceBitfield memoryBanks); +}; + +} // namespace NEO \ No newline at end of file diff --git a/shared/source/helpers/blit_commands_helper.cpp b/shared/source/helpers/blit_properties.cpp similarity index 73% rename from shared/source/helpers/blit_commands_helper.cpp rename to shared/source/helpers/blit_properties.cpp index 3312532c07..c5ca44600c 100644 --- a/shared/source/helpers/blit_commands_helper.cpp +++ b/shared/source/helpers/blit_properties.cpp @@ -1,25 +1,18 @@ /* - * Copyright (C) 2019-2023 Intel Corporation + * Copyright (C) 2023 Intel Corporation * * SPDX-License-Identifier: MIT * */ -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/command_stream/command_stream_receiver.h" -#include "shared/source/device/device.h" -#include "shared/source/helpers/engine_node_helper.h" -#include "shared/source/helpers/hw_helper.h" #include "shared/source/helpers/timestamp_packet.h" #include "shared/source/memory_manager/surface.h" namespace NEO { -namespace BlitHelperFunctions { -BlitMemoryToAllocationFunc blitMemoryToAllocation = BlitHelper::blitMemoryToAllocation; -} // namespace BlitHelperFunctions - BlitProperties BlitProperties::constructPropertiesForReadWrite(BlitterConstants::BlitDirection blitDirection, CommandStreamReceiver &commandStreamReceiver, GraphicsAllocation *memObjAllocation, @@ -168,58 +161,10 @@ void BlitProperties::setupDependenciesForAuxTranslation(BlitPropertiesContainer blitPropertiesContainer[numObjects].csrDependencies.timestampPacketContainer.push_back(&kernelTimestamps); } -BlitOperationResult BlitHelper::blitMemoryToAllocation(const Device &device, GraphicsAllocation *memory, size_t offset, const void *hostPtr, - const Vec3 &size) { - auto memoryBanks = memory->storageInfo.getMemoryBanks(); - return blitMemoryToAllocationBanks(device, memory, offset, hostPtr, size, memoryBanks); +bool BlitProperties::isImageOperation() const { + return blitDirection == BlitterConstants::BlitDirection::HostPtrToImage || + blitDirection == BlitterConstants::BlitDirection::ImageToHostPtr || + blitDirection == BlitterConstants::BlitDirection::ImageToImage; } -BlitOperationResult BlitHelper::blitMemoryToAllocationBanks(const Device &device, GraphicsAllocation *memory, size_t offset, const void *hostPtr, - const Vec3 &size, DeviceBitfield memoryBanks) { - const auto &hwInfo = device.getHardwareInfo(); - if (!hwInfo.capabilityTable.blitterOperationsSupported) { - return BlitOperationResult::Unsupported; - } - auto &gfxCoreHelper = device.getGfxCoreHelper(); - - UNRECOVERABLE_IF(memoryBanks.none()); - - auto pRootDevice = device.getRootDevice(); - - for (uint8_t tileId = 0u; tileId < 4u; tileId++) { - if (!memoryBanks.test(tileId)) { - continue; - } - - UNRECOVERABLE_IF(!pRootDevice->getDeviceBitfield().test(tileId)); - auto pDeviceForBlit = pRootDevice->getNearestGenericSubDevice(tileId); - auto &selectorCopyEngine = pDeviceForBlit->getSelectorCopyEngine(); - auto deviceBitfield = pDeviceForBlit->getDeviceBitfield(); - auto internalUsage = true; - auto bcsEngineType = EngineHelpers::getBcsEngineType(pDeviceForBlit->getRootDeviceEnvironment(), deviceBitfield, selectorCopyEngine, internalUsage); - auto bcsEngineUsage = gfxCoreHelper.preferInternalBcsEngine() ? EngineUsage::Internal : EngineUsage::Regular; - auto bcsEngine = pDeviceForBlit->tryGetEngine(bcsEngineType, bcsEngineUsage); - if (!bcsEngine) { - return BlitOperationResult::Unsupported; - } - - bcsEngine->commandStreamReceiver->initializeResources(); - bcsEngine->commandStreamReceiver->initDirectSubmission(); - BlitPropertiesContainer blitPropertiesContainer; - blitPropertiesContainer.push_back( - BlitProperties::constructPropertiesForReadWrite(BlitterConstants::BlitDirection::HostPtrToBuffer, - *bcsEngine->commandStreamReceiver, memory, nullptr, - hostPtr, - (memory->getGpuAddress() + offset), - 0, 0, 0, size, 0, 0, 0, 0)); - - const auto newTaskCount = bcsEngine->commandStreamReceiver->flushBcsTask(blitPropertiesContainer, true, false, *pDeviceForBlit); - if (newTaskCount == CompletionStamp::gpuHang) { - return BlitOperationResult::GpuHang; - } - } - - return BlitOperationResult::Success; -} - -} // namespace NEO +} // namespace NEO \ No newline at end of file diff --git a/shared/source/helpers/blit_properties.h b/shared/source/helpers/blit_properties.h new file mode 100644 index 0000000000..20215fbc5d --- /dev/null +++ b/shared/source/helpers/blit_properties.h @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once +#include "shared/source/command_stream/csr_deps.h" +#include "shared/source/gmm_helper/gmm_lib.h" +#include "shared/source/helpers/blit_properties_container.h" +#include "shared/source/helpers/vec.h" + +#include + +namespace BlitterConstants { +enum class BlitDirection : uint32_t; +} + +namespace NEO { +struct TimestampPacketDependencies; +class TagNodeBase; +class TimestampPacketContainer; +enum class AuxTranslationDirection : uint32_t; +class GraphicsAllocation; +class CommandStreamReceiver; + +struct BlitProperties { + static BlitProperties constructPropertiesForReadWrite(BlitterConstants::BlitDirection blitDirection, + CommandStreamReceiver &commandStreamReceiver, + GraphicsAllocation *memObjAllocation, + GraphicsAllocation *preallocatedHostAllocation, + const void *hostPtr, uint64_t memObjGpuVa, + uint64_t hostAllocGpuVa, const Vec3 &hostPtrOffset, + const Vec3 ©Offset, Vec3 copySize, + size_t hostRowPitch, size_t hostSlicePitch, + size_t gpuRowPitch, size_t gpuSlicePitch); + + static BlitProperties constructPropertiesForCopy(GraphicsAllocation *dstAllocation, GraphicsAllocation *srcAllocation, + const Vec3 &dstOffset, const Vec3 &srcOffset, Vec3 copySize, + size_t srcRowPitch, size_t srcSlicePitch, + size_t dstRowPitch, size_t dstSlicePitch, GraphicsAllocation *clearColorAllocation); + + static BlitProperties constructPropertiesForAuxTranslation(AuxTranslationDirection auxTranslationDirection, + GraphicsAllocation *allocation, GraphicsAllocation *clearColorAllocation); + + static void setupDependenciesForAuxTranslation(BlitPropertiesContainer &blitPropertiesContainer, TimestampPacketDependencies ×tampPacketDependencies, + TimestampPacketContainer &kernelTimestamps, const CsrDependencies &depsFromEvents, + CommandStreamReceiver &gpguCsr, CommandStreamReceiver &bcsCsr); + + TagNodeBase *outputTimestampPacket = nullptr; + BlitterConstants::BlitDirection blitDirection; + CsrDependencies csrDependencies; + AuxTranslationDirection auxTranslationDirection; + + GraphicsAllocation *dstAllocation = nullptr; + GraphicsAllocation *srcAllocation = nullptr; + GraphicsAllocation *clearColorAllocation = nullptr; + uint64_t dstGpuAddress = 0; + uint64_t srcGpuAddress = 0; + + Vec3 copySize = 0; + Vec3 dstOffset = 0; + Vec3 srcOffset = 0; + bool isSystemMemoryPoolUsed = false; + + size_t dstRowPitch = 0; + size_t dstSlicePitch = 0; + size_t srcRowPitch = 0; + size_t srcSlicePitch = 0; + Vec3 dstSize = 0; + Vec3 srcSize = 0; + size_t bytesPerPixel = 1; + GMM_YUV_PLANE_ENUM dstPlane = GMM_YUV_PLANE_ENUM::GMM_NO_PLANE; + GMM_YUV_PLANE_ENUM srcPlane = GMM_YUV_PLANE_ENUM::GMM_NO_PLANE; + + bool isImageOperation() const; +}; + +} // namespace NEO \ No newline at end of file diff --git a/shared/source/memory_manager/memory_manager.cpp b/shared/source/memory_manager/memory_manager.cpp index 57fea243f7..9a3a381537 100644 --- a/shared/source/memory_manager/memory_manager.cpp +++ b/shared/source/memory_manager/memory_manager.cpp @@ -19,7 +19,7 @@ #include "shared/source/helpers/aligned_memory.h" #include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/bindless_heaps_helper.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_helper.h" #include "shared/source/helpers/hw_helper.h" #include "shared/source/helpers/hw_info.h" #include "shared/source/helpers/memory_properties_helpers.h" diff --git a/shared/source/xe_hpc_core/command_stream_receiver_hw_xe_hpc_core.cpp b/shared/source/xe_hpc_core/command_stream_receiver_hw_xe_hpc_core.cpp index 6deddbcde4..4def7772b5 100644 --- a/shared/source/xe_hpc_core/command_stream_receiver_hw_xe_hpc_core.cpp +++ b/shared/source/xe_hpc_core/command_stream_receiver_hw_xe_hpc_core.cpp @@ -13,6 +13,7 @@ using Family = NEO::XeHpcCoreFamily; #include "shared/source/command_stream/command_stream_receiver_hw_dg2_and_later.inl" #include "shared/source/command_stream/command_stream_receiver_hw_xehp_and_later.inl" #include "shared/source/helpers/blit_commands_helper_xehp_and_later.inl" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/helpers/populate_factory.h" #include "shared/source/helpers/state_base_address_xehp_and_later.inl" diff --git a/shared/test/common/libult/ult_command_stream_receiver.h b/shared/test/common/libult/ult_command_stream_receiver.h index 0ba4009fb6..e4052042d8 100644 --- a/shared/test/common/libult/ult_command_stream_receiver.h +++ b/shared/test/common/libult/ult_command_stream_receiver.h @@ -12,7 +12,7 @@ #include "shared/source/device/device.h" #include "shared/source/direct_submission/direct_submission_hw.h" #include "shared/source/execution_environment/execution_environment.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/memory_manager/graphics_allocation.h" #include "shared/source/memory_manager/surface.h" #include "shared/source/os_interface/os_context.h" diff --git a/shared/test/unit_test/command_stream/aub_command_stream_receiver_1_tests.cpp b/shared/test/unit_test/command_stream/aub_command_stream_receiver_1_tests.cpp index e92dbd34ce..b17e79242b 100644 --- a/shared/test/unit_test/command_stream/aub_command_stream_receiver_1_tests.cpp +++ b/shared/test/unit_test/command_stream/aub_command_stream_receiver_1_tests.cpp @@ -6,7 +6,7 @@ */ #include "shared/source/aub_mem_dump/page_table_entry_bits.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/helpers/hardware_context_controller.h" #include "shared/source/helpers/hw_helper.h" #include "shared/source/memory_manager/os_agnostic_memory_manager.h" diff --git a/shared/test/unit_test/debugger/test_l0_debugger.cpp b/shared/test/unit_test/debugger/test_l0_debugger.cpp index 7f71e823c8..62fb833226 100644 --- a/shared/test/unit_test/debugger/test_l0_debugger.cpp +++ b/shared/test/unit_test/debugger/test_l0_debugger.cpp @@ -7,7 +7,7 @@ #include "shared/source/built_ins/sip.h" #include "shared/source/gen_common/reg_configs_common.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_helper.h" #include "shared/source/helpers/hw_helper.h" #include "shared/source/helpers/preamble.h" #include "shared/source/memory_manager/allocation_properties.h" diff --git a/shared/test/unit_test/helpers/blit_commands_helper_tests.cpp b/shared/test/unit_test/helpers/blit_commands_helper_tests.cpp index 2675b9b744..c69b591a65 100644 --- a/shared/test/unit_test/helpers/blit_commands_helper_tests.cpp +++ b/shared/test/unit_test/helpers/blit_commands_helper_tests.cpp @@ -9,7 +9,7 @@ #include "shared/source/command_container/command_encoder.h" #include "shared/source/command_stream/command_stream_receiver.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/test/common/cmd_parse/hw_parse.h" #include "shared/test/common/fixtures/device_fixture.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" diff --git a/shared/test/unit_test/helpers/blit_commands_helper_tests_gen12lp.cpp b/shared/test/unit_test/helpers/blit_commands_helper_tests_gen12lp.cpp index 056a991941..fb8e955618 100644 --- a/shared/test/unit_test/helpers/blit_commands_helper_tests_gen12lp.cpp +++ b/shared/test/unit_test/helpers/blit_commands_helper_tests_gen12lp.cpp @@ -1,12 +1,12 @@ /* - * Copyright (C) 2020-2022 Intel Corporation + * Copyright (C) 2020-2023 Intel Corporation * * SPDX-License-Identifier: MIT * */ #include "shared/source/command_container/command_encoder.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/test/common/cmd_parse/hw_parse.h" #include "shared/test/common/fixtures/device_fixture.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" diff --git a/shared/test/unit_test/helpers/test_blit_commands_helper_pvc_and_later.cpp b/shared/test/unit_test/helpers/test_blit_commands_helper_pvc_and_later.cpp index 22bdcc6b5d..a7d8a09a2e 100644 --- a/shared/test/unit_test/helpers/test_blit_commands_helper_pvc_and_later.cpp +++ b/shared/test/unit_test/helpers/test_blit_commands_helper_pvc_and_later.cpp @@ -6,6 +6,7 @@ */ #include "shared/source/gmm_helper/client_context/gmm_client_context.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" #include "shared/test/common/mocks/mock_gmm.h" #include "shared/test/unit_test/helpers/blit_commands_helper_tests.inl" diff --git a/shared/test/unit_test/helpers/test_blit_commands_helper_xehp_and_later.cpp b/shared/test/unit_test/helpers/test_blit_commands_helper_xehp_and_later.cpp index e033fcc1b6..db1ae3e8e7 100644 --- a/shared/test/unit_test/helpers/test_blit_commands_helper_xehp_and_later.cpp +++ b/shared/test/unit_test/helpers/test_blit_commands_helper_xehp_and_later.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2022 Intel Corporation + * Copyright (C) 2021-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -7,6 +7,7 @@ #include "shared/source/gmm_helper/client_context/gmm_client_context.h" #include "shared/source/gmm_helper/gmm_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/helpers/hw_helper.h" #include "shared/source/helpers/local_memory_access_modes.h" #include "shared/source/os_interface/hw_info_config.h" diff --git a/shared/test/unit_test/program/program_initialization_tests.cpp b/shared/test/unit_test/program/program_initialization_tests.cpp index 08b5cc6639..18bad9dae7 100644 --- a/shared/test/unit_test/program/program_initialization_tests.cpp +++ b/shared/test/unit_test/program/program_initialization_tests.cpp @@ -6,7 +6,7 @@ */ #include "shared/source/compiler_interface/external_functions.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_helper.h" #include "shared/source/helpers/local_memory_access_modes.h" #include "shared/source/program/program_initialization.h" #include "shared/test/common/compiler_interface/linker_mock.h" diff --git a/shared/test/unit_test/scratch_space_controler/scratch_space_controler_tests.cpp b/shared/test/unit_test/scratch_space_controler/scratch_space_controler_tests.cpp index 59bea9fc36..af87d67d26 100644 --- a/shared/test/unit_test/scratch_space_controler/scratch_space_controler_tests.cpp +++ b/shared/test/unit_test/scratch_space_controler/scratch_space_controler_tests.cpp @@ -6,7 +6,7 @@ */ #include "shared/source/command_stream/scratch_space_controller_base.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/test/common/fixtures/device_fixture.h" #include "shared/test/common/mocks/mock_command_stream_receiver.h" #include "shared/test/common/mocks/mock_device.h" diff --git a/shared/test/unit_test/scratch_space_controler/scratch_space_controler_xehp_and_later_tests.cpp b/shared/test/unit_test/scratch_space_controler/scratch_space_controler_xehp_and_later_tests.cpp index 550f85a4e6..e745f7edde 100644 --- a/shared/test/unit_test/scratch_space_controler/scratch_space_controler_xehp_and_later_tests.cpp +++ b/shared/test/unit_test/scratch_space_controler/scratch_space_controler_xehp_and_later_tests.cpp @@ -7,7 +7,7 @@ #include "shared/source/command_stream/scratch_space_controller_xehp_and_later.h" #include "shared/source/helpers/bindless_heaps_helper.h" -#include "shared/source/helpers/blit_commands_helper.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/test/common/fixtures/device_fixture.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" #include "shared/test/common/mocks/mock_command_stream_receiver.h"