diff --git a/level_zero/core/source/cmdlist/CMakeLists.txt b/level_zero/core/source/cmdlist/CMakeLists.txt index 81465705b7..75dea0fcad 100644 --- a/level_zero/core/source/cmdlist/CMakeLists.txt +++ b/level_zero/core/source/cmdlist/CMakeLists.txt @@ -20,6 +20,7 @@ target_sources(${L0_STATIC_LIB_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/cmdlist_memory_copy_params.h ${CMAKE_CURRENT_SOURCE_DIR}/command_to_patch.h ${CMAKE_CURRENT_SOURCE_DIR}/definitions${BRANCH_DIR_SUFFIX}cmdlist_launch_params_ext.h + ${CMAKE_CURRENT_SOURCE_DIR}/definitions${BRANCH_DIR_SUFFIX}cmdlist_memory_copy_params_ext.h ${CMAKE_CURRENT_SOURCE_DIR}/definitions${BRANCH_DIR_SUFFIX}copy_offload_mode.h ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}cmdlist_additional_args.cpp ) diff --git a/level_zero/core/source/cmdlist/cmdlist.h b/level_zero/core/source/cmdlist/cmdlist.h index 1d7eb206c4..77c3c4e741 100644 --- a/level_zero/core/source/cmdlist/cmdlist.h +++ b/level_zero/core/source/cmdlist/cmdlist.h @@ -11,6 +11,7 @@ #include "shared/source/command_stream/preemption_mode.h" #include "shared/source/command_stream/stream_properties.h" #include "shared/source/command_stream/thread_arbitration_policy.h" +#include "shared/source/helpers/blit_properties.h" #include "shared/source/helpers/cache_policy.h" #include "shared/source/helpers/common_types.h" #include "shared/source/helpers/definitions/command_encoder_args.h" @@ -154,6 +155,10 @@ struct CommandList : _ze_command_list_handle_t { virtual ze_result_t appendMemoryCopy(void *dstptr, const void *srcptr, size_t size, ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, CmdListMemoryCopyParams &memoryCopyParams) = 0; + virtual ze_result_t appendMemoryCopyWithParameters(void *dstptr, const void *srcptr, size_t size, + const void *pNext, + ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, + ze_event_handle_t *phWaitEvents) = 0; virtual ze_result_t appendPageFaultCopy(NEO::GraphicsAllocation *dstptr, NEO::GraphicsAllocation *srcptr, size_t size, bool flushHost) = 0; virtual ze_result_t appendMemoryCopyRegion(void *dstPtr, const ze_copy_region_t *dstRegion, @@ -169,6 +174,9 @@ struct CommandList : _ze_command_list_handle_t { virtual ze_result_t appendMemoryFill(void *ptr, const void *pattern, size_t patternSize, size_t size, ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, CmdListMemoryCopyParams &memoryCopyParams) = 0; + virtual ze_result_t appendMemoryFillWithParameters(void *ptr, const void *pattern, + size_t patternSize, size_t size, const void *pNext, ze_event_handle_t hSignalEvent, + uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents) = 0; virtual ze_result_t appendMemoryPrefetch(const void *ptr, size_t count) = 0; virtual ze_result_t appendSignalEvent(ze_event_handle_t hEvent, bool relaxedOrderingDispatch) = 0; virtual ze_result_t appendWaitOnEvents(uint32_t numEvents, ze_event_handle_t *phEvent, CommandToPatchContainer *outWaitCmds, @@ -256,6 +264,8 @@ struct CommandList : _ze_command_list_handle_t { ze_result_t validateLaunchParams(const Kernel &kernel, const CmdListKernelLaunchParams &launchParams) const; + void setAdditionalBlitPropertiesFromMemoryCopyParams(NEO::BlitProperties &blitProperties, const CmdListMemoryCopyParams &memoryCopyParams) const; + void setOrdinal(uint32_t ord) { ordinal = ord; } void setCommandListPerThreadScratchSize(uint32_t slotId, uint32_t size) { UNRECOVERABLE_IF(slotId > 1); @@ -457,6 +467,7 @@ struct CommandList : _ze_command_list_handle_t { return closedCmdList; } ze_result_t obtainLaunchParamsFromExtensions(const ze_base_desc_t *desc, CmdListKernelLaunchParams &launchParams, ze_kernel_handle_t kernelHandle) const; + ze_result_t obtainMemoryCopyParamsFromExtensions(const ze_base_desc_t *desc, CmdListMemoryCopyParams &memoryCopyParams) const; void setCaptureTarget(Graph *graph) { this->captureTarget = graph; diff --git a/level_zero/core/source/cmdlist/cmdlist_additional_args.cpp b/level_zero/core/source/cmdlist/cmdlist_additional_args.cpp index cf677ca5dd..18acb7c9d8 100644 --- a/level_zero/core/source/cmdlist/cmdlist_additional_args.cpp +++ b/level_zero/core/source/cmdlist/cmdlist_additional_args.cpp @@ -62,4 +62,11 @@ ze_result_t CommandList::obtainLaunchParamsFromExtensions(const ze_base_desc_t * return ZE_RESULT_SUCCESS; } +void CommandList::setAdditionalBlitPropertiesFromMemoryCopyParams(NEO::BlitProperties &blitProperties, const CmdListMemoryCopyParams &memoryCopyParams) const { +} + +ze_result_t CommandList::obtainMemoryCopyParamsFromExtensions(const ze_base_desc_t *desc, CmdListMemoryCopyParams &memoryCopyParams) const { + return ZE_RESULT_SUCCESS; +} + } // namespace L0 diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.h b/level_zero/core/source/cmdlist/cmdlist_hw.h index 09adf32936..0120482d1a 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.h +++ b/level_zero/core/source/cmdlist/cmdlist_hw.h @@ -156,6 +156,10 @@ struct CommandListCoreFamily : public CommandListImp { ze_result_t appendMemoryCopy(void *dstptr, const void *srcptr, size_t size, ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, CmdListMemoryCopyParams &memoryCopyParams) override; + ze_result_t appendMemoryCopyWithParameters(void *dstptr, const void *srcptr, size_t size, + const void *pNext, + ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, + ze_event_handle_t *phWaitEvents) override; ze_result_t appendPageFaultCopy(NEO::GraphicsAllocation *dstAllocation, NEO::GraphicsAllocation *srcAllocation, size_t size, @@ -177,6 +181,11 @@ struct CommandListCoreFamily : public CommandListImp { ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, CmdListMemoryCopyParams &memoryCopyParams) override; + ze_result_t appendMemoryFillWithParameters(void *ptr, const void *pattern, + size_t patternSize, size_t size, + const void *pNext, + ze_event_handle_t hSignalEvent, + uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents) override; ze_result_t appendMIBBStart(uint64_t address, size_t predication, bool secondLevel) override; ze_result_t appendMIBBEnd() override; @@ -254,7 +263,8 @@ struct CommandListCoreFamily : public CommandListImp { NEO::GraphicsAllocation *srcPtrAlloc, uint64_t srcOffset, uint64_t size, - Event *signalEvent); + Event *signalEvent, + CmdListMemoryCopyParams &memoryCopyParams); MOCKABLE_VIRTUAL ze_result_t appendMemoryCopyBlitRegion(AlignedAllocationData *srcAllocationData, AlignedAllocationData *dstAllocationData, diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.inl b/level_zero/core/source/cmdlist/cmdlist_hw.inl index 28b41d59f8..0bb6219c41 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.inl +++ b/level_zero/core/source/cmdlist/cmdlist_hw.inl @@ -1551,7 +1551,8 @@ ze_result_t CommandListCoreFamily::appendMemoryCopyBlit(uintptr_t NEO::GraphicsAllocation *srcPtrAlloc, uint64_t srcOffset, uint64_t size, - Event *signalEvent) { + Event *signalEvent, + CmdListMemoryCopyParams &memoryCopyParams) { if (dstPtrAlloc) { dstOffset += ptrDiff(dstPtr, dstPtrAlloc->getGpuAddress()); } @@ -1568,6 +1569,8 @@ ze_result_t CommandListCoreFamily::appendMemoryCopyBlit(uintptr_t blitProperties.computeStreamPartitionCount = this->partitionCount; blitProperties.highPriority = isHighPriorityImmediateCmdList(); + setAdditionalBlitPropertiesFromMemoryCopyParams(blitProperties, memoryCopyParams); + addResidency(dstPtrAlloc, srcPtrAlloc, clearColorAllocation); size_t nBlitsPerRow = NEO::BlitCommandsHelper::getNumberOfBlitsForCopyPerRow(blitProperties.copySize, device->getNEODevice()->getRootDeviceEnvironmentRef(), blitProperties.isSystemMemoryPoolUsed); @@ -1737,9 +1740,10 @@ ze_result_t CommandListCoreFamily::appendPageFaultCopy(NEO::Graph uintptr_t srcAddress = static_cast(srcAllocation->getGpuAddress()); ze_result_t ret = ZE_RESULT_ERROR_UNKNOWN; if (isCopyOnly(false)) { + CmdListMemoryCopyParams memoryCopyParams{}; return appendMemoryCopyBlit(dstAddress, dstAllocation, 0u, srcAddress, srcAllocation, 0u, - size, nullptr); + size, nullptr, memoryCopyParams); } else { CmdListKernelLaunchParams launchParams = {}; launchParams.isKernelSplitOperation = rightSize > 0; @@ -1931,7 +1935,7 @@ ze_result_t CommandListCoreFamily::appendMemoryCopy(void *dstptr, ret = appendMemoryCopyBlit(dstAllocationStruct.alignedAllocationPtr, dstAllocationStruct.alloc, dstAllocationStruct.offset, srcAllocationStruct.alignedAllocationPtr, - srcAllocationStruct.alloc, srcAllocationStruct.offset, size, signalEvent); + srcAllocationStruct.alloc, srcAllocationStruct.offset, size, signalEvent, memoryCopyParams); } else { if (NEO::debugManager.flags.FlushTlbBeforeCopy.get() == 1) { NEO::PipeControlArgs args; @@ -2029,6 +2033,20 @@ ze_result_t CommandListCoreFamily::appendMemoryCopy(void *dstptr, return ret; } +template +ze_result_t CommandListCoreFamily::appendMemoryCopyWithParameters(void *dstptr, + const void *srcptr, + size_t size, + const void *pNext, + ze_event_handle_t hSignalEvent, + uint32_t numWaitEvents, + ze_event_handle_t *phWaitEvents) { + CmdListMemoryCopyParams memoryCopyParams{}; + obtainMemoryCopyParamsFromExtensions(static_cast(pNext), memoryCopyParams); + + return appendMemoryCopy(dstptr, srcptr, size, hSignalEvent, numWaitEvents, phWaitEvents, memoryCopyParams); +} + template ze_result_t CommandListCoreFamily::appendMemoryCopyRegion(void *dstPtr, const ze_copy_region_t *dstRegion, @@ -2659,6 +2677,21 @@ ze_result_t CommandListCoreFamily::appendMemoryFill(void *ptr, return res; } +template +ze_result_t CommandListCoreFamily::appendMemoryFillWithParameters(void *ptr, + const void *pattern, + size_t patternSize, + size_t size, + const void *pNext, + ze_event_handle_t hSignalEvent, + uint32_t numWaitEvents, + ze_event_handle_t *phWaitEvents) { + CmdListMemoryCopyParams memoryCopyParams{}; + obtainMemoryCopyParamsFromExtensions(static_cast(pNext), memoryCopyParams); + + return appendMemoryFill(ptr, pattern, patternSize, size, hSignalEvent, numWaitEvents, phWaitEvents, memoryCopyParams); +} + template ze_result_t CommandListCoreFamily::appendBlitFill(void *ptr, const void *pattern, size_t patternSize, size_t size, Event *signalEvent, uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, CmdListMemoryCopyParams &memoryCopyParams) { @@ -2723,6 +2756,8 @@ ze_result_t CommandListCoreFamily::appendBlitFill(void *ptr, cons return ZE_RESULT_ERROR_INVALID_ARGUMENT; } + setAdditionalBlitPropertiesFromMemoryCopyParams(blitProperties, memoryCopyParams); + if (useAdditionalBlitProperties) { setAdditionalBlitProperties(blitProperties, signalEvent, useAdditionalTimestamp); } diff --git a/level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl b/level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl index 4f05a0b39d..3c82a7e45c 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl +++ b/level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl @@ -827,9 +827,10 @@ ze_result_t CommandListCoreFamilyImmediate::appendPageFaultCopy(N uintptr_t srcAddress = static_cast(srcAllocation->getGpuAddress()); auto splitCall = [&](CommandListCoreFamilyImmediate *subCmdList, uintptr_t dstAddressParam, uintptr_t srcAddressParam, size_t sizeParam, ze_event_handle_t hSignalEventParam) { + CmdListMemoryCopyParams memoryCopyParams{}; subCmdList->appendMemoryCopyBlit(dstAddressParam, dstAllocation, 0u, srcAddressParam, srcAllocation, 0u, - sizeParam, nullptr); + sizeParam, nullptr, memoryCopyParams); return subCmdList->CommandListCoreFamily::appendSignalEvent(hSignalEventParam, false); }; diff --git a/level_zero/core/source/cmdlist/cmdlist_memory_copy_params.h b/level_zero/core/source/cmdlist/cmdlist_memory_copy_params.h index 2041036d7a..94e55f4800 100644 --- a/level_zero/core/source/cmdlist/cmdlist_memory_copy_params.h +++ b/level_zero/core/source/cmdlist/cmdlist_memory_copy_params.h @@ -7,12 +7,15 @@ #pragma once +#include "cmdlist_memory_copy_params_ext.h" + namespace L0 { struct CmdListMemoryCopyParams { bool relaxedOrderingDispatch = false; bool forceDisableCopyOnlyInOrderSignaling = false; bool copyOffloadAllowed = false; bool taskCountUpdateRequired = false; + CmdListMemoryCopyParamsExt paramsExt{}; }; } // namespace L0 diff --git a/level_zero/core/source/cmdlist/definitions/cmdlist_memory_copy_params_ext.h b/level_zero/core/source/cmdlist/definitions/cmdlist_memory_copy_params_ext.h new file mode 100644 index 0000000000..1d0f75704a --- /dev/null +++ b/level_zero/core/source/cmdlist/definitions/cmdlist_memory_copy_params_ext.h @@ -0,0 +1,15 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +namespace L0 { + +struct CmdListMemoryCopyParamsExt { +}; + +} // namespace L0 diff --git a/level_zero/core/test/unit_tests/mocks/mock_cmdlist.h b/level_zero/core/test/unit_tests/mocks/mock_cmdlist.h index 8af25b74f2..e07502fb5d 100644 --- a/level_zero/core/test/unit_tests/mocks/mock_cmdlist.h +++ b/level_zero/core/test/unit_tests/mocks/mock_cmdlist.h @@ -493,6 +493,15 @@ struct Mock : public CommandList { uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, CmdListMemoryCopyParams &memoryCopyParams)); + ADDMETHOD_NOBASE(appendMemoryCopyWithParameters, ze_result_t, ZE_RESULT_SUCCESS, + (void *dstptr, + const void *srcptr, + size_t size, + const void *pNext, + ze_event_handle_t hEvent, + uint32_t numWaitEvents, + ze_event_handle_t *phWaitEvents)); + ADDMETHOD_NOBASE(appendPageFaultCopy, ze_result_t, ZE_RESULT_SUCCESS, (NEO::GraphicsAllocation * dstptr, NEO::GraphicsAllocation *srcptr, @@ -525,6 +534,16 @@ struct Mock : public CommandList { uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, CmdListMemoryCopyParams &memoryCopyParams)); + ADDMETHOD_NOBASE(appendMemoryFillWithParameters, ze_result_t, ZE_RESULT_SUCCESS, + (void *ptr, + const void *pattern, + size_t pattern_size, + size_t size, + const void *pNext, + ze_event_handle_t hEvent, + uint32_t numWaitEvents, + ze_event_handle_t *phWaitEvents)); + ADDMETHOD_NOBASE(appendSignalEvent, ze_result_t, ZE_RESULT_SUCCESS, (ze_event_handle_t hEvent, bool relaxedOrderingDispatch)); @@ -670,7 +689,7 @@ class MockCommandListCoreFamily : public CommandListCoreFamily { uint64_t dstOffset, uintptr_t srcPtr, NEO::GraphicsAllocation *srcPtrAlloc, uint64_t srcOffset, - uint64_t size, Event *signalEvent)); + uint64_t size, Event *signalEvent, CmdListMemoryCopyParams &memoryCopyParams)); ADDMETHOD_VOIDRETURN(allocateOrReuseKernelPrivateMemory, false, diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_2.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_2.cpp index 79e1108c9c..5ae5b6a3a0 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_2.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_2.cpp @@ -82,7 +82,7 @@ class MockCommandListHw : public WhiteBox<::L0::CommandListCoreFamily>>(); + commandList->initialize(device, NEO::EngineGroupType::renderCompute, 0u); + + ze_result_t result = commandList->obtainMemoryCopyParamsFromExtensions(&desc, memoryCopyParams); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); +} + HWTEST_F(CommandListTest, givenComputeCommandListAnd2dRegionWhenMemoryCopyRegionInUsmHostAllocationCalledThenBuiltinFlagAndDestinationAllocSystemIsSet) { auto commandList = std::make_unique>>(); commandList->initialize(device, NEO::EngineGroupType::renderCompute, 0u); diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_blit.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_blit.cpp index b864a5a0b5..cf597aaf09 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_blit.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_blit.cpp @@ -45,7 +45,7 @@ class MockCommandListForMemFill : public WhiteBox<::L0::CommandListCoreFamily(dstPtr), 0x1000, 0, sizeof(uint32_t), MemoryPool::system4KBPages, MemoryManager::maxOsContextCount); - commandList->appendMemoryCopyBlit(ptrOffset(dstPtr, dstOffset), &mockAllocationDst, 0, ptrOffset(srcPtr, srcOffset), &mockAllocationSrc, 0, copySize, nullptr); + CmdListMemoryCopyParams memoryCopyParams{}; + commandList->appendMemoryCopyBlit(ptrOffset(dstPtr, dstOffset), &mockAllocationDst, 0, ptrOffset(srcPtr, srcOffset), &mockAllocationSrc, 0, copySize, nullptr, memoryCopyParams); auto &commandContainer = commandList->getCmdContainer(); GenCmdList genCmdList; diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_memory_extension.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_memory_extension.cpp index fa0f4dbeec..393ce2e27d 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_memory_extension.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_memory_extension.cpp @@ -123,7 +123,7 @@ class MockCommandListExtensionHw : public WhiteBox<::L0::CommandListCoreFamily namespace NEO { @@ -114,6 +116,8 @@ struct BlitProperties { GMM_YUV_PLANE_ENUM srcPlane = GMM_YUV_PLANE_ENUM::GMM_NO_PLANE; bool isSystemMemoryPoolUsed = false; bool highPriority = false; + + BlitPropertiesExt propertiesExt{}; }; } // namespace NEO diff --git a/shared/source/helpers/blit_properties_ext.h b/shared/source/helpers/blit_properties_ext.h new file mode 100644 index 0000000000..a47c179781 --- /dev/null +++ b/shared/source/helpers/blit_properties_ext.h @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +namespace NEO { + +struct BlitPropertiesExt { +}; + +} // namespace NEO