From 31bcea128f20df81ab7b15ba9b76a58997d263b3 Mon Sep 17 00:00:00 2001 From: Narendra Bagria Date: Tue, 26 Aug 2025 06:55:23 +0000 Subject: [PATCH] feature: system allocator support for image APIs Related-To: NEO-15461 Signed-off-by: Young Jin Yoon Signed-off-by: Narendra Bagria --- level_zero/core/source/cmdlist/cmdlist_hw.h | 7 +- level_zero/core/source/cmdlist/cmdlist_hw.inl | 93 +++-- .../sources/cmdlist/test_cmdlist_2.cpp | 326 +++++++++++++++++- .../sources/cmdlist/test_cmdlist_blit.cpp | 30 +- .../cmdlist/test_cmdlist_memory_extension.cpp | 4 +- .../command_stream_receiver_hw_base.inl | 12 +- .../command_stream_receiver_hw_gen12lp.cpp | 14 +- .../blit_commands_helper_xehp_and_later.inl | 16 +- shared/source/helpers/blit_properties.cpp | 4 +- ...ommand_stream_receiver_hw_xe2_hpg_core.cpp | 14 +- .../command_stream_receiver_hw_xe3_core.cpp | 14 +- ...command_stream_receiver_hw_xe_hpc_core.cpp | 6 +- ...command_stream_receiver_hw_xe_hpg_core.cpp | 13 +- .../blitter_properties_tests.cpp | 21 ++ .../helpers/blit_commands_helper_tests.cpp | 255 ++++++++++++++ 15 files changed, 736 insertions(+), 93 deletions(-) diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.h b/level_zero/core/source/cmdlist/cmdlist_hw.h index d9e2236ea6..50b38926b6 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.h +++ b/level_zero/core/source/cmdlist/cmdlist_hw.h @@ -286,8 +286,8 @@ struct CommandListCoreFamily : public CommandListImp { uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents, CmdListMemoryCopyParams &memoryCopyParams); - MOCKABLE_VIRTUAL ze_result_t appendCopyImageBlit(NEO::GraphicsAllocation *src, - NEO::GraphicsAllocation *dst, + MOCKABLE_VIRTUAL ze_result_t appendCopyImageBlit(uintptr_t srcPtr, NEO::GraphicsAllocation *src, + uintptr_t dstPtr, NEO::GraphicsAllocation *dst, const Vec3 &srcOffsets, const Vec3 &dstOffsets, size_t srcRowPitch, size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, @@ -416,7 +416,8 @@ struct CommandListCoreFamily : public CommandListImp { bool singleEventPacketRequired(bool inputSinglePacketEventRequest) const; void programEventL3Flush(Event *event); virtual ze_result_t flushInOrderCounterSignal(bool waitOnInOrderCounterRequired) { return ZE_RESULT_SUCCESS; }; - bool isCopyOffloadAllowed(const NEO::GraphicsAllocation &srcAllocation, const NEO::GraphicsAllocation &dstAllocation) const; + bool isCopyOffloadAllowed(const NEO::GraphicsAllocation *srcAllocation, const NEO::GraphicsAllocation *dstAllocation) const; + bool isSharedSystemEnabled() const; void setAdditionalKernelLaunchParams(CmdListKernelLaunchParams &launchParams, Kernel &kernel) const; void dispatchInOrderPostOperationBarrier(Event *signalOperation, bool dcFlushRequired, bool copyOperation); NEO::GraphicsAllocation *getDeviceCounterAllocForResidency(NEO::GraphicsAllocation *counterDeviceAlloc); diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.inl b/level_zero/core/source/cmdlist/cmdlist_hw.inl index c8fab9ea25..df6e2a3635 100644 --- a/level_zero/core/source/cmdlist/cmdlist_hw.inl +++ b/level_zero/core/source/cmdlist/cmdlist_hw.inl @@ -777,6 +777,8 @@ ze_result_t CommandListCoreFamily::appendImageCopyFromMemoryExt(z return ZE_RESULT_ERROR_INVALID_NULL_POINTER; } + bool sharedSystemEnabled = isSharedSystemEnabled(); + auto image = Image::fromHandle(hDstImage); auto bytesPerPixel = static_cast(image->getImageInfo().surfaceFormat->imageElementSizeInBytes); Vec3 imgSize = {image->getImageDesc().width, @@ -820,13 +822,17 @@ ze_result_t CommandListCoreFamily::appendImageCopyFromMemoryExt(z uint64_t bufferSize = getInputBufferSize(image->getImageInfo().imgDesc.imageType, srcRowPitch, srcSlicePitch, pDstRegion); - auto allocationStruct = getAlignedAllocationData(this->device, false, srcPtr, bufferSize, true, false); - if (allocationStruct.alloc == nullptr) { + auto allocationStruct = getAlignedAllocationData(this->device, sharedSystemEnabled, srcPtr, bufferSize, true, false); + if (allocationStruct.alloc == nullptr && sharedSystemEnabled == false) { return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY; } memoryCopyParams.taskCountUpdateRequired |= CommandList::isExternalHostPtrAlloc(allocationStruct.alloc); + if ((allocationStruct.alloc == nullptr) && (NEO::debugManager.flags.EmitMemAdvisePriorToCopyForNonUsm.get() == 1)) { + appendMemAdvise(device, reinterpret_cast(allocationStruct.alignedAllocationPtr), bufferSize, static_cast(ZE_MEMORY_ADVICE_SET_SYSTEM_MEMORY_PREFERRED_LOCATION)); + } + DriverHandleImp *driverHandle = static_cast(device->getDriverHandle()); if (driverHandle->isRemoteImageNeeded(image, device)) { L0::Image *peerImage = nullptr; @@ -838,7 +844,7 @@ ze_result_t CommandListCoreFamily::appendImageCopyFromMemoryExt(z image = peerImage; } - memoryCopyParams.copyOffloadAllowed = isCopyOffloadAllowed(*allocationStruct.alloc, *image->getAllocation()); + memoryCopyParams.copyOffloadAllowed = isCopyOffloadAllowed(allocationStruct.alloc, image->getAllocation()); if (isCopyOnly(memoryCopyParams.copyOffloadAllowed)) { if ((bytesPerPixel == 3) || (bytesPerPixel == 6) || image->isMimickedImage()) { @@ -846,7 +852,7 @@ ze_result_t CommandListCoreFamily::appendImageCopyFromMemoryExt(z } size_t imgRowPitch = image->getImageInfo().rowPitch; size_t imgSlicePitch = image->getImageInfo().slicePitch; - auto status = appendCopyImageBlit(allocationStruct.alloc, image->getAllocation(), + auto status = appendCopyImageBlit(allocationStruct.alignedAllocationPtr, allocationStruct.alloc, image->getAllocation()->getGpuAddress(), image->getAllocation(), {0, 0, 0}, {pDstRegion->originX, pDstRegion->originY, pDstRegion->originZ}, srcRowPitch, srcSlicePitch, imgRowPitch, imgSlicePitch, bytesPerPixel, {pDstRegion->width, pDstRegion->height, pDstRegion->depth}, {pDstRegion->width, pDstRegion->height, pDstRegion->depth}, imgSize, event, numWaitEvents, phWaitEvents, memoryCopyParams); @@ -894,9 +900,7 @@ ze_result_t CommandListCoreFamily::appendImageCopyFromMemoryExt(z auto lock = device->getBuiltinFunctionsLib()->obtainUniqueOwnership(); Kernel *builtinKernel = device->getBuiltinFunctionsLib()->getImageFunction(builtInType); - builtinKernel->setArgBufferWithAlloc(0u, allocationStruct.alignedAllocationPtr, - allocationStruct.alloc, - nullptr); + builtinSetArgCopy(builtinKernel, 0, reinterpret_cast(&allocationStruct.alignedAllocationPtr), allocationStruct.alloc); builtinKernel->setArgRedescribedImage(1u, image->toHandle(), false); builtinKernel->setArgumentValue(2u, sizeof(size_t), &allocationStruct.offset); @@ -982,6 +986,8 @@ ze_result_t CommandListCoreFamily::appendImageCopyToMemoryExt(voi return ZE_RESULT_ERROR_INVALID_NULL_HANDLE; } + bool sharedSystemEnabled = isSharedSystemEnabled(); + auto image = Image::fromHandle(hSrcImage); auto bytesPerPixel = static_cast(image->getImageInfo().surfaceFormat->imageElementSizeInBytes); Vec3 imgSize = {image->getImageDesc().width, @@ -1025,13 +1031,17 @@ ze_result_t CommandListCoreFamily::appendImageCopyToMemoryExt(voi uint64_t bufferSize = getInputBufferSize(image->getImageInfo().imgDesc.imageType, destRowPitch, destSlicePitch, pSrcRegion); - auto allocationStruct = getAlignedAllocationData(this->device, false, dstPtr, bufferSize, false, false); - if (allocationStruct.alloc == nullptr) { + auto allocationStruct = getAlignedAllocationData(this->device, sharedSystemEnabled, dstPtr, bufferSize, false, false); + if (allocationStruct.alloc == nullptr && sharedSystemEnabled == false) { return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY; } memoryCopyParams.taskCountUpdateRequired |= CommandList::isExternalHostPtrAlloc(allocationStruct.alloc); + if ((allocationStruct.alloc == nullptr) && (NEO::debugManager.flags.EmitMemAdvisePriorToCopyForNonUsm.get() == 1)) { + appendMemAdvise(device, reinterpret_cast(allocationStruct.alignedAllocationPtr), bufferSize, static_cast(ZE_MEMORY_ADVICE_SET_SYSTEM_MEMORY_PREFERRED_LOCATION)); + } + DriverHandleImp *driverHandle = static_cast(device->getDriverHandle()); if (driverHandle->isRemoteImageNeeded(image, device)) { L0::Image *peerImage = nullptr; @@ -1043,7 +1053,7 @@ ze_result_t CommandListCoreFamily::appendImageCopyToMemoryExt(voi image = peerImage; } - memoryCopyParams.copyOffloadAllowed = isCopyOffloadAllowed(*image->getAllocation(), *allocationStruct.alloc); + memoryCopyParams.copyOffloadAllowed = isCopyOffloadAllowed(image->getAllocation(), allocationStruct.alloc); if (isCopyOnly(memoryCopyParams.copyOffloadAllowed)) { if ((bytesPerPixel == 3) || (bytesPerPixel == 6) || image->isMimickedImage()) { @@ -1051,7 +1061,7 @@ ze_result_t CommandListCoreFamily::appendImageCopyToMemoryExt(voi } size_t imgRowPitch = image->getImageInfo().rowPitch; size_t imgSlicePitch = image->getImageInfo().slicePitch; - auto status = appendCopyImageBlit(image->getAllocation(), allocationStruct.alloc, + auto status = appendCopyImageBlit(image->getAllocation()->getGpuAddress(), image->getAllocation(), allocationStruct.alignedAllocationPtr, allocationStruct.alloc, {pSrcRegion->originX, pSrcRegion->originY, pSrcRegion->originZ}, {0, 0, 0}, imgRowPitch, imgSlicePitch, destRowPitch, destSlicePitch, bytesPerPixel, {pSrcRegion->width, pSrcRegion->height, pSrcRegion->depth}, imgSize, {pSrcRegion->width, pSrcRegion->height, pSrcRegion->depth}, event, numWaitEvents, phWaitEvents, memoryCopyParams); @@ -1110,9 +1120,7 @@ ze_result_t CommandListCoreFamily::appendImageCopyToMemoryExt(voi Kernel *builtinKernel = device->getBuiltinFunctionsLib()->getImageFunction(builtInType); builtinKernel->setArgRedescribedImage(0u, image->toHandle(), false); - builtinKernel->setArgBufferWithAlloc(1u, allocationStruct.alignedAllocationPtr, - allocationStruct.alloc, - nullptr); + builtinSetArgCopy(builtinKernel, 1, reinterpret_cast(&allocationStruct.alignedAllocationPtr), allocationStruct.alloc); uint32_t origin[] = {pSrcRegion->originX, pSrcRegion->originY, pSrcRegion->originZ, @@ -1163,15 +1171,20 @@ ze_result_t CommandListCoreFamily::appendImageCopyToMemoryExt(voi ze_group_count_t kernelArgs{pSrcRegion->width / groupSizeX, pSrcRegion->height / groupSizeY, pSrcRegion->depth / groupSizeZ}; - auto dstAllocationType = allocationStruct.alloc->getAllocationType(); CmdListKernelLaunchParams launchParams = {}; launchParams.isBuiltInKernel = true; - launchParams.isDestinationAllocationInSystemMemory = - (dstAllocationType == NEO::AllocationType::bufferHostMemory) || - (dstAllocationType == NEO::AllocationType::externalHostPtr); + if (allocationStruct.alloc == nullptr) { + launchParams.isDestinationAllocationInSystemMemory = true; + launchParams.isDestinationAllocationImported = false; + } else { + auto dstAllocationType = allocationStruct.alloc->getAllocationType(); + launchParams.isDestinationAllocationInSystemMemory = + (dstAllocationType == NEO::AllocationType::bufferHostMemory) || + (dstAllocationType == NEO::AllocationType::externalHostPtr); - if constexpr (checkIfAllocationImportedRequired()) { - launchParams.isDestinationAllocationImported = this->isAllocationImported(allocationStruct.alloc, device->getDriverHandle()->getSvmAllocsManager()); + if constexpr (checkIfAllocationImportedRequired()) { + launchParams.isDestinationAllocationImported = this->isAllocationImported(allocationStruct.alloc, device->getDriverHandle()->getSvmAllocsManager()); + } } launchParams.relaxedOrderingDispatch = memoryCopyParams.relaxedOrderingDispatch; ret = CommandListCoreFamily::appendLaunchKernel(builtinKernel->toHandle(), kernelArgs, @@ -1254,7 +1267,7 @@ ze_result_t CommandListCoreFamily::appendImageCopyRegion(ze_image srcImage = peerImage; } - memoryCopyParams.copyOffloadAllowed = isCopyOffloadAllowed(*srcImage->getAllocation(), *dstImage->getAllocation()); + memoryCopyParams.copyOffloadAllowed = isCopyOffloadAllowed(srcImage->getAllocation(), dstImage->getAllocation()); if (isCopyOnly(memoryCopyParams.copyOffloadAllowed)) { auto bytesPerPixel = static_cast(srcImage->getImageInfo().surfaceFormat->imageElementSizeInBytes); @@ -1273,7 +1286,7 @@ ze_result_t CommandListCoreFamily::appendImageCopyRegion(ze_image auto dstSlicePitch = (dstImage->getImageInfo().imgDesc.imageType == NEO::ImageType::image1DArray ? 1 : dstRegion.height) * dstRowPitch; - auto status = appendCopyImageBlit(srcImage->getAllocation(), dstImage->getAllocation(), + auto status = appendCopyImageBlit(srcImage->getAllocation()->getGpuAddress(), srcImage->getAllocation(), dstImage->getAllocation()->getGpuAddress(), dstImage->getAllocation(), {srcRegion.originX, srcRegion.originY, srcRegion.originZ}, {dstRegion.originX, dstRegion.originY, dstRegion.originZ}, srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, bytesPerPixel, {srcRegion.width, srcRegion.height, srcRegion.depth}, srcImgSize, dstImgSize, event, numWaitEvents, phWaitEvents, memoryCopyParams); @@ -1649,7 +1662,9 @@ ze_result_t CommandListCoreFamily::appendMemoryCopyBlitRegion(Ali } template -ze_result_t CommandListCoreFamily::appendCopyImageBlit(NEO::GraphicsAllocation *src, +ze_result_t CommandListCoreFamily::appendCopyImageBlit(uintptr_t srcPtr, + NEO::GraphicsAllocation *src, + uintptr_t dstPtr, NEO::GraphicsAllocation *dst, const Vec3 &srcOffsets, const Vec3 &dstOffsets, size_t srcRowPitch, size_t srcSlicePitch, @@ -1671,9 +1686,9 @@ ze_result_t CommandListCoreFamily::appendCopyImageBlit(NEO::Graph auto clearColorAllocation = device->getNEODevice()->getDefaultEngine().commandStreamReceiver->getClearColorAllocation(); - auto blitProperties = NEO::BlitProperties::constructPropertiesForCopy(dst, src, - dstOffsets, srcOffsets, copySize, srcRowPitch, srcSlicePitch, - dstRowPitch, dstSlicePitch, clearColorAllocation); + auto blitProperties = NEO::BlitProperties::constructPropertiesForSystemCopy(dst, src, dstPtr, srcPtr, + dstOffsets, srcOffsets, copySize, srcRowPitch, srcSlicePitch, + dstRowPitch, dstSlicePitch, clearColorAllocation); blitProperties.computeStreamPartitionCount = this->partitionCount; blitProperties.highPriority = isHighPriorityImmediateCmdList(); blitProperties.bytesPerPixel = bytesPerPixel; @@ -1769,8 +1784,17 @@ ze_result_t CommandListCoreFamily::appendPageFaultCopy(NEO::Graph } template -bool CommandListCoreFamily::isCopyOffloadAllowed(const NEO::GraphicsAllocation &srcAllocation, const NEO::GraphicsAllocation &dstAllocation) const { - return !(srcAllocation.isAllocatedInLocalMemoryPool() && dstAllocation.isAllocatedInLocalMemoryPool()) && isCopyOffloadEnabled(); +bool CommandListCoreFamily::isCopyOffloadAllowed(const NEO::GraphicsAllocation *srcAllocation, const NEO::GraphicsAllocation *dstAllocation) const { + if (srcAllocation == nullptr || dstAllocation == nullptr) { + return isCopyOffloadEnabled(); + } + return !(srcAllocation->isAllocatedInLocalMemoryPool() && dstAllocation->isAllocatedInLocalMemoryPool()) && isCopyOffloadEnabled(); +} + +template +bool CommandListCoreFamily::isSharedSystemEnabled() const { + NEO::Device *neoDevice = device->getNEODevice(); + return neoDevice->areSharedSystemAllocationsAllowed() && (NEO::debugManager.flags.TreatNonUsmForTransfersAsSharedSystem.get() == 1); } template @@ -1783,7 +1807,7 @@ ze_result_t CommandListCoreFamily::appendMemoryCopy(void *dstptr, CmdListMemoryCopyParams &memoryCopyParams) { NEO::Device *neoDevice = device->getNEODevice(); - bool sharedSystemEnabled = ((neoDevice->areSharedSystemAllocationsAllowed()) && (NEO::debugManager.flags.TreatNonUsmForTransfersAsSharedSystem.get() == 1)); + bool sharedSystemEnabled = isSharedSystemEnabled(); uint32_t callId = 0; if (NEO::debugManager.flags.EnableSWTags.get()) { @@ -1815,11 +1839,8 @@ ze_result_t CommandListCoreFamily::appendMemoryCopy(void *dstptr, appendMemAdvise(device, reinterpret_cast(srcAllocationStruct.alignedAllocationPtr), size, static_cast(ZE_MEMORY_ADVICE_SET_SYSTEM_MEMORY_PREFERRED_LOCATION)); } - if (dstAllocationStruct.alloc == nullptr || srcAllocationStruct.alloc == nullptr) { - memoryCopyParams.copyOffloadAllowed = true; - } else { - memoryCopyParams.copyOffloadAllowed = isCopyOffloadAllowed(*srcAllocationStruct.alloc, *dstAllocationStruct.alloc); - } + memoryCopyParams.copyOffloadAllowed = isCopyOffloadAllowed(srcAllocationStruct.alloc, dstAllocationStruct.alloc); + const bool isCopyOnlyEnabled = isCopyOnly(memoryCopyParams.copyOffloadAllowed); const bool inOrderCopyOnlySignalingAllowed = this->isInOrderExecutionEnabled() && !memoryCopyParams.forceDisableCopyOnlyInOrderSignaling && isCopyOnlyEnabled; @@ -2065,7 +2086,7 @@ ze_result_t CommandListCoreFamily::appendMemoryCopyRegion(void *d CommandList::isExternalHostPtrAlloc(srcAllocationStruct.alloc); } - memoryCopyParams.copyOffloadAllowed = isCopyOffloadAllowed(*srcAllocationStruct.alloc, *dstAllocationStruct.alloc); + memoryCopyParams.copyOffloadAllowed = isCopyOffloadAllowed(srcAllocationStruct.alloc, dstAllocationStruct.alloc); const bool isCopyOnlyEnabled = isCopyOnly(memoryCopyParams.copyOffloadAllowed); const bool inOrderCopyOnlySignalingAllowed = this->isInOrderExecutionEnabled() && !memoryCopyParams.forceDisableCopyOnlyInOrderSignaling && isCopyOnlyEnabled; @@ -2356,7 +2377,7 @@ ze_result_t CommandListCoreFamily::appendMemoryFill(void *ptr, memoryCopyParams.copyOffloadAllowed = isCopyOffloadEnabled() && (patternSize <= this->maxFillPatternSizeForCopyEngine); NEO::Device *neoDevice = device->getNEODevice(); - bool sharedSystemEnabled = ((neoDevice->areSharedSystemAllocationsAllowed()) && (NEO::debugManager.flags.TreatNonUsmForTransfersAsSharedSystem.get() == 1)); + bool sharedSystemEnabled = isSharedSystemEnabled(); uint32_t callId = 0; if (NEO::debugManager.flags.EnableSWTags.get()) { callId = neoDevice->getRootDeviceEnvironment().tagsManager->incrementAndGetCurrentCallCount(); diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_2.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_2.cpp index 41e17b0a84..d36563c42c 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_2.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_2.cpp @@ -141,8 +141,8 @@ class MockCommandListHw : public WhiteBox<::L0::CommandListCoreFamily &srcOffsets, const Vec3 &dstOffsets, size_t srcRowPitch, size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, @@ -1682,6 +1682,328 @@ HWTEST_F(CommandListAppendMemoryCopyBlit, whenAppendMemoryCopyBlitIsAppendedAndN EXPECT_NE(firstBatchBufferAllocation, secondBatchBufferAllocation); } +struct CommandListAppendImage : Test { + CmdListMemoryCopyParams copyParams = {}; +}; + +HWTEST2_F(CommandListAppendImage, givenCommandListWhenAppendImageCopyFromMemoryIsCalledWithNullGraphicsAllocationAndSharedSystemNotEnabledThenErrorIsReturned, IsAtLeastXe2HpgCore) { + DebugManagerStateRestore restore; + debugManager.flags.EmitMemAdvisePriorToCopyForNonUsm.set(1); + + auto commandList = std::make_unique>(); + commandList->failAlignedAlloc = true; + commandList->initialize(device, NEO::EngineGroupType::copy, 0u); + ASSERT_EQ(ZE_RESULT_SUCCESS, commandList->initialize(device, NEO::EngineGroupType::copy, 0u)); + + void *srcPtr = reinterpret_cast(0x1234); + + ze_image_desc_t zeDesc = {}; + zeDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + auto image = std::make_unique>>(); + image->initialize(device, &zeDesc); + + EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY, commandList->appendImageCopyFromMemory(image->toHandle(), srcPtr, nullptr, nullptr, 0, nullptr, copyParams)); +} + +HWTEST2_F(CommandListAppendImage, givenCommandListWhenAppendImageCopyFromMemoryIsCalledWithNullGraphicsAllocationAndSharedSystemEnabledThenSuccessIsReturned, IsAtLeastXe2HpgCore) { + DebugManagerStateRestore restore; + debugManager.flags.EnableSharedSystemUsmSupport.set(1); + debugManager.flags.TreatNonUsmForTransfersAsSharedSystem.set(1); + debugManager.flags.EmitMemAdvisePriorToCopyForNonUsm.set(1); + + auto commandList = std::make_unique>(); + commandList->failAlignedAlloc = true; + commandList->initialize(device, NEO::EngineGroupType::copy, 0u); + ASSERT_EQ(ZE_RESULT_SUCCESS, commandList->initialize(device, NEO::EngineGroupType::copy, 0u)); + + auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo(); + VariableBackup sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities}; + sharedSystemMemCapabilities = 0xf; + + void *srcPtr = reinterpret_cast(0x1234); + + ze_image_desc_t zeDesc = {}; + zeDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + auto image = std::make_unique>>(); + image->initialize(device, &zeDesc); + + EXPECT_EQ(ZE_RESULT_SUCCESS, commandList->appendImageCopyFromMemory(image->toHandle(), srcPtr, nullptr, nullptr, 0, nullptr, copyParams)); +} + +HWTEST2_F(CommandListAppendImage, givenCommandListWhenAppendImageCopyFromMemoryIsCalledWithValidGraphicsAllocationThenSuccessIsReturned, IsAtLeastXe2HpgCore) { + DebugManagerStateRestore restore; + debugManager.flags.EmitMemAdvisePriorToCopyForNonUsm.set(1); + + auto commandList = std::make_unique>(); + commandList->failAlignedAlloc = false; + commandList->initialize(device, NEO::EngineGroupType::copy, 0u); + ASSERT_EQ(ZE_RESULT_SUCCESS, commandList->initialize(device, NEO::EngineGroupType::copy, 0u)); + + void *srcPtr = reinterpret_cast(0x1234); + + ze_image_desc_t zeDesc = {}; + zeDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + auto image = std::make_unique>>(); + image->initialize(device, &zeDesc); + + EXPECT_EQ(ZE_RESULT_SUCCESS, commandList->appendImageCopyFromMemory(image->toHandle(), srcPtr, nullptr, nullptr, 0, nullptr, copyParams)); +} + +HWTEST2_F(CommandListAppendImage, givenCommandListWhenAppendImageCopyToMemoryIsCalledWithNullGraphicsAllocationAndSharedSystemNotEnabledThenErrorIsReturned, IsAtLeastXe2HpgCore) { + DebugManagerStateRestore restore; + debugManager.flags.EmitMemAdvisePriorToCopyForNonUsm.set(1); + + auto commandList = std::make_unique>(); + commandList->failAlignedAlloc = true; + commandList->initialize(device, NEO::EngineGroupType::copy, 0u); + ASSERT_EQ(ZE_RESULT_SUCCESS, commandList->initialize(device, NEO::EngineGroupType::copy, 0u)); + + ze_image_desc_t zeDesc = {}; + zeDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + auto image = std::make_unique>>(); + image->initialize(device, &zeDesc); + + void *dstPtr = reinterpret_cast(0x1234); + + EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY, commandList->appendImageCopyToMemory(dstPtr, image->toHandle(), nullptr, nullptr, 0, nullptr, copyParams)); +} + +HWTEST2_F(CommandListAppendImage, givenCommandListWhenAppendImageCopyToMemoryIsCalledWithNullGraphicsAllocationAndSharedSystemEnabledThenSuccessIsReturned, IsAtLeastXe2HpgCore) { + DebugManagerStateRestore restore; + debugManager.flags.EnableSharedSystemUsmSupport.set(1); + debugManager.flags.TreatNonUsmForTransfersAsSharedSystem.set(1); + debugManager.flags.EmitMemAdvisePriorToCopyForNonUsm.set(1); + + auto commandList = std::make_unique>(); + commandList->failAlignedAlloc = true; + commandList->initialize(device, NEO::EngineGroupType::copy, 0u); + ASSERT_EQ(ZE_RESULT_SUCCESS, commandList->initialize(device, NEO::EngineGroupType::copy, 0u)); + + auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo(); + VariableBackup sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities}; + sharedSystemMemCapabilities = 0xf; + + ze_image_desc_t zeDesc = {}; + zeDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + auto image = std::make_unique>>(); + image->initialize(device, &zeDesc); + + void *dstPtr = reinterpret_cast(0x1234); + + EXPECT_EQ(ZE_RESULT_SUCCESS, commandList->appendImageCopyToMemory(dstPtr, image->toHandle(), nullptr, nullptr, 0, nullptr, copyParams)); +} + +HWTEST2_F(CommandListAppendImage, givenCommandListWhenAppendImageCopyToMemoryIsCalledWithValidGraphicsAllocationThenSuccessIsReturned, IsAtLeastXe2HpgCore) { + DebugManagerStateRestore restore; + debugManager.flags.EmitMemAdvisePriorToCopyForNonUsm.set(1); + + auto commandList = std::make_unique>(); + commandList->failAlignedAlloc = false; + commandList->initialize(device, NEO::EngineGroupType::copy, 0u); + ASSERT_EQ(ZE_RESULT_SUCCESS, commandList->initialize(device, NEO::EngineGroupType::copy, 0u)); + + ze_image_desc_t zeDesc = {}; + zeDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + auto image = std::make_unique>>(); + image->initialize(device, &zeDesc); + + void *dstPtr = reinterpret_cast(0x1234); + + EXPECT_EQ(ZE_RESULT_SUCCESS, commandList->appendImageCopyToMemory(dstPtr, image->toHandle(), nullptr, nullptr, 0, nullptr, copyParams)); +} + +HWTEST2_F(CommandListAppendImage, givenCopyCommandListWhenAppendImageCopyFromMemoryIsCalledWithSharedSystemAndEmitMemAdviseEnabledThenSuccessIsReturned, IsAtLeastXe2HpgCore) { + DebugManagerStateRestore restore; + debugManager.flags.EnableSharedSystemUsmSupport.set(1); + debugManager.flags.TreatNonUsmForTransfersAsSharedSystem.set(1); + debugManager.flags.EmitMemAdvisePriorToCopyForNonUsm.set(1); + + auto commandList = std::make_unique>>(); + ASSERT_EQ(ZE_RESULT_SUCCESS, commandList->initialize(device, NEO::EngineGroupType::copy, 0u)); + + auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo(); + VariableBackup sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities}; + sharedSystemMemCapabilities = 0xf; + + void *srcPtr = reinterpret_cast(0x1234); + + ze_image_desc_t zeDesc = {}; + zeDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + zeDesc.type = ZE_IMAGE_TYPE_3D; + zeDesc.width = 4; + zeDesc.height = 2; + zeDesc.depth = 2; + + L0::Image *imagePtr = nullptr; + ASSERT_EQ(ZE_RESULT_SUCCESS, L0::Image::create(device->getNEODevice()->getHardwareInfo().platform.eProductFamily, device, &zeDesc, &imagePtr)); + std::unique_ptr image(imagePtr); + ze_image_region_t dstImgRegion = {2, 1, 1, 4, 2, 2}; + + EXPECT_EQ(ZE_RESULT_SUCCESS, commandList->appendImageCopyFromMemory(image->toHandle(), srcPtr, &dstImgRegion, nullptr, 0, nullptr, copyParams)); +} + +HWTEST2_F(CommandListAppendImage, givenCopyCommandListWhenAppendImageCopyFromMemoryIsCalledWithSharedSystemEnabledAndEmitMemAdviseNotEnabledThenSuccessIsReturned, IsAtLeastXe2HpgCore) { + DebugManagerStateRestore restore; + debugManager.flags.EnableSharedSystemUsmSupport.set(1); + debugManager.flags.TreatNonUsmForTransfersAsSharedSystem.set(1); + debugManager.flags.EmitMemAdvisePriorToCopyForNonUsm.set(-1); + + auto commandList = std::make_unique>>(); + ASSERT_EQ(ZE_RESULT_SUCCESS, commandList->initialize(device, NEO::EngineGroupType::copy, 0u)); + + auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo(); + VariableBackup sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities}; + sharedSystemMemCapabilities = 0xf; + + void *srcPtr = reinterpret_cast(0x1234); + + ze_image_desc_t zeDesc = {}; + zeDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + zeDesc.type = ZE_IMAGE_TYPE_3D; + zeDesc.width = 4; + zeDesc.height = 2; + zeDesc.depth = 2; + + L0::Image *imagePtr = nullptr; + ASSERT_EQ(ZE_RESULT_SUCCESS, L0::Image::create(device->getNEODevice()->getHardwareInfo().platform.eProductFamily, device, &zeDesc, &imagePtr)); + std::unique_ptr image(imagePtr); + ze_image_region_t dstImgRegion = {2, 1, 1, 4, 2, 2}; + + EXPECT_EQ(ZE_RESULT_SUCCESS, commandList->appendImageCopyFromMemory(image->toHandle(), srcPtr, &dstImgRegion, nullptr, 0, nullptr, copyParams)); +} + +HWTEST2_F(CommandListAppendImage, givenComputeCommandListWhenAppendImageCopyFromMemoryIsCalledWithSharedSystemEnabledThenSuccessIsReturned, IsAtLeastXe2HpgCore) { + DebugManagerStateRestore restore; + debugManager.flags.EnableSharedSystemUsmSupport.set(1); + debugManager.flags.TreatNonUsmForTransfersAsSharedSystem.set(1); + debugManager.flags.EmitMemAdvisePriorToCopyForNonUsm.set(1); + + auto kernel = device->getBuiltinFunctionsLib()->getImageFunction(ImageBuiltin::copyBufferToImage3dBytes); + auto mockBuiltinKernel = static_cast *>(kernel); + mockBuiltinKernel->setArgRedescribedImageCallBase = false; + + auto commandList = std::make_unique>>(); + ASSERT_EQ(ZE_RESULT_SUCCESS, commandList->initialize(device, NEO::EngineGroupType::renderCompute, 0u)); + + auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo(); + VariableBackup sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities}; + sharedSystemMemCapabilities = 0xf; + + void *srcPtr = reinterpret_cast(0x1234); + + ze_image_desc_t zeDesc = {}; + zeDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + zeDesc.type = ZE_IMAGE_TYPE_3D; + zeDesc.width = 4; + zeDesc.height = 2; + zeDesc.depth = 2; + + L0::Image *imagePtr = nullptr; + ASSERT_EQ(ZE_RESULT_SUCCESS, L0::Image::create(device->getNEODevice()->getHardwareInfo().platform.eProductFamily, device, &zeDesc, &imagePtr)); + std::unique_ptr image(imagePtr); + ze_image_region_t dstImgRegion = {2, 1, 1, 4, 2, 2}; + + EXPECT_EQ(ZE_RESULT_SUCCESS, commandList->appendImageCopyFromMemory(image->toHandle(), srcPtr, &dstImgRegion, nullptr, 0, nullptr, copyParams)); +} + +HWTEST2_F(CommandListAppendImage, givenCopyCommandListWhenAppendImageCopyToMemoryIsCalledWithSharedSystemAndEmitMemAdviseEnabledThenSuccessIsReturned, IsAtLeastXe2HpgCore) { + DebugManagerStateRestore restore; + debugManager.flags.EnableSharedSystemUsmSupport.set(1); + debugManager.flags.TreatNonUsmForTransfersAsSharedSystem.set(1); + debugManager.flags.EmitMemAdvisePriorToCopyForNonUsm.set(1); + + auto commandList = std::make_unique>>(); + ASSERT_EQ(ZE_RESULT_SUCCESS, commandList->initialize(device, NEO::EngineGroupType::copy, 0u)); + + auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo(); + VariableBackup sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities}; + sharedSystemMemCapabilities = 0xf; + + ze_image_desc_t zeDesc = {}; + zeDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + zeDesc.type = ZE_IMAGE_TYPE_3D; + zeDesc.width = 4; + zeDesc.height = 2; + zeDesc.depth = 2; + + L0::Image *imagePtr = nullptr; + auto result = L0::Image::create(device->getNEODevice()->getHardwareInfo().platform.eProductFamily, device, &zeDesc, &imagePtr); + ASSERT_EQ(ZE_RESULT_SUCCESS, result); + std::unique_ptr image(imagePtr); + ze_image_region_t srcImgRegion = {2, 1, 1, 4, 2, 2}; + + void *dstPtr = reinterpret_cast(0x1234); + + EXPECT_EQ(ZE_RESULT_SUCCESS, commandList->appendImageCopyToMemory(dstPtr, image->toHandle(), &srcImgRegion, nullptr, 0, nullptr, copyParams)); +} + +HWTEST2_F(CommandListAppendImage, givenCopyCommandListWhenAppendImageCopyToMemoryIsCalledWithSharedSystemEnabledAndEmitMemAdviseNotEnabledThenSuccessIsReturned, IsAtLeastXe2HpgCore) { + DebugManagerStateRestore restore; + + debugManager.flags.EnableSharedSystemUsmSupport.set(1); + debugManager.flags.TreatNonUsmForTransfersAsSharedSystem.set(1); + debugManager.flags.EmitMemAdvisePriorToCopyForNonUsm.set(-1); + + auto commandList = std::make_unique>>(); + ASSERT_EQ(ZE_RESULT_SUCCESS, commandList->initialize(device, NEO::EngineGroupType::copy, 0u)); + + auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo(); + VariableBackup sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities}; + sharedSystemMemCapabilities = 0xf; + + ze_image_desc_t zeDesc = {}; + zeDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + zeDesc.type = ZE_IMAGE_TYPE_3D; + zeDesc.width = 4; + zeDesc.height = 2; + zeDesc.depth = 2; + + L0::Image *imagePtr = nullptr; + auto result = L0::Image::create(device->getNEODevice()->getHardwareInfo().platform.eProductFamily, device, &zeDesc, &imagePtr); + ASSERT_EQ(ZE_RESULT_SUCCESS, result); + std::unique_ptr image(imagePtr); + ze_image_region_t srcImgRegion = {2, 1, 1, 4, 2, 2}; + + void *dstPtr = reinterpret_cast(0x1234); + + EXPECT_EQ(ZE_RESULT_SUCCESS, commandList->appendImageCopyToMemory(dstPtr, image->toHandle(), &srcImgRegion, nullptr, 0, nullptr, copyParams)); +} + +HWTEST2_F(CommandListAppendImage, givenComputeCommandListWhenAppendImageCopyToMemoryIsCalledWithSharedSystemEnabledThenSuccessIsReturned, IsAtLeastXe2HpgCore) { + DebugManagerStateRestore restore; + debugManager.flags.EnableSharedSystemUsmSupport.set(1); + debugManager.flags.TreatNonUsmForTransfersAsSharedSystem.set(1); + debugManager.flags.EmitMemAdvisePriorToCopyForNonUsm.set(1); + + auto kernel = device->getBuiltinFunctionsLib()->getImageFunction(ImageBuiltin::copyImage3dToBufferBytes); + auto mockBuiltinKernel = static_cast *>(kernel); + mockBuiltinKernel->setArgRedescribedImageCallBase = false; + + auto commandList = std::make_unique>>(); + ASSERT_EQ(ZE_RESULT_SUCCESS, commandList->initialize(device, NEO::EngineGroupType::renderCompute, 0u)); + + auto &hwInfo = *device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo(); + VariableBackup sharedSystemMemCapabilities{&hwInfo.capabilityTable.sharedSystemMemCapabilities}; + sharedSystemMemCapabilities = 0xf; + + ze_image_desc_t zeDesc = {}; + zeDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + zeDesc.type = ZE_IMAGE_TYPE_3D; + zeDesc.width = 4; + zeDesc.height = 2; + zeDesc.depth = 2; + + L0::Image *imagePtr = nullptr; + auto result = L0::Image::create(device->getNEODevice()->getHardwareInfo().platform.eProductFamily, device, &zeDesc, &imagePtr); + ASSERT_EQ(ZE_RESULT_SUCCESS, result); + std::unique_ptr image(imagePtr); + ze_image_region_t srcImgRegion = {2, 1, 1, 4, 2, 2}; + + void *dstPtr = reinterpret_cast(0x1234); + + EXPECT_EQ(ZE_RESULT_SUCCESS, commandList->appendImageCopyToMemory(dstPtr, image->toHandle(), &srcImgRegion, nullptr, 0, nullptr, copyParams)); +} + template struct MockL0GfxCoreHelperSupportsCmdListHeapSharingHw : L0::L0GfxCoreHelperHw { bool platformSupportsCmdListHeapSharing() const override { return true; } diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_blit.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_blit.cpp index 523535b9b0..5343913f6a 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_blit.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_blit.cpp @@ -334,7 +334,7 @@ HWTEST_F(AppendMemoryCopyTests, givenCopyCommandListWhenTimestampPassedToImageCo MemoryPool::system4KBPages, MemoryManager::maxOsContextCount); CmdListMemoryCopyParams copyParams = {}; - commandList->appendCopyImageBlit(&mockAllocationDst, &mockAllocationSrc, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, event.get(), 0, nullptr, copyParams); + commandList->appendCopyImageBlit(mockAllocationSrc.getGpuAddress(), &mockAllocationSrc, mockAllocationDst.getGpuAddress(), &mockAllocationDst, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, event.get(), 0, nullptr, copyParams); GenCmdList cmdList; ASSERT_TRUE(FamilyType::Parse::parseCommandBuffer( cmdList, ptrOffset(commandList->getCmdContainer().getCommandStream()->getCpuBase(), 0), commandList->getCmdContainer().getCommandStream()->getUsed())); @@ -372,7 +372,7 @@ HWTEST2_F(AppendMemoryCopyTests, givenWaitWhenWhenAppendBlitCalledThenProgramSem CmdListMemoryCopyParams copyParams = {}; { - commandList->appendCopyImageBlit(&mockAllocationDst, &mockAllocationSrc, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, event.get(), 0, nullptr, copyParams); + commandList->appendCopyImageBlit(mockAllocationSrc.getGpuAddress(), &mockAllocationSrc, mockAllocationDst.getGpuAddress(), &mockAllocationDst, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, event.get(), 0, nullptr, copyParams); GenCmdList cmdList; ASSERT_TRUE(FamilyType::Parse::parseCommandBuffer(cmdList, ptrOffset(cmdStream->getCpuBase(), offset), cmdStream->getUsed() - offset)); @@ -384,7 +384,7 @@ HWTEST2_F(AppendMemoryCopyTests, givenWaitWhenWhenAppendBlitCalledThenProgramSem offset = cmdStream->getUsed(); auto eventHandle = event->toHandle(); - commandList->appendCopyImageBlit(&mockAllocationDst, &mockAllocationSrc, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, nullptr, 1, &eventHandle, copyParams); + commandList->appendCopyImageBlit(mockAllocationSrc.getGpuAddress(), &mockAllocationSrc, mockAllocationDst.getGpuAddress(), &mockAllocationDst, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, nullptr, 1, &eventHandle, copyParams); GenCmdList cmdList; ASSERT_TRUE(FamilyType::Parse::parseCommandBuffer(cmdList, ptrOffset(cmdStream->getCpuBase(), offset), cmdStream->getUsed() - offset)); auto itor = find(cmdList.begin(), cmdList.end()); @@ -455,7 +455,7 @@ HWTEST2_F(AppendMemoryCopyTests, givenCopyCommandListWhenTiled1DArrayImagePassed size_t arrayLevels = 8; size_t depth = 1; CmdListMemoryCopyParams copyParams = {}; - commandList->appendCopyImageBlit(&mockAllocationDst, &mockAllocationSrc, {0, 0, 0}, {0, 0, 0}, 4, 4, 4, 4, 1, {1, arrayLevels, depth}, {1, arrayLevels, depth}, {1, arrayLevels, depth}, nullptr, 0, nullptr, copyParams); + commandList->appendCopyImageBlit(mockAllocationSrc.getGpuAddress(), &mockAllocationSrc, mockAllocationDst.getGpuAddress(), &mockAllocationDst, {0, 0, 0}, {0, 0, 0}, 4, 4, 4, 4, 1, {1, arrayLevels, depth}, {1, arrayLevels, depth}, {1, arrayLevels, depth}, nullptr, 0, nullptr, copyParams); GenCmdList cmdList; ASSERT_TRUE(FamilyType::Parse::parseCommandBuffer( cmdList, ptrOffset(commandList->getCmdContainer().getCommandStream()->getCpuBase(), 0), commandList->getCmdContainer().getCommandStream()->getUsed())); @@ -490,7 +490,7 @@ HWTEST2_F(AppendMemoryCopyTests, givenCopyCommandListWhenNotTiled1DArrayImagePas size_t arrayLevels = 8; size_t depth = 1; CmdListMemoryCopyParams copyParams = {}; - commandList->appendCopyImageBlit(&mockAllocationDst, &mockAllocationSrc, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, arrayLevels, depth}, {1, arrayLevels, depth}, {1, arrayLevels, depth}, nullptr, 0, nullptr, copyParams); + commandList->appendCopyImageBlit(mockAllocationSrc.getGpuAddress(), &mockAllocationSrc, mockAllocationDst.getGpuAddress(), &mockAllocationDst, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, arrayLevels, depth}, {1, arrayLevels, depth}, {1, arrayLevels, depth}, nullptr, 0, nullptr, copyParams); GenCmdList cmdList; ASSERT_TRUE(FamilyType::Parse::parseCommandBuffer( cmdList, ptrOffset(commandList->getCmdContainer().getCommandStream()->getCpuBase(), 0), commandList->getCmdContainer().getCommandStream()->getUsed())); @@ -637,7 +637,7 @@ HWTEST_F(AppendMemoryCopyTests, givenCopyOnlyCommandListWithUseAdditionalBlitPro EXPECT_EQ(0u, commandList->additionalBlitPropertiesCalled); EXPECT_EQ(0u, commandList->inOrderPatchCmds.size()); CmdListMemoryCopyParams copyParams = {}; - commandList->appendCopyImageBlit(&mockAllocationDst, &mockAllocationSrc, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, nullptr, 0, nullptr, copyParams); + commandList->appendCopyImageBlit(mockAllocationSrc.getGpuAddress(), &mockAllocationSrc, mockAllocationDst.getGpuAddress(), &mockAllocationDst, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, nullptr, 0, nullptr, copyParams); EXPECT_EQ(0u, commandList->additionalBlitPropertiesCalled); EXPECT_EQ(1u, commandList->appendSignalInOrderDependencyCounterCalled); @@ -649,7 +649,7 @@ HWTEST_F(AppendMemoryCopyTests, givenCopyOnlyCommandListWithUseAdditionalBlitPro commandList->useAdditionalBlitProperties = true; - commandList->appendCopyImageBlit(&mockAllocationDst, &mockAllocationSrc, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, nullptr, 0, nullptr, copyParams); + commandList->appendCopyImageBlit(mockAllocationSrc.getGpuAddress(), &mockAllocationSrc, mockAllocationDst.getGpuAddress(), &mockAllocationDst, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, nullptr, 0, nullptr, copyParams); EXPECT_EQ(1u, commandList->additionalBlitPropertiesCalled); EXPECT_EQ(1u, commandList->appendSignalInOrderDependencyCounterCalled); if (commandList->inOrderCmdsPatchingEnabled()) { @@ -674,7 +674,7 @@ HWTEST_F(AppendMemoryCopyTests, givenCopyOnlyCommandListWithUseAdditionalBlitPro EXPECT_EQ(0u, commandList->additionalBlitPropertiesCalled); EXPECT_EQ(0u, commandList->inOrderPatchCmds.size()); CmdListMemoryCopyParams copyParams = {}; - commandList->appendCopyImageBlit(&mockAllocationDst, &mockAllocationSrc, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, nullptr, 0, nullptr, copyParams); + commandList->appendCopyImageBlit(mockAllocationSrc.getGpuAddress(), &mockAllocationSrc, mockAllocationDst.getGpuAddress(), &mockAllocationDst, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, nullptr, 0, nullptr, copyParams); EXPECT_EQ(1u, commandList->additionalBlitPropertiesCalled); EXPECT_EQ(0u, commandList->inOrderPatchCmds.size()); } @@ -885,7 +885,7 @@ HWTEST_F(AppendMemoryCopyTests, givenCopyOnlyCommandListWithUseAdditionalBlitPro EXPECT_EQ(0u, commandList->appendSignalInOrderDependencyCounterCalled); EXPECT_EQ(0u, commandList->inOrderPatchCmds.size()); CmdListMemoryCopyParams copyParams = {}; - commandList->appendCopyImageBlit(&mockAllocationDst, &mockAllocationSrc, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, nullptr, 0, nullptr, copyParams); + commandList->appendCopyImageBlit(mockAllocationSrc.getGpuAddress(), &mockAllocationSrc, mockAllocationDst.getGpuAddress(), &mockAllocationDst, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, nullptr, 0, nullptr, copyParams); EXPECT_EQ(1u, commandList->additionalBlitPropertiesCalled); EXPECT_EQ(0u, commandList->appendSignalInOrderDependencyCounterCalled); if (commandList->inOrderCmdsPatchingEnabled()) { @@ -907,6 +907,18 @@ HWTEST_F(AppendMemoryCopyTests, givenCopyOnlyCommandListWithUseAdditionalBlitPro } } +HWTEST_F(AppendMemoryCopyTests, givenCopyOnlyCommandListWithNonZeroNumWaitEventsAndNullEventHandlesWhenCallingAppendCopyImageBlitThenErrorIsReturned) { + auto commandList = std::make_unique>>(); + commandList->initialize(device, NEO::EngineGroupType::copy, 0u); + NEO::MockGraphicsAllocation mockAllocationSrc(0, 1u /*num gmms*/, NEO::AllocationType::buffer, reinterpret_cast(0x1234), 0x1000, 0, sizeof(uint32_t), MemoryPool::system4KBPages, MemoryManager::maxOsContextCount); + NEO::MockGraphicsAllocation mockAllocationDst(0, 1u /*num gmms*/, NEO::AllocationType::image, reinterpret_cast(0x2345), 0x1000, 0, sizeof(uint32_t), MemoryPool::system4KBPages, MemoryManager::maxOsContextCount); + + CmdListMemoryCopyParams copyParams = {}; + auto result = commandList->appendCopyImageBlit(mockAllocationSrc.getGpuAddress(), &mockAllocationSrc, mockAllocationDst.getGpuAddress(), &mockAllocationDst, {0, 0, 0}, {0, 0, 0}, 1, 1, 1, 1, 1, {1, 1, 1}, {1, 1, 1}, {1, 1, 1}, nullptr, 1, nullptr, copyParams); + + EXPECT_EQ(result, ZE_RESULT_ERROR_INVALID_ARGUMENT); +} + HWTEST_F(AppendMemoryCopyTests, givenCopyOnlyCommandListWithUseAdditionalBlitPropertiesWhenPatchingCommandsAfterCallingMemoryFillWithTwoBytesPatternThenCommandsRemainsTheSame) { auto commandList = std::make_unique>(); commandList->initialize(device, NEO::EngineGroupType::copy, ZE_COMMAND_LIST_FLAG_IN_ORDER); diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_memory_extension.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_memory_extension.cpp index f53a8d5ed4..fa0f4dbeec 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_memory_extension.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_memory_extension.cpp @@ -183,8 +183,8 @@ class MockCommandListExtensionHw : public WhiteBox<::L0::CommandListCoreFamily &srcOffsets, const Vec3 &dstOffsets, size_t srcRowPitch, size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, diff --git a/shared/source/command_stream/command_stream_receiver_hw_base.inl b/shared/source/command_stream/command_stream_receiver_hw_base.inl index 69e059dba1..e5186d38a0 100644 --- a/shared/source/command_stream/command_stream_receiver_hw_base.inl +++ b/shared/source/command_stream/command_stream_receiver_hw_base.inl @@ -1086,8 +1086,8 @@ TaskCountType CommandStreamReceiverHw::flushBcsTask(const BlitPropert if (blitProperties.blitSyncProperties.outputTimestampPacket) { bool deviceToHostPostSyncFenceRequired = getProductHelper().isDeviceToHostCopySignalingFenceRequired() && - !blitProperties.dstAllocation->isAllocatedInLocalMemoryPool() && - blitProperties.srcAllocation->isAllocatedInLocalMemoryPool(); + (blitProperties.dstAllocation && !blitProperties.dstAllocation->isAllocatedInLocalMemoryPool()) && + (blitProperties.srcAllocation && blitProperties.srcAllocation->isAllocatedInLocalMemoryPool()); if (deviceToHostPostSyncFenceRequired) { MemorySynchronizationCommands::addAdditionalSynchronization(commandStream, tagAllocation->getGpuAddress(), NEO::FenceType::release, peekRootDeviceEnvironment()); @@ -1106,8 +1106,12 @@ TaskCountType CommandStreamReceiverHw::flushBcsTask(const BlitPropert } blitProperties.csrDependencies.makeResident(*this); - blitProperties.srcAllocation->prepareHostPtrForResidency(this); - blitProperties.dstAllocation->prepareHostPtrForResidency(this); + if (blitProperties.srcAllocation) { + blitProperties.srcAllocation->prepareHostPtrForResidency(this); + } + if (blitProperties.dstAllocation) { + blitProperties.dstAllocation->prepareHostPtrForResidency(this); + } makeResident(*blitProperties.srcAllocation); makeResident(*blitProperties.dstAllocation); if (blitProperties.clearColorAllocation) { diff --git a/shared/source/gen12lp/command_stream_receiver_hw_gen12lp.cpp b/shared/source/gen12lp/command_stream_receiver_hw_gen12lp.cpp index a17e51c756..fe0c2c7f15 100644 --- a/shared/source/gen12lp/command_stream_receiver_hw_gen12lp.cpp +++ b/shared/source/gen12lp/command_stream_receiver_hw_gen12lp.cpp @@ -291,12 +291,14 @@ void BlitCommandsHelper::appendBlitCommandsForImages(const BlitPropertie auto dstRowPitch = static_cast(blitProperties.dstRowPitch); uint32_t mipTailLod = 0; auto compressionDetails = 0u; - - getBlitAllocationProperties(*srcAllocation, srcRowPitch, srcQPitch, tileType, mipTailLod, compressionDetails, - rootDeviceEnvironment, blitProperties.srcPlane); - getBlitAllocationProperties(*dstAllocation, dstRowPitch, dstQPitch, tileType, mipTailLod, compressionDetails, - rootDeviceEnvironment, blitProperties.dstPlane); - + if (srcAllocation) { + getBlitAllocationProperties(*srcAllocation, srcRowPitch, srcQPitch, tileType, mipTailLod, compressionDetails, + rootDeviceEnvironment, blitProperties.srcPlane); + } + if (dstAllocation) { + getBlitAllocationProperties(*dstAllocation, dstRowPitch, dstQPitch, tileType, mipTailLod, compressionDetails, + rootDeviceEnvironment, blitProperties.dstPlane); + } blitCmd.setSourcePitch(srcRowPitch); blitCmd.setDestinationPitch(dstRowPitch); diff --git a/shared/source/helpers/blit_commands_helper_xehp_and_later.inl b/shared/source/helpers/blit_commands_helper_xehp_and_later.inl index fcaa28c850..8749ef4969 100644 --- a/shared/source/helpers/blit_commands_helper_xehp_and_later.inl +++ b/shared/source/helpers/blit_commands_helper_xehp_and_later.inl @@ -80,7 +80,7 @@ template void BlitCommandsHelper::appendSurfaceType(const BlitProperties &blitProperties, typename GfxFamily::XY_BLOCK_COPY_BLT &blitCmd) { using XY_BLOCK_COPY_BLT = typename GfxFamily::XY_BLOCK_COPY_BLT; - if (blitProperties.srcAllocation->getDefaultGmm()) { + if (blitProperties.srcAllocation && blitProperties.srcAllocation->getDefaultGmm()) { auto resInfo = blitProperties.srcAllocation->getDefaultGmm()->gmmResourceInfo.get(); auto resourceType = resInfo->getResourceType(); auto isArray = resInfo->getArraySize() > 1; @@ -100,7 +100,7 @@ void BlitCommandsHelper::appendSurfaceType(const BlitProperties &blit } } - if (blitProperties.dstAllocation->getDefaultGmm()) { + if (blitProperties.dstAllocation && blitProperties.dstAllocation->getDefaultGmm()) { auto resInfo = blitProperties.dstAllocation->getDefaultGmm()->gmmResourceInfo.get(); auto resourceType = resInfo->getResourceType(); auto isArray = resInfo->getArraySize() > 1; @@ -212,10 +212,14 @@ void BlitCommandsHelper::appendBlitCommandsForImages(const BlitProper auto srcCompressionFormat = blitCmd.getSourceCompressionFormat(); auto dstCompressionFormat = blitCmd.getDestinationCompressionFormat(); - getBlitAllocationProperties(*srcAllocation, srcRowPitch, srcQPitch, srcTileType, srcMipTailLod, srcCompressionFormat, - rootDeviceEnvironment, blitProperties.srcPlane); - getBlitAllocationProperties(*dstAllocation, dstRowPitch, dstQPitch, dstTileType, dstMipTailLod, dstCompressionFormat, - rootDeviceEnvironment, blitProperties.dstPlane); + if (srcAllocation) { + getBlitAllocationProperties(*srcAllocation, srcRowPitch, srcQPitch, srcTileType, srcMipTailLod, srcCompressionFormat, + rootDeviceEnvironment, blitProperties.srcPlane); + } + if (dstAllocation) { + getBlitAllocationProperties(*dstAllocation, dstRowPitch, dstQPitch, dstTileType, dstMipTailLod, dstCompressionFormat, + rootDeviceEnvironment, blitProperties.dstPlane); + } srcSlicePitch = std::max(srcSlicePitch, srcRowPitch * srcQPitch); dstSlicePitch = std::max(dstSlicePitch, dstRowPitch * dstQPitch); diff --git a/shared/source/helpers/blit_properties.cpp b/shared/source/helpers/blit_properties.cpp index a61a766cd4..b20535de58 100644 --- a/shared/source/helpers/blit_properties.cpp +++ b/shared/source/helpers/blit_properties.cpp @@ -244,13 +244,13 @@ bool BlitProperties::isImageOperation() const { blitDirection == BlitterConstants::BlitDirection::imageToImage; } bool BlitProperties::isSrc1DTiledArray() const { - if (srcAllocation->getDefaultGmm()) { + if (srcAllocation && srcAllocation->getDefaultGmm()) { return is1DTiledArray(srcAllocation->getDefaultGmm()->gmmResourceInfo.get()); } return false; } bool BlitProperties::isDst1DTiledArray() const { - if (dstAllocation->getDefaultGmm()) { + if (dstAllocation && dstAllocation->getDefaultGmm()) { return is1DTiledArray(dstAllocation->getDefaultGmm()->gmmResourceInfo.get()); } return false; diff --git a/shared/source/xe2_hpg_core/command_stream_receiver_hw_xe2_hpg_core.cpp b/shared/source/xe2_hpg_core/command_stream_receiver_hw_xe2_hpg_core.cpp index 0b4bedd8a2..e623f03707 100644 --- a/shared/source/xe2_hpg_core/command_stream_receiver_hw_xe2_hpg_core.cpp +++ b/shared/source/xe2_hpg_core/command_stream_receiver_hw_xe2_hpg_core.cpp @@ -68,35 +68,35 @@ void BlitCommandsHelper::appendBlitCommandsBlockCopy(const BlitPropertie auto dstAllocation = blitProperties.dstAllocation; auto srcAllocation = blitProperties.srcAllocation; - if (srcAllocation->isCompressionEnabled()) { + if (srcAllocation && srcAllocation->isCompressionEnabled()) { auto resourceFormat = srcAllocation->getDefaultGmm()->gmmResourceInfo->getResourceFormat(); srcCompressionFormat = rootDeviceEnvironment.getGmmClientContext()->getSurfaceStateCompressionFormat(resourceFormat); } - if (dstAllocation->isCompressionEnabled()) { + if (dstAllocation && dstAllocation->isCompressionEnabled()) { auto resourceFormat = dstAllocation->getDefaultGmm()->gmmResourceInfo->getResourceFormat(); dstCompressionFormat = rootDeviceEnvironment.getGmmClientContext()->getSurfaceStateCompressionFormat(resourceFormat); } if (debugManager.flags.ForceBufferCompressionFormat.get() != -1) { - if (srcAllocation->isCompressionEnabled()) { + if (srcAllocation && srcAllocation->isCompressionEnabled()) { srcCompressionFormat = static_cast(debugManager.flags.ForceBufferCompressionFormat.get()); } - if (dstAllocation->isCompressionEnabled()) { + if (dstAllocation && dstAllocation->isCompressionEnabled()) { dstCompressionFormat = static_cast(debugManager.flags.ForceBufferCompressionFormat.get()); } } DEBUG_BREAK_IF((AuxTranslationDirection::none != blitProperties.auxTranslationDirection) && - (blitProperties.dstAllocation != blitProperties.srcAllocation || !blitProperties.dstAllocation->isCompressionEnabled())); + (blitProperties.dstAllocation != blitProperties.srcAllocation || (blitProperties.dstAllocation && !blitProperties.dstAllocation->isCompressionEnabled()))); blitCmd.setSourceCompressionFormat(static_cast(srcCompressionFormat)); blitCmd.setDestinationCompressionFormat(static_cast(dstCompressionFormat)); - if (MemoryPoolHelper::isSystemMemoryPool(blitProperties.dstAllocation->getMemoryPool())) { + if (!dstAllocation || MemoryPoolHelper::isSystemMemoryPool(dstAllocation->getMemoryPool())) { blitCmd.setDestinationTargetMemory(XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_SYSTEM_MEM); } - if (MemoryPoolHelper::isSystemMemoryPool(blitProperties.srcAllocation->getMemoryPool())) { + if (!srcAllocation || MemoryPoolHelper::isSystemMemoryPool(srcAllocation->getMemoryPool())) { blitCmd.setSourceTargetMemory(XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_SYSTEM_MEM); } diff --git a/shared/source/xe3_core/command_stream_receiver_hw_xe3_core.cpp b/shared/source/xe3_core/command_stream_receiver_hw_xe3_core.cpp index 59286414fd..dce53fbd64 100644 --- a/shared/source/xe3_core/command_stream_receiver_hw_xe3_core.cpp +++ b/shared/source/xe3_core/command_stream_receiver_hw_xe3_core.cpp @@ -67,35 +67,35 @@ void BlitCommandsHelper::appendBlitCommandsBlockCopy(const BlitPropertie auto dstAllocation = blitProperties.dstAllocation; auto srcAllocation = blitProperties.srcAllocation; - if (srcAllocation->isCompressionEnabled()) { + if (srcAllocation && srcAllocation->isCompressionEnabled()) { auto resourceFormat = srcAllocation->getDefaultGmm()->gmmResourceInfo->getResourceFormat(); srcCompressionFormat = rootDeviceEnvironment.getGmmClientContext()->getSurfaceStateCompressionFormat(resourceFormat); } - if (dstAllocation->isCompressionEnabled()) { + if (dstAllocation && dstAllocation->isCompressionEnabled()) { auto resourceFormat = dstAllocation->getDefaultGmm()->gmmResourceInfo->getResourceFormat(); dstCompressionFormat = rootDeviceEnvironment.getGmmClientContext()->getSurfaceStateCompressionFormat(resourceFormat); } if (debugManager.flags.ForceBufferCompressionFormat.get() != -1) { - if (srcAllocation->isCompressionEnabled()) { + if (srcAllocation && srcAllocation->isCompressionEnabled()) { srcCompressionFormat = static_cast(debugManager.flags.ForceBufferCompressionFormat.get()); } - if (dstAllocation->isCompressionEnabled()) { + if (dstAllocation && dstAllocation->isCompressionEnabled()) { dstCompressionFormat = static_cast(debugManager.flags.ForceBufferCompressionFormat.get()); } } DEBUG_BREAK_IF((AuxTranslationDirection::none != blitProperties.auxTranslationDirection) && - (blitProperties.dstAllocation != blitProperties.srcAllocation || !blitProperties.dstAllocation->isCompressionEnabled())); + (blitProperties.dstAllocation != blitProperties.srcAllocation || (blitProperties.dstAllocation && !blitProperties.dstAllocation->isCompressionEnabled()))); blitCmd.setSourceCompressionFormat(static_cast(srcCompressionFormat)); blitCmd.setDestinationCompressionFormat(static_cast(dstCompressionFormat)); - if (MemoryPoolHelper::isSystemMemoryPool(blitProperties.dstAllocation->getMemoryPool())) { + if (!dstAllocation || MemoryPoolHelper::isSystemMemoryPool(dstAllocation->getMemoryPool())) { blitCmd.setDestinationTargetMemory(XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_SYSTEM_MEM); } - if (MemoryPoolHelper::isSystemMemoryPool(blitProperties.srcAllocation->getMemoryPool())) { + if (!srcAllocation || MemoryPoolHelper::isSystemMemoryPool(srcAllocation->getMemoryPool())) { blitCmd.setSourceTargetMemory(XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_SYSTEM_MEM); } diff --git a/shared/source/xe_hpc_core/command_stream_receiver_hw_xe_hpc_core.cpp b/shared/source/xe_hpc_core/command_stream_receiver_hw_xe_hpc_core.cpp index 3819b1f5f5..9e6e811dab 100644 --- a/shared/source/xe_hpc_core/command_stream_receiver_hw_xe_hpc_core.cpp +++ b/shared/source/xe_hpc_core/command_stream_receiver_hw_xe_hpc_core.cpp @@ -142,7 +142,7 @@ void BlitCommandsHelper::appendBlitCommandsMemCopy(const BlitProperties } DEBUG_BREAK_IF((AuxTranslationDirection::none != blitProperties.auxTranslationDirection) && - (dstAllocation != srcAllocation || !dstAllocation->isCompressionEnabled())); + (dstAllocation != srcAllocation || (dstAllocation && !dstAllocation->isCompressionEnabled()))); } template <> @@ -209,8 +209,8 @@ void BlitCommandsHelper::encodeWa(LinearStream &cmdStream, const BlitPro const bool applyForDst = (debugManager.flags.EnableBcsSwControlWa.get() & dstInSystemMemOnly); const bool applyAlways = (debugManager.flags.EnableBcsSwControlWa.get() == enableAlways); - const bool enableWa = (!blitProperties.srcAllocation->isAllocatedInLocalMemoryPool() && applyForSrc) || - (!blitProperties.dstAllocation->isAllocatedInLocalMemoryPool() && applyForDst) || + const bool enableWa = ((blitProperties.srcAllocation && !blitProperties.srcAllocation->isAllocatedInLocalMemoryPool()) && applyForSrc) || + ((blitProperties.dstAllocation && !blitProperties.dstAllocation->isAllocatedInLocalMemoryPool()) && applyForDst) || applyAlways; uint32_t newValue = enableWa ? waEnabledMMioValue : waDisabledMMioValue; diff --git a/shared/source/xe_hpg_core/command_stream_receiver_hw_xe_hpg_core.cpp b/shared/source/xe_hpg_core/command_stream_receiver_hw_xe_hpg_core.cpp index 931ccc707d..d16dedae3b 100644 --- a/shared/source/xe_hpg_core/command_stream_receiver_hw_xe_hpg_core.cpp +++ b/shared/source/xe_hpg_core/command_stream_receiver_hw_xe_hpg_core.cpp @@ -49,9 +49,9 @@ template <> void BlitCommandsHelper::adjustControlSurfaceType(const BlitProperties &blitProperties, typename Family::XY_BLOCK_COPY_BLT &blitCmd) { using CONTROL_SURFACE_TYPE = typename Family::XY_BLOCK_COPY_BLT::CONTROL_SURFACE_TYPE; using COMPRESSION_ENABLE = typename Family::XY_BLOCK_COPY_BLT::COMPRESSION_ENABLE; - auto srcAllocation = blitProperties.srcAllocation; - if (srcAllocation->getDefaultGmm()) { + auto srcAllocation = blitProperties.srcAllocation; + if (srcAllocation && srcAllocation->getDefaultGmm()) { auto gmmResourceInfo = srcAllocation->getDefaultGmm()->gmmResourceInfo.get(); auto resInfo = gmmResourceInfo->getResourceFlags()->Info; if (resInfo.MediaCompressed) { @@ -62,7 +62,7 @@ void BlitCommandsHelper::adjustControlSurfaceType(const BlitProperties & } auto dstAllocation = blitProperties.dstAllocation; - if (dstAllocation->getDefaultGmm()) { + if (dstAllocation && dstAllocation->getDefaultGmm()) { auto gmmResourceInfo = dstAllocation->getDefaultGmm()->gmmResourceInfo.get(); auto resInfo = gmmResourceInfo->getResourceFlags()->Info; if (resInfo.MediaCompressed) { @@ -90,12 +90,13 @@ void BlitCommandsHelper::appendBlitCommandsBlockCopy(const BlitPropertie compressionEnabledField = static_cast(debugManager.flags.ForceCompressionDisabledForCompressedBlitCopies.get()); } - if (blitProperties.dstAllocation->isCompressionEnabled()) { + if (blitProperties.dstAllocation && blitProperties.dstAllocation->isCompressionEnabled()) { blitCmd.setDestinationCompressionEnable(compressionEnabledField); blitCmd.setDestinationAuxiliarysurfacemode(XY_BLOCK_COPY_BLT::AUXILIARY_SURFACE_MODE_AUX_CCS_E); blitCmd.setDestinationCompressionFormat(compressionFormat); } - if (blitProperties.srcAllocation->isCompressionEnabled()) { + + if (blitProperties.srcAllocation && blitProperties.srcAllocation->isCompressionEnabled()) { blitCmd.setSourceCompressionEnable(compressionEnabledField); blitCmd.setSourceAuxiliarysurfacemode(XY_BLOCK_COPY_BLT::AUXILIARY_SURFACE_MODE_AUX_CCS_E); blitCmd.setSourceCompressionFormat(compressionFormat); @@ -128,7 +129,7 @@ void BlitCommandsHelper::appendBlitCommandsBlockCopy(const BlitPropertie } DEBUG_BREAK_IF((AuxTranslationDirection::none != blitProperties.auxTranslationDirection) && - (blitProperties.dstAllocation != blitProperties.srcAllocation || !blitProperties.dstAllocation->isCompressionEnabled())); + (blitProperties.dstAllocation != blitProperties.srcAllocation || (blitProperties.dstAllocation && !blitProperties.dstAllocation->isCompressionEnabled()))); auto mocs = rootDeviceEnvironment.getGmmHelper()->getUncachedMOCS(); diff --git a/shared/test/unit_test/blit_properties/blitter_properties_tests.cpp b/shared/test/unit_test/blit_properties/blitter_properties_tests.cpp index d01185243f..15363fe3c6 100644 --- a/shared/test/unit_test/blit_properties/blitter_properties_tests.cpp +++ b/shared/test/unit_test/blit_properties/blitter_properties_tests.cpp @@ -99,6 +99,7 @@ TEST_F(BlitPropertiesTests, givenBlitPropertiesWhenDstIs1DTiledArrayThenTransfor EXPECT_EQ(blitProperties.dstOffset.y, 0u); EXPECT_EQ(blitProperties.dstOffset.z, size.y); } + TEST_F(BlitPropertiesTests, givenBlitPropertiesWhenDstAndSrcIsNot1DTiledArrayThenSizeAndOffsetNotchanged) { blitProperties.transform1DArrayTo2DArrayIfNeeded(); @@ -143,4 +144,24 @@ TEST_F(BlitPropertiesTests, givenGmmResInfoForTiled1DWhenIs1DTiledArrayCalledThe resourceInfoSrc->mockResourceCreateParams.ArraySize = 1; EXPECT_FALSE(blitProperties.is1DTiledArray(resourceInfoSrc)); } + +TEST_F(BlitPropertiesTests, givenNullSrcAllocationWhenCallingIsSrc1DTiledArrayThenFalseIsReturned) { + blitProperties.srcAllocation = nullptr; + EXPECT_FALSE(blitProperties.isSrc1DTiledArray()); +} + +TEST_F(BlitPropertiesTests, givenNullDstAllocationWhenCallingIsDst1DTiledArrayThenFalseIsReturned) { + blitProperties.dstAllocation = nullptr; + EXPECT_FALSE(blitProperties.isDst1DTiledArray()); +} + +TEST_F(BlitPropertiesTests, givenSrcAllocationWithNullGmmWhenCallingIsSrc1DTiledArrayThenFalseIsReturned) { + blitProperties.srcAllocation->setGmm(nullptr, 0); + EXPECT_FALSE(blitProperties.isSrc1DTiledArray()); +} + +TEST_F(BlitPropertiesTests, givenDstAllocationWithNullGmmWhenCallingIsDst1DTiledArrayThenFalseIsReturned) { + blitProperties.dstAllocation->setGmm(nullptr, 0); + EXPECT_FALSE(blitProperties.isDst1DTiledArray()); +} } // namespace NEO diff --git a/shared/test/unit_test/helpers/blit_commands_helper_tests.cpp b/shared/test/unit_test/helpers/blit_commands_helper_tests.cpp index 9ac76d3c29..4d18618f70 100644 --- a/shared/test/unit_test/helpers/blit_commands_helper_tests.cpp +++ b/shared/test/unit_test/helpers/blit_commands_helper_tests.cpp @@ -1422,3 +1422,258 @@ HWTEST2_F(BlitTests, givenXyCopyBltCommandWhenApplyBlitPropertiesIsCalledThenNot NEO::BlitCommandsHelper::applyAdditionalBlitProperties(properties, bltCmd, pDevice->getRootDeviceEnvironment(), true); EXPECT_EQ(memcmp(&bltCmd, &bltCmdBefore, sizeof(XY_COPY_BLT)), 0); } + +HWTEST2_F(BlitTests, givenSrcAndDstAllocationWithCompressionEnabledWhenAppendBlitCommandsBlockCopyThenSetSrcAndDstTargetMemToLocalMemAndCompressionHandled, IsXeHpgCore) { + using XY_BLOCK_COPY_BLT = typename FamilyType::XY_BLOCK_COPY_BLT; + XY_BLOCK_COPY_BLT blitCmd = FamilyType::cmdInitXyBlockCopyBlt; + + blitCmd.setSourceX1CoordinateLeft(0); + blitCmd.setSourceY1CoordinateTop(0); + blitCmd.setDestinationX2CoordinateRight(1); + blitCmd.setDestinationY2CoordinateBottom(1); + + BlitProperties properties{}; + auto gmm = std::make_unique(pDevice->getGmmHelper()); + gmm->setCompressionEnabled(true); + MockGraphicsAllocation mockDstAllocation(0, 1u /*num gmms*/, AllocationType::internalHostMemory, reinterpret_cast(0x1234), + 0x2000, 0, sizeof(uint32_t), MemoryPool::system4KBPages, MemoryManager::maxOsContextCount); + MockGraphicsAllocation mockSrcAllocation(0, 1u /*num gmms*/, AllocationType::internalHostMemory, reinterpret_cast(0x2345), + 0x2000, 0, sizeof(uint32_t), MemoryPool::system4KBPages, MemoryManager::maxOsContextCount); + mockDstAllocation.setGmm(gmm.get(), 0); + mockSrcAllocation.setGmm(gmm.get(), 0); + + properties.dstAllocation = &mockDstAllocation; + properties.srcAllocation = &mockSrcAllocation; + + BlitCommandsHelper::appendBlitCommandsBlockCopy(properties, blitCmd, pDevice->getRootDeviceEnvironment()); + + EXPECT_EQ(blitCmd.getDestinationTargetMemory(), XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_LOCAL_MEM); + EXPECT_EQ(blitCmd.getSourceTargetMemory(), XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_LOCAL_MEM); + EXPECT_EQ(blitCmd.getDestinationCompressionEnable(), XY_BLOCK_COPY_BLT::COMPRESSION_ENABLE::COMPRESSION_ENABLE_COMPRESSION_ENABLE); + EXPECT_EQ(blitCmd.getSourceCompressionEnable(), XY_BLOCK_COPY_BLT::COMPRESSION_ENABLE::COMPRESSION_ENABLE_COMPRESSION_ENABLE); + EXPECT_EQ(blitCmd.getDestinationAuxiliarysurfacemode(), XY_BLOCK_COPY_BLT::AUXILIARY_SURFACE_MODE_AUX_CCS_E); + EXPECT_EQ(blitCmd.getSourceAuxiliarysurfacemode(), XY_BLOCK_COPY_BLT::AUXILIARY_SURFACE_MODE_AUX_CCS_E); +} + +HWTEST2_F(BlitTests, givenSrcAndDstAllocationWithoutCompressionEnabledWhenAppendBlitCommandsBlockCopyThenSetSrcAndDstTargetMemToLocalMemAndCompressionHandled, IsXeHpgCore) { + using XY_BLOCK_COPY_BLT = typename FamilyType::XY_BLOCK_COPY_BLT; + XY_BLOCK_COPY_BLT blitCmd = FamilyType::cmdInitXyBlockCopyBlt; + + blitCmd.setSourceX1CoordinateLeft(0); + blitCmd.setSourceY1CoordinateTop(0); + blitCmd.setDestinationX2CoordinateRight(1); + blitCmd.setDestinationY2CoordinateBottom(1); + + BlitProperties properties{}; + auto gmm = std::make_unique(pDevice->getGmmHelper()); + gmm->setCompressionEnabled(false); + MockGraphicsAllocation mockDstAllocation(0, 1u /*num gmms*/, AllocationType::internalHostMemory, reinterpret_cast(0x1234), + 0x2000, 0, sizeof(uint32_t), MemoryPool::system4KBPages, MemoryManager::maxOsContextCount); + MockGraphicsAllocation mockSrcAllocation(0, 1u /*num gmms*/, AllocationType::internalHostMemory, reinterpret_cast(0x2345), + 0x2000, 0, sizeof(uint32_t), MemoryPool::system4KBPages, MemoryManager::maxOsContextCount); + mockDstAllocation.setGmm(gmm.get(), 0); + mockSrcAllocation.setGmm(gmm.get(), 0); + + properties.dstAllocation = &mockDstAllocation; + properties.srcAllocation = &mockSrcAllocation; + + BlitCommandsHelper::appendBlitCommandsBlockCopy(properties, blitCmd, pDevice->getRootDeviceEnvironment()); + + EXPECT_EQ(blitCmd.getDestinationTargetMemory(), XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_LOCAL_MEM); + EXPECT_EQ(blitCmd.getSourceTargetMemory(), XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_LOCAL_MEM); + EXPECT_EQ(blitCmd.getDestinationCompressionEnable(), 0u); + EXPECT_EQ(blitCmd.getSourceCompressionEnable(), 0u); + EXPECT_EQ(blitCmd.getDestinationAuxiliarysurfacemode(), 0u); + EXPECT_EQ(blitCmd.getSourceAuxiliarysurfacemode(), 0u); +} + +HWTEST2_F(BlitTests, givenNullAllocationsWhenAppendBlitCommandsBlockCopyThenSetSrcAndDstTargetMemToLocalMemAndCompressionHandled, IsXeHpgCore) { + using XY_BLOCK_COPY_BLT = typename FamilyType::XY_BLOCK_COPY_BLT; + XY_BLOCK_COPY_BLT blitCmd = FamilyType::cmdInitXyBlockCopyBlt; + + blitCmd.setSourceX1CoordinateLeft(0); + blitCmd.setSourceY1CoordinateTop(0); + blitCmd.setDestinationX2CoordinateRight(1); + blitCmd.setDestinationY2CoordinateBottom(1); + + BlitProperties properties{}; + + properties.dstAllocation = nullptr; + properties.srcAllocation = nullptr; + + BlitCommandsHelper::appendBlitCommandsBlockCopy(properties, blitCmd, pDevice->getRootDeviceEnvironment()); + + EXPECT_EQ(blitCmd.getDestinationTargetMemory(), XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_LOCAL_MEM); + EXPECT_EQ(blitCmd.getSourceTargetMemory(), XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_LOCAL_MEM); + EXPECT_EQ(blitCmd.getDestinationCompressionEnable(), 0u); + EXPECT_EQ(blitCmd.getSourceCompressionEnable(), 0u); + EXPECT_EQ(blitCmd.getDestinationAuxiliarysurfacemode(), 0u); + EXPECT_EQ(blitCmd.getSourceAuxiliarysurfacemode(), 0u); +} + +HWTEST2_F(BlitTests, givenNullAllocationsWhenAdjustControlSurfaceTypeThenSurfaceTypeWillNotChange, IsXeHpgCore) { + using XY_BLOCK_COPY_BLT = typename FamilyType::XY_BLOCK_COPY_BLT; + XY_BLOCK_COPY_BLT blitCmd = FamilyType::cmdInitXyBlockCopyBlt; + + BlitProperties properties{}; + properties.srcAllocation = nullptr; + properties.dstAllocation = nullptr; + + constexpr auto initialSrcSurface = 0; // CONTROL_SURFACE_TYPE_3D + constexpr auto initialDstSurface = 0; // CONTROL_SURFACE_TYPE_3D + + BlitCommandsHelper::adjustControlSurfaceType(properties, blitCmd); + + EXPECT_EQ(blitCmd.getSourceControlSurfaceType(), initialSrcSurface); + EXPECT_EQ(blitCmd.getDestinationControlSurfaceType(), initialDstSurface); +} + +HWTEST2_F(BlitTests, givenSrcAndDstAllocationWithCompressionEnabledAndForceBufferCompressionFormatSetWhenAppendBlitCommandsBlockCopyIsCalledThenSetSrcAndDstTargetMemToSystemMemAndCompressionHandled, IsAtLeastXe2HpgCore) { + using XY_BLOCK_COPY_BLT = typename FamilyType::XY_BLOCK_COPY_BLT; + XY_BLOCK_COPY_BLT blitCmd = FamilyType::cmdInitXyBlockCopyBlt; + + blitCmd.setSourceX1CoordinateLeft(0); + blitCmd.setSourceY1CoordinateTop(0); + blitCmd.setDestinationX2CoordinateRight(1); + blitCmd.setDestinationY2CoordinateBottom(1); + + BlitProperties properties = {}; + DebugManagerStateRestore dbgRestore; + + uint32_t compressionFormat = 1; + debugManager.flags.ForceBufferCompressionFormat.set(static_cast(compressionFormat)); + auto gmm = std::make_unique(pDevice->getGmmHelper()); + gmm->setCompressionEnabled(true); + MockGraphicsAllocation mockDstAllocation(0, 1u /*num gmms*/, AllocationType::buffer, reinterpret_cast(0x1234), 0x1000, 0, sizeof(uint32_t), MemoryPool::system4KBPages, MemoryManager::maxOsContextCount); + MockGraphicsAllocation mockSrcAllocation(0, 1u /*num gmms*/, AllocationType::buffer, reinterpret_cast(0x2345), 0x2000, 0, sizeof(uint32_t), MemoryPool::system4KBPages, MemoryManager::maxOsContextCount); + mockDstAllocation.setGmm(gmm.get(), 0); + mockSrcAllocation.setGmm(gmm.get(), 0); + + properties.dstAllocation = &mockDstAllocation; + properties.srcAllocation = &mockSrcAllocation; + + BlitCommandsHelper::appendBlitCommandsBlockCopy(properties, blitCmd, pDevice->getRootDeviceEnvironment()); + + EXPECT_EQ(blitCmd.getDestinationCompressionFormat(), compressionFormat); + EXPECT_EQ(blitCmd.getSourceCompressionFormat(), compressionFormat); + EXPECT_EQ(blitCmd.getDestinationTargetMemory(), XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_SYSTEM_MEM); + EXPECT_EQ(blitCmd.getSourceTargetMemory(), XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_SYSTEM_MEM); +} + +HWTEST2_F(BlitTests, givenSrcAllocationWithoutCompressionEnabledAndForceBufferCompressionFormatSetWhenAppendBlitCommandsBlockCopyIsCalledThenSetSrcTargetMemToSystemMemAndCompressionHandled, IsAtLeastXe2HpgCore) { + using XY_BLOCK_COPY_BLT = typename FamilyType::XY_BLOCK_COPY_BLT; + XY_BLOCK_COPY_BLT blitCmd = FamilyType::cmdInitXyBlockCopyBlt; + + blitCmd.setSourceX1CoordinateLeft(0); + blitCmd.setSourceY1CoordinateTop(0); + blitCmd.setDestinationX2CoordinateRight(1); + blitCmd.setDestinationY2CoordinateBottom(1); + + BlitProperties properties = {}; + DebugManagerStateRestore dbgRestore; + + MockGraphicsAllocation mockSrcAllocation(0, 1u /*num gmms*/, AllocationType::internalHostMemory, reinterpret_cast(0x1234), + 0x2000, 0, sizeof(uint32_t), MemoryPool::system4KBPages, MemoryManager::maxOsContextCount); + + auto gmm = std::make_unique(pDevice->getGmmHelper()); + gmm->setCompressionEnabled(false); + mockSrcAllocation.setGmm(gmm.get(), 0); + + properties.srcAllocation = &mockSrcAllocation; + properties.dstAllocation = nullptr; + + debugManager.flags.ForceBufferCompressionFormat.set(1); + + auto initialDstCompressionFormat = blitCmd.getDestinationCompressionFormat(); + auto initialSrcCompressionFormat = blitCmd.getSourceCompressionFormat(); + + BlitCommandsHelper::appendBlitCommandsBlockCopy(properties, blitCmd, pDevice->getRootDeviceEnvironment()); + + EXPECT_EQ(blitCmd.getDestinationCompressionFormat(), initialDstCompressionFormat); + EXPECT_EQ(blitCmd.getSourceCompressionFormat(), initialSrcCompressionFormat); + EXPECT_EQ(blitCmd.getDestinationTargetMemory(), XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_SYSTEM_MEM); + EXPECT_EQ(blitCmd.getSourceTargetMemory(), XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_SYSTEM_MEM); +} + +HWTEST2_F(BlitTests, givenDstAllocationWithoutCompressionEnabledAndForceBufferCompressionFormatSetWhenAppendBlitCommandsBlockCopyIsCalledThenSetDstTargetMemToSystemMemAndCompressionHandled, IsAtLeastXe2HpgCore) { + using XY_BLOCK_COPY_BLT = typename FamilyType::XY_BLOCK_COPY_BLT; + XY_BLOCK_COPY_BLT blitCmd = FamilyType::cmdInitXyBlockCopyBlt; + + blitCmd.setSourceX1CoordinateLeft(0); + blitCmd.setSourceY1CoordinateTop(0); + blitCmd.setDestinationX2CoordinateRight(1); + blitCmd.setDestinationY2CoordinateBottom(1); + + BlitProperties properties = {}; + DebugManagerStateRestore dbgRestore; + + MockGraphicsAllocation mockDstAllocation(0, 1u /*num gmms*/, AllocationType::internalHostMemory, reinterpret_cast(0x1234), + 0x2000, 0, sizeof(uint32_t), MemoryPool::system4KBPages, MemoryManager::maxOsContextCount); + + auto gmm = std::make_unique(pDevice->getGmmHelper()); + gmm->setCompressionEnabled(false); + mockDstAllocation.setGmm(gmm.get(), 0); + + properties.dstAllocation = &mockDstAllocation; + properties.srcAllocation = nullptr; + + debugManager.flags.ForceBufferCompressionFormat.set(1); + + auto initialDstCompressionFormat = blitCmd.getDestinationCompressionFormat(); + auto initialSrcCompressionFormat = blitCmd.getSourceCompressionFormat(); + + BlitCommandsHelper::appendBlitCommandsBlockCopy(properties, blitCmd, pDevice->getRootDeviceEnvironment()); + + EXPECT_EQ(blitCmd.getDestinationCompressionFormat(), initialDstCompressionFormat); + EXPECT_EQ(blitCmd.getSourceCompressionFormat(), initialSrcCompressionFormat); + EXPECT_EQ(blitCmd.getDestinationTargetMemory(), XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_SYSTEM_MEM); + EXPECT_EQ(blitCmd.getSourceTargetMemory(), XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_SYSTEM_MEM); +} + +HWTEST2_F(BlitTests, givenNullAllocationsAndForceBufferCompressionFormatSetWhenAppendBlitCommandsBlockCopyIsCalledThenSetSrcAndDstTargetMemToSystemMemAndCompressionHandled, IsAtLeastXe2HpgCore) { + using XY_BLOCK_COPY_BLT = typename FamilyType::XY_BLOCK_COPY_BLT; + XY_BLOCK_COPY_BLT blitCmd = FamilyType::cmdInitXyBlockCopyBlt; + + blitCmd.setSourceX1CoordinateLeft(0); + blitCmd.setSourceY1CoordinateTop(0); + blitCmd.setDestinationX2CoordinateRight(1); + blitCmd.setDestinationY2CoordinateBottom(1); + + BlitProperties properties = {}; + DebugManagerStateRestore dbgRestore; + + properties.dstAllocation = nullptr; + properties.srcAllocation = nullptr; + + debugManager.flags.ForceBufferCompressionFormat.set(1); + + auto initialDstCompressionFormat = blitCmd.getDestinationCompressionFormat(); + auto initialSrcCompressionFormat = blitCmd.getSourceCompressionFormat(); + + BlitCommandsHelper::appendBlitCommandsBlockCopy(properties, blitCmd, pDevice->getRootDeviceEnvironment()); + + EXPECT_EQ(blitCmd.getDestinationCompressionFormat(), initialDstCompressionFormat); + EXPECT_EQ(blitCmd.getSourceCompressionFormat(), initialSrcCompressionFormat); + EXPECT_EQ(blitCmd.getDestinationTargetMemory(), XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_SYSTEM_MEM); + EXPECT_EQ(blitCmd.getSourceTargetMemory(), XY_BLOCK_COPY_BLT::TARGET_MEMORY::TARGET_MEMORY_SYSTEM_MEM); +} + +HWTEST2_F(BlitTests, givenNullAllocationsWhenAppendBlitCommandsForImagesThenSlicePitchesWillBeUpdated, IsNotPVC) { + using XY_BLOCK_COPY_BLT = typename FamilyType::XY_BLOCK_COPY_BLT; + XY_BLOCK_COPY_BLT blitCmd = FamilyType::cmdInitXyBlockCopyBlt; + + BlitProperties properties{}; + properties.dstAllocation = nullptr; + properties.srcAllocation = nullptr; + properties.dstSize = {8, 12, 1}; + properties.srcSize = {10, 10, 1}; + properties.dstRowPitch = 0x40; + properties.srcRowPitch = 0x10; + uint32_t dstSlicePitch = 0; + uint32_t srcSlicePitch = 0; + + BlitCommandsHelper::appendBlitCommandsForImages(properties, blitCmd, pDevice->getRootDeviceEnvironment(), srcSlicePitch, dstSlicePitch); + + EXPECT_NE(dstSlicePitch, 0u); + EXPECT_NE(srcSlicePitch, 0u); +}