diff --git a/level_zero/core/source/image/image.h b/level_zero/core/source/image/image.h index 57581ea057..2c468e430c 100644 --- a/level_zero/core/source/image/image.h +++ b/level_zero/core/source/image/image.h @@ -15,6 +15,7 @@ namespace NEO { struct ImageInfo; class GraphicsAllocation; struct ImageDescriptor; +struct SurfaceStateInHeapInfo; } // namespace NEO namespace L0 { @@ -42,6 +43,8 @@ struct Image : _ze_image_handle_t { virtual NEO::ImageInfo getImageInfo() = 0; virtual ze_image_desc_t getImageDesc() = 0; virtual ze_result_t getMemoryProperties(ze_image_memory_properties_exp_t *pMemoryProperties) = 0; + virtual ze_result_t allocateBindlessSlot() = 0; + virtual NEO::SurfaceStateInHeapInfo *getBindlessSlot() = 0; static Image *fromHandle(ze_image_handle_t handle) { return static_cast(handle); } diff --git a/level_zero/core/source/image/image_imp.cpp b/level_zero/core/source/image/image_imp.cpp index da448c9662..b480e4a18c 100644 --- a/level_zero/core/source/image/image_imp.cpp +++ b/level_zero/core/source/image/image_imp.cpp @@ -8,6 +8,10 @@ #include "level_zero/core/source/image/image_imp.h" #include "shared/source/device/device.h" +#include "shared/source/execution_environment/execution_environment.h" +#include "shared/source/execution_environment/root_device_environment.h" +#include "shared/source/helpers/bindless_heaps_helper.h" +#include "shared/source/helpers/gfx_core_helper.h" #include "shared/source/helpers/hw_info.h" #include "shared/source/memory_manager/memory_manager.h" @@ -22,6 +26,11 @@ namespace L0 { ImageAllocatorFn imageFactory[IGFX_MAX_PRODUCT] = {}; ImageImp::~ImageImp() { + if (isImageView() && this->device != nullptr && this->allocation) { + if (bindlessInfo.get() && this->device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[this->allocation->getRootDeviceIndex()]->getBindlessHeapsHelper() != nullptr) { + this->device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[this->allocation->getRootDeviceIndex()]->getBindlessHeapsHelper()->releaseSSToReusePool(*bindlessInfo); + } + } if (!isImageView() && this->device != nullptr) { this->device->getNEODevice()->getMemoryManager()->freeGraphicsMemory(this->allocation); } @@ -77,6 +86,36 @@ ze_result_t ImageImp::createView(Device *device, const ze_image_desc_t *desc, ze return result; } +ze_result_t ImageImp::allocateBindlessSlot() { + if (!isImageView()) { + if (!this->device->getNEODevice()->getMemoryManager()->allocateBindlessSlot(allocation)) { + return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + if (allocation->getBindlessOffset() != std::numeric_limits::max()) { + bindlessInfo = std::make_unique(allocation->getBindlessInfo()); + } + return ZE_RESULT_SUCCESS; + } + auto bindlessHelper = this->device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[allocation->getRootDeviceIndex()]->getBindlessHeapsHelper(); + + if (bindlessHelper && !bindlessInfo) { + auto &gfxCoreHelper = this->device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[allocation->getRootDeviceIndex()]->getHelper(); + const auto surfStateCount = 2; + auto surfaceStateSize = surfStateCount * gfxCoreHelper.getRenderSurfaceStateSize(); + + auto surfaceStateInfo = bindlessHelper->allocateSSInHeap(surfaceStateSize, allocation, NEO::BindlessHeapsHelper::GLOBAL_SSH); + if (surfaceStateInfo.heapAllocation == nullptr) { + return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + bindlessInfo = std::make_unique(surfaceStateInfo); + } + return ZE_RESULT_SUCCESS; +} + +NEO::SurfaceStateInHeapInfo *ImageImp::getBindlessSlot() { + return bindlessInfo.get(); +} + ze_result_t Image::create(uint32_t productFamily, Device *device, const ze_image_desc_t *desc, Image **pImage) { ze_result_t result = ZE_RESULT_SUCCESS; ImageAllocatorFn allocator = nullptr; diff --git a/level_zero/core/source/image/image_imp.h b/level_zero/core/source/image/image_imp.h index cc62a15658..07f542f1eb 100644 --- a/level_zero/core/source/image/image_imp.h +++ b/level_zero/core/source/image/image_imp.h @@ -11,6 +11,7 @@ #include "level_zero/core/source/image/image.h" +#include #include namespace L0 { @@ -43,11 +44,15 @@ struct ImageImp : public Image { return sourceImageFormatDesc.has_value(); } + ze_result_t allocateBindlessSlot() override; + NEO::SurfaceStateInHeapInfo *getBindlessSlot() override; + protected: Device *device = nullptr; NEO::ImageInfo imgInfo = {}; NEO::GraphicsAllocation *allocation = nullptr; ze_image_desc_t imageFormatDesc = {}; std::optional sourceImageFormatDesc = {}; + std::unique_ptr bindlessInfo; }; } // namespace L0 diff --git a/level_zero/core/source/kernel/kernel_imp.cpp b/level_zero/core/source/kernel/kernel_imp.cpp index 753cfa68dc..83a63d995c 100644 --- a/level_zero/core/source/kernel/kernel_imp.cpp +++ b/level_zero/core/source/kernel/kernel_imp.cpp @@ -557,20 +557,20 @@ ze_result_t KernelImp::setArgRedescribedImage(uint32_t argIndex, ze_image_handle const auto surfaceStateSize = gfxCoreHelper.getRenderSurfaceStateSize(); if (bindlessHeapsHelper) { - if (!this->module->getDevice()->getNEODevice()->getMemoryManager()->allocateBindlessSlot(image->getAllocation())) { + if (image->allocateBindlessSlot() != ZE_RESULT_SUCCESS) { return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - auto ssInHeap = image->getAllocation()->getBindlessInfo(); + auto ssInHeap = image->getBindlessSlot(); auto patchLocation = ptrOffset(getCrossThreadData(), arg.bindless); // redescribed image's surface state is after image's state - auto bindlessSlotOffset = ssInHeap.surfaceStateOffset + surfaceStateSize; + auto bindlessSlotOffset = ssInHeap->surfaceStateOffset + surfaceStateSize; auto patchValue = gfxCoreHelper.getBindlessSurfaceExtendedMessageDescriptorValue(static_cast(bindlessSlotOffset)); patchWithRequiredSize(const_cast(patchLocation), sizeof(patchValue), patchValue); - image->copyRedescribedSurfaceStateToSSH(ptrOffset(ssInHeap.ssPtr, surfaceStateSize), 0u); + image->copyRedescribedSurfaceStateToSSH(ptrOffset(ssInHeap->ssPtr, surfaceStateSize), 0u); isBindlessOffsetSet[argIndex] = true; - this->residencyContainer.push_back(ssInHeap.heapAllocation); + this->residencyContainer.push_back(ssInHeap->heapAllocation); } else { auto ssPtr = ptrOffset(surfaceStateHeapData.get(), getSurfaceStateIndexForBindlessOffset(arg.bindless) * surfaceStateSize); @@ -758,18 +758,24 @@ ze_result_t KernelImp::setArgImage(uint32_t argIndex, size_t argSize, const void if (kernelImmData->getDescriptor().kernelAttributes.imageAddressingMode == NEO::KernelDescriptor::Bindless) { NEO::BindlessHeapsHelper *bindlessHeapsHelper = this->module->getDevice()->getNEODevice()->getBindlessHeapsHelper(); + auto &gfxCoreHelper = this->module->getDevice()->getNEODevice()->getRootDeviceEnvironmentRef().getHelper(); + auto surfaceStateSize = gfxCoreHelper.getRenderSurfaceStateSize(); if (bindlessHeapsHelper) { - if (!this->module->getDevice()->getNEODevice()->getMemoryManager()->allocateBindlessSlot(image->getAllocation())) { + if (image->allocateBindlessSlot() != ZE_RESULT_SUCCESS) { return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - auto ssPtr = patchBindlessSurfaceState(image->getAllocation(), arg.bindless); + auto ssInHeap = image->getBindlessSlot(); + auto patchLocation = ptrOffset(getCrossThreadData(), arg.bindless); + auto bindlessSlotOffset = ssInHeap->surfaceStateOffset; + auto patchValue = gfxCoreHelper.getBindlessSurfaceExtendedMessageDescriptorValue(static_cast(bindlessSlotOffset)); + patchWithRequiredSize(const_cast(patchLocation), sizeof(patchValue), patchValue); + + image->copySurfaceStateToSSH(ssInHeap->ssPtr, 0u, isMediaBlockImage); isBindlessOffsetSet[argIndex] = true; - image->copySurfaceStateToSSH(ssPtr, 0u, isMediaBlockImage); + this->residencyContainer.push_back(ssInHeap->heapAllocation); } else { - auto &gfxCoreHelper = this->module->getDevice()->getNEODevice()->getRootDeviceEnvironmentRef().getHelper(); - auto surfaceStateSize = gfxCoreHelper.getRenderSurfaceStateSize(); auto ssPtr = ptrOffset(surfaceStateHeapData.get(), getSurfaceStateIndexForBindlessOffset(arg.bindless) * surfaceStateSize); image->copySurfaceStateToSSH(ssPtr, 0u, isMediaBlockImage); } diff --git a/level_zero/core/test/black_box_tests/zello_image_view.cpp b/level_zero/core/test/black_box_tests/zello_image_view.cpp index f920f2fff1..ca8cbaaa57 100644 --- a/level_zero/core/test/black_box_tests/zello_image_view.cpp +++ b/level_zero/core/test/black_box_tests/zello_image_view.cpp @@ -106,10 +106,6 @@ void testAppendImageViewNV12Copy(ze_context_handle_t &context, ze_device_handle_ 0, 0}; ze_image_handle_t planeYImageView; - - SUCCESS_OR_TERMINATE( - zeImageViewCreateExp(context, device, &imageViewDescPlaneY, srcImg, &planeYImageView)); - planeYdesc.stype = ZE_STRUCTURE_TYPE_IMAGE_VIEW_PLANAR_EXT_DESC; SUCCESS_OR_TERMINATE( zeImageViewCreateExt(context, device, &imageViewDescPlaneY, srcImg, &planeYImageView)); @@ -132,10 +128,6 @@ void testAppendImageViewNV12Copy(ze_context_handle_t &context, ze_device_handle_ 0, 0}; ze_image_handle_t planeUVImageView; - - SUCCESS_OR_TERMINATE( - zeImageViewCreateExp(context, device, &imageViewDescPlaneUV, srcImg, &planeUVImageView)); - planeUVdesc.stype = ZE_STRUCTURE_TYPE_IMAGE_VIEW_PLANAR_EXT_DESC; SUCCESS_OR_TERMINATE( zeImageViewCreateExt(context, device, &imageViewDescPlaneUV, srcImg, &planeUVImageView)); @@ -289,9 +281,11 @@ void testAppendImageViewNV12Copy(ze_context_handle_t &context, ze_device_handle_ } // cleanup - SUCCESS_OR_TERMINATE(zeImageDestroy(srcImg)); SUCCESS_OR_TERMINATE(zeImageDestroy(planeYImageView)); SUCCESS_OR_TERMINATE(zeImageDestroy(planeUVImageView)); + + SUCCESS_OR_TERMINATE(zeImageDestroy(srcImg)); + SUCCESS_OR_TERMINATE(zeCommandListDestroy(cmdList)); SUCCESS_OR_TERMINATE(zeCommandQueueDestroy(cmdQueue)); SUCCESS_OR_TERMINATE(zeKernelDestroy(kernel)); diff --git a/level_zero/core/test/unit_tests/sources/kernel/test_kernel.cpp b/level_zero/core/test/unit_tests/sources/kernel/test_kernel.cpp index 91b387e85f..e78ce84640 100644 --- a/level_zero/core/test/unit_tests/sources/kernel/test_kernel.cpp +++ b/level_zero/core/test/unit_tests/sources/kernel/test_kernel.cpp @@ -2609,6 +2609,121 @@ HWTEST2_F(SetKernelArg, givenImageBindlessKernelAndGlobalBindlessHelperWhenSetAr EXPECT_TRUE(kernel->isBindlessOffsetSet[3]); } +HWTEST2_F(SetKernelArg, givenGlobalBindlessHelperAndImageViewWhenAllocatingBindlessSlotThenViewHasDifferentSlotThanParentImage, ImageSupport) { + createKernel(); + + neoDevice->getExecutionEnvironment()->rootDeviceEnvironments[neoDevice->getRootDeviceIndex()]->createBindlessHeapsHelper(neoDevice->getMemoryManager(), + neoDevice->getNumGenericSubDevices() > 1, + neoDevice->getRootDeviceIndex(), + neoDevice->getDeviceBitfield()); + auto &imageArg = const_cast(kernel->kernelImmData->getDescriptor().payloadMappings.explicitArgs[3].template as()); + auto &addressingMode = kernel->kernelImmData->getDescriptor().kernelAttributes.imageAddressingMode; + const_cast(addressingMode) = NEO::KernelDescriptor::Bindless; + imageArg.bindless = 0x0; + imageArg.bindful = undefined; + ze_image_desc_t desc = {}; + desc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + + auto imageHW = std::make_unique>(); + auto ret = imageHW->initialize(device, &desc); + ASSERT_EQ(ZE_RESULT_SUCCESS, ret); + EXPECT_EQ(ZE_RESULT_SUCCESS, imageHW->allocateBindlessSlot()); + + ze_image_handle_t imageViewHandle; + desc.stype = ZE_STRUCTURE_TYPE_IMAGE_VIEW_PLANAR_EXT_DESC; + ret = imageHW->createView(device, &desc, &imageViewHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, ret); + + auto imageView = Image::fromHandle(imageViewHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, imageView->allocateBindlessSlot()); + auto ssInHeap = imageHW->getBindlessSlot(); + auto ssInHeapView = imageView->getBindlessSlot(); + + ASSERT_NE(nullptr, ssInHeap); + ASSERT_NE(nullptr, ssInHeapView); + + EXPECT_NE(ssInHeap->surfaceStateOffset, ssInHeapView->surfaceStateOffset); + + // calling allocateBindlessSlot again should not change slot + EXPECT_EQ(ZE_RESULT_SUCCESS, imageView->allocateBindlessSlot()); + auto ssInHeapView2 = imageView->getBindlessSlot(); + EXPECT_EQ(ssInHeapView->surfaceStateOffset, ssInHeapView2->surfaceStateOffset); + + imageView->destroy(); +} + +HWTEST2_F(SetKernelArg, givenGlobalBindlessHelperImageViewAndNoAvailableSpaceOnSshWhenAllocatingBindlessSlotThenOutOfMemoryErrorReturned, ImageSupport) { + createKernel(); + auto mockMemManager = static_cast(neoDevice->getMemoryManager()); + auto bindlessHelper = new MockBindlesHeapsHelper(mockMemManager, + neoDevice->getNumGenericSubDevices() > 1, + neoDevice->getRootDeviceIndex(), + neoDevice->getDeviceBitfield()); + neoDevice->getExecutionEnvironment()->rootDeviceEnvironments[neoDevice->getRootDeviceIndex()]->bindlessHeapsHelper.reset(bindlessHelper); + + auto &imageArg = const_cast(kernel->kernelImmData->getDescriptor().payloadMappings.explicitArgs[3].template as()); + auto &addressingMode = kernel->kernelImmData->getDescriptor().kernelAttributes.imageAddressingMode; + const_cast(addressingMode) = NEO::KernelDescriptor::Bindless; + imageArg.bindless = 0x0; + imageArg.bindful = undefined; + ze_image_desc_t desc = {}; + desc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + + auto imageHW = std::make_unique>(); + auto ret = imageHW->initialize(device, &desc); + ASSERT_EQ(ZE_RESULT_SUCCESS, ret); + + mockMemManager->failInDevicePool = true; + mockMemManager->failAllocateSystemMemory = true; + bindlessHelper->globalSsh->getSpace(bindlessHelper->globalSsh->getAvailableSpace()); + + ze_image_handle_t imageViewHandle; + desc.stype = ZE_STRUCTURE_TYPE_IMAGE_VIEW_PLANAR_EXT_DESC; + ret = imageHW->createView(device, &desc, &imageViewHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, ret); + + auto imageView = Image::fromHandle(imageViewHandle); + EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY, imageView->allocateBindlessSlot()); + auto ssInHeap = imageHW->getBindlessSlot(); + auto ssInHeapView = imageView->getBindlessSlot(); + + EXPECT_EQ(nullptr, ssInHeap); + EXPECT_EQ(nullptr, ssInHeapView); + imageView->destroy(); +} + +HWTEST2_F(SetKernelArg, givenNoGlobalBindlessHelperAndImageViewWhenAllocatingBindlessSlotThenSlotIsNotAllocated, ImageSupport) { + createKernel(); + + auto &imageArg = const_cast(kernel->kernelImmData->getDescriptor().payloadMappings.explicitArgs[3].template as()); + auto &addressingMode = kernel->kernelImmData->getDescriptor().kernelAttributes.imageAddressingMode; + const_cast(addressingMode) = NEO::KernelDescriptor::Bindless; + imageArg.bindless = 0x0; + imageArg.bindful = undefined; + ze_image_desc_t desc = {}; + desc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + + auto imageHW = std::make_unique>(); + auto ret = imageHW->initialize(device, &desc); + ASSERT_EQ(ZE_RESULT_SUCCESS, ret); + EXPECT_EQ(ZE_RESULT_SUCCESS, imageHW->allocateBindlessSlot()); + + ze_image_handle_t imageViewHandle; + desc.stype = ZE_STRUCTURE_TYPE_IMAGE_VIEW_PLANAR_EXT_DESC; + ret = imageHW->createView(device, &desc, &imageViewHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, ret); + + auto imageView = Image::fromHandle(imageViewHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, imageView->allocateBindlessSlot()); + auto ssInHeap = imageHW->getBindlessSlot(); + auto ssInHeapView = imageView->getBindlessSlot(); + + ASSERT_EQ(nullptr, ssInHeap); + ASSERT_EQ(nullptr, ssInHeapView); + + imageView->destroy(); +} + HWTEST2_F(SetKernelArg, givenImageAndBindlessKernelWhenSetArgRedescribedImageCalledThenCopySurfaceStateToSSHCalledWithCorrectArgs, ImageSupport) { Mock mockModule(this->device, nullptr); Mock mockKernel;