Map/unmap enqueue fixes [2/n]: CPU operations on limited range

- Curently each non-zerocopy CPU operation on map/unmap make a full copy
  using hostPtr
- This commit adds functionality to select specific range of copy
- Multiple mapping with different size is not supported yet,
  so copy will be made on full range for now. This is for future usage.

Change-Id: I7652e85482ba6fffb2474169447baf9b080dcd1e

runtime/mem_obj/buffer.cpp

@@ -291,10 +291,19 @@ bool Buffer::bufferRectPitchSet(const size_t *bufferOrigin,
     return true;
 }
 
-void *Buffer::transferDataToHostPtr() {
-    DBG_LOG(LogMemoryObject, __FUNCTION__, "hostPtr:", hostPtr, "size:", size, "memoryStorage:", memoryStorage);
-    memcpy_s(hostPtr, size, memoryStorage, size);
-    return hostPtr;
+void Buffer::transferData(void *dst, void *src, size_t copySize, size_t copyOffset) {
+    DBG_LOG(LogMemoryObject, __FUNCTION__, " hostPtr: ", hostPtr, ", size: ", copySize, ", offset: ", copyOffset, ", memoryStorage: ", memoryStorage);
+    auto dstPtr = ptrOffset(dst, copyOffset);
+    auto srcPtr = ptrOffset(src, copyOffset);
+    memcpy_s(dstPtr, copySize, srcPtr, copySize);
+}
+
+void Buffer::transferDataToHostPtr(std::array<size_t, 3> copySize, std::array<size_t, 3> copyOffset) {
+    transferData(hostPtr, memoryStorage, copySize[0], copyOffset[0]);
+}
+
+void Buffer::transferDataFromHostPtr(std::array<size_t, 3> copySize, std::array<size_t, 3> copyOffset) {
+    transferData(memoryStorage, hostPtr, copySize[0], copyOffset[0]);
 }
 
 size_t Buffer::calculateHostPtrSize(const size_t *origin, const size_t *region, size_t rowPitch, size_t slicePitch) {

runtime/mem_obj/buffer.h

@@ -100,7 +100,8 @@ class Buffer : public MemObj {
 
     static size_t calculateHostPtrSize(const size_t *origin, const size_t *region, size_t rowPitch, size_t slicePitch);
 
-    void *transferDataToHostPtr() override;
+    void transferDataToHostPtr(std::array<size_t, 3> copySize, std::array<size_t, 3> copyOffset) override;
+    void transferDataFromHostPtr(std::array<size_t, 3> copySize, std::array<size_t, 3> copyOffset) override;
 
     bool isReadWriteOnCpuAllowed(cl_bool blocking, cl_uint numEventsInWaitList, void *ptr, size_t size);
 
@@ -125,6 +126,8 @@ class Buffer : public MemObj {
                             bool &allocateMemory,
                             bool &copyMemoryFromHostPtr,
                             MemoryManager *memMngr);
+
+    void transferData(void *dst, void *src, size_t copySize, size_t copyOffset);
 };
 
 template <typename GfxFamily>

runtime/mem_obj/image.cpp

@@ -81,23 +81,25 @@ Image::Image(Context *context,
         setSurfaceOffsets(0, 0, 0, 0);
 }
 
-void Image::transferData(void *src, size_t srcRowPitch, size_t srcSlicePitch, void *dest, size_t destRowPitch, size_t destSlicePitch, cl_image_desc *imageDesc, size_t pixelSize, size_t imageCount) {
-    size_t imageHeight = getValidParam(imageDesc->image_height);
-    size_t imageDepth = getValidParam(imageDesc->image_depth);
-    size_t lineWidth = getValidParam(imageDesc->image_width) * pixelSize;
+void Image::transferData(void *dest, size_t destRowPitch, size_t destSlicePitch,
+                         void *src, size_t srcRowPitch, size_t srcSlicePitch,
+                         std::array<size_t, 3> &copyRegion, std::array<size_t, 3> &copyOrigin) {
+
+    size_t pixelSize = surfaceFormatInfo.ImageElementSizeInBytes;
+    size_t lineWidth = copyRegion[0] * pixelSize;
 
     DBG_LOG(LogMemoryObject, __FUNCTION__, "memcpy dest:", dest, "sizeRowToCopy:", lineWidth, "src:", src);
-    for (size_t count = 0; count < imageCount; count++) {
-        for (size_t depth = 0; depth < imageDepth; ++depth) {
-            auto currentImage = std::max(depth, count);
-            auto srcPtr = ptrOffset(src, srcSlicePitch * currentImage);
-            auto destPtr = ptrOffset(dest, destSlicePitch * currentImage);
 
-            for (size_t height = 0; height < imageHeight; ++height) {
-                memcpy_s(destPtr, lineWidth, srcPtr, lineWidth);
-                srcPtr = ptrOffset(srcPtr, srcRowPitch);
-                destPtr = ptrOffset(destPtr, destRowPitch);
-            }
+    for (size_t slice = copyOrigin[2]; slice < (copyOrigin[2] + copyRegion[2]); slice++) {
+        auto srcSliceOffset = ptrOffset(src, srcSlicePitch * slice);
+        auto dstSliceOffset = ptrOffset(dest, destSlicePitch * slice);
+
+        for (size_t height = copyOrigin[1]; height < (copyOrigin[1] + copyRegion[1]); height++) {
+            auto srcRowOffset = ptrOffset(srcSliceOffset, srcRowPitch * height);
+            auto dstRowOffset = ptrOffset(dstSliceOffset, destRowPitch * height);
+
+            memcpy_s(ptrOffset(dstRowOffset, copyOrigin[0] * pixelSize), lineWidth,
+                     ptrOffset(srcRowOffset, copyOrigin[0] * pixelSize), lineWidth);
         }
     }
 }
@@ -212,39 +214,35 @@ Image *Image::create(Context *context,
         } else {
             gmm = new Gmm();
             gmm->queryImageParams(imgInfo, hwInfo);
+
+            errcodeRet = CL_OUT_OF_HOST_MEMORY;
             if (flags & CL_MEM_USE_HOST_PTR) {
-                errcodeRet = CL_INVALID_HOST_PTR;
-                if (hostPtr) {
-                    size_t pointerPassedSize = hostPtrRowPitch * imageHeight * imageDepth * imageCount;
-                    auto alignedSizePassedPointer = alignSizeWholePage(const_cast<void *>(hostPtr), pointerPassedSize);
-                    auto alignedSizeRequiredForAllocation = alignSizeWholePage(const_cast<void *>(hostPtr), imgInfo.size);
+                size_t pointerPassedSize = hostPtrRowPitch * imageHeight * imageDepth * imageCount;
+                auto alignedSizePassedPointer = alignSizeWholePage(const_cast<void *>(hostPtr), pointerPassedSize);
+                auto alignedSizeRequiredForAllocation = alignSizeWholePage(const_cast<void *>(hostPtr), imgInfo.size);
 
-                    // Passed pointer doesn't have enough memory, copy is needed
-                    copyRequired = (alignedSizeRequiredForAllocation > alignedSizePassedPointer) |
-                                   (imgInfo.rowPitch != hostPtrRowPitch) |
-                                   (imgInfo.slicePitch != hostPtrSlicePitch) |
-                                   ((reinterpret_cast<uintptr_t>(hostPtr) & (MemoryConstants::cacheLineSize - 1)) != 0) |
-                                   isTilingAllowed;
+                // Passed pointer doesn't have enough memory, copy is needed
+                copyRequired = (alignedSizeRequiredForAllocation > alignedSizePassedPointer) |
+                               (imgInfo.rowPitch != hostPtrRowPitch) |
+                               (imgInfo.slicePitch != hostPtrSlicePitch) |
+                               ((reinterpret_cast<uintptr_t>(hostPtr) & (MemoryConstants::cacheLineSize - 1)) != 0) |
+                               isTilingAllowed;
 
-                    if (copyRequired && !context->isSharedContext) {
-                        errcodeRet = CL_OUT_OF_HOST_MEMORY;
-                        memory = memoryManager->allocateGraphicsMemoryForImage(imgInfo, gmm);
-                        zeroCopy = false;
-                        transferNeeded = true;
-                    } else {
-                        // To avoid having two pointers in a MemObj we cast off the const here
-                        // However, in USE_HOST_PTR cases we shouldn't be modifying the memory
-                        memory = memoryManager->allocateGraphicsMemory(imgInfo.size, hostPtr);
-                        memory->gmm = gmm;
-                        zeroCopy = true;
-                    }
+                if (copyRequired && !context->isSharedContext) {
+                    memory = memoryManager->allocateGraphicsMemoryForImage(imgInfo, gmm);
+                    zeroCopy = false;
+                    transferNeeded = true;
+                } else {
+                    memory = memoryManager->allocateGraphicsMemory(imgInfo.size, hostPtr);
+                    memory->gmm = gmm;
+                    zeroCopy = true;
                 }
             } else {
-                errcodeRet = CL_OUT_OF_HOST_MEMORY;
                 memory = memoryManager->allocateGraphicsMemoryForImage(imgInfo, gmm);
                 zeroCopy = true;
             }
         }
+        transferNeeded |= !!(flags & CL_MEM_COPY_HOST_PTR);
 
         switch (imageDesc->image_type) {
         case CL_MEM_OBJECT_IMAGE3D:
@@ -286,19 +284,6 @@ Image *Image::create(Context *context,
 
         DBG_LOG(LogMemoryObject, __FUNCTION__, "hostPtr:", hostPtr, "size:", memory->getUnderlyingBufferSize(), "memoryStorage:", memory->getUnderlyingBuffer(), "GPU address:", std::hex, memory->getGpuAddress());
 
-        if (!isTilingAllowed) {
-            errcodeRet = CL_INVALID_VALUE;
-            if (flags & CL_MEM_COPY_HOST_PTR || transferNeeded) {
-                if (hostPtr) {
-                    Image::transferData((void *)hostPtr, hostPtrRowPitch, hostPtrSlicePitch,
-                                        memory->getUnderlyingBuffer(), imgInfo.rowPitch, imgInfo.slicePitch,
-                                        (cl_image_desc *)imageDesc, surfaceFormat->ImageElementSizeInBytes, imageCount);
-                } else {
-                    memoryManager->freeGraphicsMemory(memory);
-                    break;
-                }
-            }
-        }
         if (parentImage) {
             imageDescriptor.image_height = imageHeight;
             imageDescriptor.image_width = imageWidth;
@@ -341,39 +326,33 @@ Image *Image::create(Context *context,
         }
         errcodeRet = CL_SUCCESS;
 
-        if (isTilingAllowed) {
-            if (flags & CL_MEM_COPY_HOST_PTR || transferNeeded) {
-                if (!hostPtr) {
-                    errcodeRet = CL_INVALID_VALUE;
-                    image->release();
-                    image = nullptr;
-                    memory = nullptr;
-                    break;
-                }
-                auto cmdQ = context->getSpecialQueue();
+        if (transferNeeded) {
+            std::array<size_t, 3> copyOrigin = {{0, 0, 0}};
+            std::array<size_t, 3> copyRegion = {{imageWidth, imageHeight, std::max(imageDepth, imageCount)}};
 
-                size_t Origin[] = {0, 0, 0};
-                size_t Region[] = {imageWidth, imageHeight, imageDepth};
-                if (imageDesc->image_type == CL_MEM_OBJECT_IMAGE2D_ARRAY) {
-                    Region[2] = imageDesc->image_array_size;
-                }
+            if (isTilingAllowed) {
+                auto cmdQ = context->getSpecialQueue();
 
                 if (IsNV12Image(&image->getImageFormat())) {
                     errcodeRet = image->writeNV12Planes(hostPtr, hostPtrRowPitch);
                 } else {
-                    errcodeRet = cmdQ->enqueueWriteImage(image, CL_TRUE, Origin, Region,
+                    errcodeRet = cmdQ->enqueueWriteImage(image, CL_TRUE, &copyOrigin[0], &copyRegion[0],
                                                          hostPtrRowPitch, hostPtrSlicePitch,
                                                          hostPtr, 0, nullptr, nullptr);
                 }
-                if (errcodeRet != CL_SUCCESS) {
-                    image->release();
-                    image = nullptr;
-                    memory = nullptr;
-                    break;
-                }
+            } else {
+                image->transferData(memory->getUnderlyingBuffer(), imgInfo.rowPitch, imgInfo.slicePitch,
+                                    const_cast<void *>(hostPtr), hostPtrRowPitch, hostPtrSlicePitch,
+                                    copyRegion, copyOrigin);
             }
         }
 
+        if (errcodeRet != CL_SUCCESS) {
+            image->release();
+            image = nullptr;
+            memory = nullptr;
+            break;
+        }
     } while (false);
 
     return image;
@@ -860,16 +839,16 @@ Image *Image::redescribe() {
     return image;
 }
 
-void *Image::transferDataToHostPtr() {
-    Image::transferData(graphicsAllocation->getUnderlyingBuffer(), imageDesc.image_row_pitch, imageDesc.image_slice_pitch,
-                        hostPtr, hostPtrRowPitch, hostPtrSlicePitch, &imageDesc, surfaceFormatInfo.ImageElementSizeInBytes, imageCount);
-    return hostPtr;
+void Image::transferDataToHostPtr(std::array<size_t, 3> copySize, std::array<size_t, 3> copyOffset) {
+    transferData(hostPtr, hostPtrRowPitch, hostPtrSlicePitch,
+                 graphicsAllocation->getUnderlyingBuffer(), imageDesc.image_row_pitch, imageDesc.image_slice_pitch,
+                 copySize, copyOffset);
 }
 
-void Image::transferDataFromHostPtrToMemoryStorage() {
-    Image::transferData(hostPtr, hostPtrRowPitch, hostPtrSlicePitch,
-                        memoryStorage, imageDesc.image_row_pitch, imageDesc.image_slice_pitch,
-                        &imageDesc, surfaceFormatInfo.ImageElementSizeInBytes, imageCount);
+void Image::transferDataFromHostPtr(std::array<size_t, 3> copySize, std::array<size_t, 3> copyOffset) {
+    transferData(memoryStorage, imageDesc.image_row_pitch, imageDesc.image_slice_pitch,
+                 hostPtr, hostPtrRowPitch, hostPtrSlicePitch,
+                 copySize, copyOffset);
 }
 
 cl_int Image::writeNV12Planes(const void *hostPtr, size_t hostPtrRowPitch) {

runtime/mem_obj/image.h

@@ -138,12 +138,13 @@ class Image : public MemObj {
     const cl_image_desc &getImageDesc() const;
     const cl_image_format &getImageFormat() const;
     const SurfaceFormatInfo &getSurfaceFormatInfo() const;
-    void *transferDataToHostPtr() override;
+
+    void transferDataToHostPtr(std::array<size_t, 3> copySize, std::array<size_t, 3> copyOffset) override;
+    void transferDataFromHostPtr(std::array<size_t, 3> copySize, std::array<size_t, 3> copyOffset) override;
 
     Image *redescribe();
     Image *redescribeFillImage();
     ImageCreatFunc createFunction;
-    void transferDataFromHostPtrToMemoryStorage() override;
 
     uint32_t getQPitch() { return qPitch; }
     void setQPitch(uint32_t qPitch) { this->qPitch = qPitch; }
@@ -151,7 +152,7 @@ class Image : public MemObj {
     void setHostPtrRowPitch(size_t pitch) { this->hostPtrRowPitch = pitch; }
     size_t getHostPtrSlicePitch() { return hostPtrSlicePitch; }
     void setHostPtrSlicePitch(size_t pitch) { this->hostPtrSlicePitch = pitch; }
-    size_t getImageCount() { return imageCount; }
+    size_t getImageCount() const { return imageCount; }
     void setImageCount(size_t imageCount) { this->imageCount = imageCount; }
     bool allowTiling() const override { return this->isTiledImage; }
     void setImageRowPitch(size_t rowPitch) { imageDesc.image_row_pitch = rowPitch; }
@@ -176,7 +177,6 @@ class Image : public MemObj {
     cl_int writeNV12Planes(const void *hostPtr, size_t hostPtrRowPitch);
     void setMcsSurfaceInfo(McsSurfaceInfo &info) { mcsSurfaceInfo = info; }
     const McsSurfaceInfo &getMcsSurfaceInfo() { return mcsSurfaceInfo; }
-    static void transferData(void *src, size_t srcRowPitch, size_t srcSlicePitch, void *dest, size_t destRowPitch, size_t destSlicePitch, cl_image_desc *imageDesc, size_t pixelSize, size_t imageCount);
 
     const bool isTiledImage;
 
@@ -197,6 +197,10 @@ class Image : public MemObj {
 
     void getOsSpecificImageInfo(const cl_mem_info &paramName, size_t *srcParamSize, void **srcParam);
 
+    void transferData(void *dst, size_t dstRowPitch, size_t dstSlicePitch,
+                      void *src, size_t srcRowPitch, size_t srcSlicePitch,
+                      std::array<size_t, 3> &copyRegion, std::array<size_t, 3> &copyOrigin);
+
     cl_image_format imageFormat;
     cl_image_desc imageDesc;
     SurfaceFormatInfo surfaceFormatInfo;

runtime/mem_obj/mem_obj.cpp

@@ -247,11 +247,6 @@ bool MemObj::isMemObjWithHostPtrSVM() const {
     return isHostPtrSVM;
 }
 
-void MemObj::transferDataFromHostPtrToMemoryStorage() {
-    size_t dataBytesToTransfer = std::min(size, hostPtrMinSize);
-    memcpy_s(memoryStorage, size, hostPtr, dataBytesToTransfer);
-}
-
 GraphicsAllocation *MemObj::getGraphicsAllocation() {
     return graphicsAllocation;
 }

runtime/mem_obj/mem_obj.h

@@ -28,6 +28,7 @@
 #include <atomic>
 #include <cstdint>
 #include <vector>
+#include <array>
 
 namespace OCLRT {
 class GraphicsAllocation;
@@ -88,8 +89,9 @@ class MemObj : public BaseObject<_cl_mem> {
     void decMapCount();
     bool isMemObjZeroCopy() const;
     bool isMemObjWithHostPtrSVM() const;
-    virtual void *transferDataToHostPtr() { return nullptr; };
-    virtual void transferDataFromHostPtrToMemoryStorage();
+    virtual void transferDataToHostPtr(std::array<size_t, 3> copySize, std::array<size_t, 3> copyOffset) { UNRECOVERABLE_IF(true); };
+    virtual void transferDataFromHostPtr(std::array<size_t, 3> copySize, std::array<size_t, 3> copyOffset) { UNRECOVERABLE_IF(true); };
+
     GraphicsAllocation *getGraphicsAllocation();
     GraphicsAllocation *getMcsAllocation() { return mcsAllocation; }
     void setMcsAllocation(GraphicsAllocation *alloc) { mcsAllocation = alloc; }