Refactor graphics memory allocation scheme

- replace createGraphicsAllocationWithRequiredBitness with more general
methodallocateGraphicsMemoryInPreferredPool based on passed
 AllocationData
- proper flags for allocation selected based on AllocationType

- remove allocateGraphicsMemory(size_t size, size_t alignment)
and use allocateGraphicsMemory(size_t size) instead where default
alignment is sufficient, otherwise use full options version:
allocateGraphicsMemory(size_t size, size_t alignment,
 bool forcePin, bool uncacheable)

Change-Id: I2da891f372ee181253cb840568a61b33c0d71fc9

runtime/memory_manager/graphics_allocation.h

@@ -61,6 +61,20 @@ class GraphicsAllocation : public IDNode<GraphicsAllocation> {
         TAG_BUFFER,
         LINEAR_STREAM,
         FILL_PATTERN,
+        PIPE,
+        EVENT_TAG_BUFFER,
+        COMMAND_BUFFER,
+        PRINTF_SURFACE,
+        GLOBAL_SURFACE,
+        PRIVATE_SURFACE,
+        CONSTANT_SURFACE,
+        SCRATCH_SURFACE,
+        CSR_SURFACE,
+        INSTRUCTION_HEAP,
+        INDIRECT_OBJECT_HEAP,
+        SURFACE_STATE_HEAP,
+        DYNAMIC_STATE_HEAP,
+        SHARED_RESOURCE,
         NON_AUB_WRITABLE = 0x40000000,
         WRITABLE = 0x80000000
     };

runtime/memory_manager/memory_manager.cpp

@@ -116,7 +116,7 @@ GraphicsAllocation *MemoryManager::allocateGraphicsMemoryForSVM(size_t size, boo
     if (enable64kbpages) {
         graphicsAllocation = allocateGraphicsMemory64kb(size, MemoryConstants::pageSize64k, false);
     } else {
-        graphicsAllocation = allocateGraphicsMemory(size, MemoryConstants::pageSize);
+        graphicsAllocation = allocateGraphicsMemory(size);
     }
     if (graphicsAllocation) {
         graphicsAllocation->setCoherent(coherent);
@@ -165,7 +165,7 @@ void MemoryManager::cleanGraphicsMemoryCreatedFromHostPtr(GraphicsAllocation *gr
 
 GraphicsAllocation *MemoryManager::createGraphicsAllocationWithPadding(GraphicsAllocation *inputGraphicsAllocation, size_t sizeWithPadding) {
     if (!paddingAllocation) {
-        paddingAllocation = allocateGraphicsMemory(paddingBufferSize, MemoryConstants::pageSize);
+        paddingAllocation = allocateGraphicsMemory(paddingBufferSize);
     }
     return createPaddedAllocation(inputGraphicsAllocation, sizeWithPadding);
 }
@@ -367,4 +367,67 @@ RequirementsStatus MemoryManager::checkAllocationsForOverlapping(AllocationRequi
     return status;
 }
 
+bool MemoryManager::getAllocationData(AllocationData &allocationData, bool mustBeZeroCopy, bool allocateMemory, bool forcePin, bool uncacheable, const void *hostPtr, size_t size, GraphicsAllocation::AllocationType type) {
+    UNRECOVERABLE_IF(hostPtr == nullptr && !allocateMemory);
+
+    bool allow64KbPages = false;
+    bool allow32Bit = false;
+
+    switch (type) {
+    case GraphicsAllocation::AllocationType::BUFFER:
+    case GraphicsAllocation::AllocationType::PIPE:
+    case GraphicsAllocation::AllocationType::SCRATCH_SURFACE:
+    case GraphicsAllocation::AllocationType::PRIVATE_SURFACE:
+    case GraphicsAllocation::AllocationType::PRINTF_SURFACE:
+    case GraphicsAllocation::AllocationType::CONSTANT_SURFACE:
+    case GraphicsAllocation::AllocationType::GLOBAL_SURFACE:
+        allow64KbPages = true;
+        allow32Bit = true;
+        break;
+    default:
+        break;
+    }
+
+    allocationData.flags.mustBeZeroCopy = mustBeZeroCopy;
+    allocationData.flags.allocateMemory = allocateMemory;
+    allocationData.flags.allow32Bit = allow32Bit;
+    allocationData.flags.allow64kbPages = allow64KbPages;
+    allocationData.flags.forcePin = forcePin;
+    allocationData.flags.uncacheable = uncacheable;
+
+    if (allocationData.flags.mustBeZeroCopy) {
+        allocationData.flags.useSystemMemory = true;
+    }
+
+    allocationData.hostPtr = hostPtr;
+    allocationData.size = size;
+    allocationData.type = type;
+
+    if (allocationData.flags.allocateMemory) {
+        allocationData.hostPtr = nullptr;
+    }
+    return true;
+}
+
+GraphicsAllocation *MemoryManager::allocateGraphicsMemoryInPreferredPool(bool mustBeZeroCopy, bool allocateMemory, bool forcePin, bool uncacheable, const void *hostPtr, size_t size, GraphicsAllocation::AllocationType type) {
+    AllocationData allocationData;
+    getAllocationData(allocationData, mustBeZeroCopy, allocateMemory, forcePin, uncacheable, hostPtr, size, type);
+    UNRECOVERABLE_IF(allocationData.type == GraphicsAllocation::AllocationType::IMAGE || allocationData.type == GraphicsAllocation::AllocationType::SHARED_RESOURCE);
+
+    return allocateGraphicsMemory(allocationData);
+}
+
+GraphicsAllocation *MemoryManager::allocateGraphicsMemory(const AllocationData &allocationData) {
+    if (force32bitAllocations && allocationData.flags.allow32Bit && is64bit) {
+        return allocate32BitGraphicsMemory(allocationData.size, allocationData.hostPtr, AllocationOrigin::EXTERNAL_ALLOCATION);
+    }
+    if (allocationData.hostPtr) {
+        return allocateGraphicsMemory(allocationData.size, allocationData.hostPtr, allocationData.flags.forcePin);
+    }
+    if (enable64kbpages && allocationData.flags.allow64kbPages) {
+        return allocateGraphicsMemory64kb(allocationData.size, MemoryConstants::pageSize64k, allocationData.flags.forcePin);
+    }
+    return allocateGraphicsMemory(allocationData.size, MemoryConstants::pageSize, allocationData.flags.forcePin, allocationData.flags.uncacheable);
+}
+
 } // namespace OCLRT

runtime/memory_manager/memory_manager.h

@@ -60,6 +60,27 @@ enum AllocationOrigin {
     INTERNAL_ALLOCATION
 };
 
+struct AllocationData {
+    union {
+        struct {
+            uint32_t mustBeZeroCopy : 1;
+            uint32_t allocateMemory : 1;
+            uint32_t allow64kbPages : 1;
+            uint32_t allow32Bit : 1;
+            uint32_t useSystemMemory : 1;
+            uint32_t forcePin : 1;
+            uint32_t uncacheable : 1;
+            uint32_t reserved : 25;
+        } flags;
+        uint32_t allFlags = 0;
+    };
+    static_assert(sizeof(AllocationData::flags) == sizeof(AllocationData::allFlags), "");
+
+    GraphicsAllocation::AllocationType type = GraphicsAllocation::AllocationType::UNKNOWN;
+    const void *hostPtr = nullptr;
+    size_t size = 0;
+};
+
 struct AlignedMallocRestrictions {
     uintptr_t minAddress;
 };
@@ -93,12 +114,8 @@ class MemoryManager {
     virtual void addAllocationToHostPtrManager(GraphicsAllocation *memory) = 0;
     virtual void removeAllocationFromHostPtrManager(GraphicsAllocation *memory) = 0;
 
-    virtual GraphicsAllocation *allocateGraphicsMemory(size_t size) {
-        return allocateGraphicsMemory(size, static_cast<size_t>(0u));
-    }
-
-    virtual GraphicsAllocation *allocateGraphicsMemory(size_t size, size_t alignment) {
-        return allocateGraphicsMemory(size, alignment, false, false);
+    GraphicsAllocation *allocateGraphicsMemory(size_t size) {
+        return allocateGraphicsMemory(size, MemoryConstants::preferredAlignment, false, false);
     }
 
     virtual GraphicsAllocation *allocateGraphicsMemory(size_t size, size_t alignment, bool forcePin, bool uncacheable) = 0;
@@ -110,10 +127,12 @@ class MemoryManager {
     }
     virtual GraphicsAllocation *allocateGraphicsMemory(size_t size, const void *ptr, bool forcePin);
 
-    virtual GraphicsAllocation *allocate32BitGraphicsMemory(size_t size, void *ptr, AllocationOrigin allocationOrigin) = 0;
+    virtual GraphicsAllocation *allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) = 0;
 
     virtual GraphicsAllocation *allocateGraphicsMemoryForImage(ImageInfo &imgInfo, Gmm *gmm) = 0;
 
+    MOCKABLE_VIRTUAL GraphicsAllocation *allocateGraphicsMemoryInPreferredPool(bool mustBeZeroCopy, bool allocateMemory, bool forcePin, bool uncacheable, const void *hostPtr, size_t size, GraphicsAllocation::AllocationType type);
+
     GraphicsAllocation *allocateGraphicsMemoryForSVM(size_t size, bool coherent);
 
     virtual GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, bool requireSpecificBitness, bool reuseBO) = 0;
@@ -177,25 +196,6 @@ class MemoryManager {
     bool peekForce32BitAllocations() { return force32bitAllocations; }
     void setForce32BitAllocations(bool newValue);
 
-    GraphicsAllocation *createGraphicsAllocationWithRequiredBitness(size_t size, void *ptr) {
-        return createGraphicsAllocationWithRequiredBitness(size, ptr, false);
-    }
-
-    MOCKABLE_VIRTUAL GraphicsAllocation *createGraphicsAllocationWithRequiredBitness(size_t size, void *ptr, bool forcePin) {
-        if (force32bitAllocations && is64bit) {
-            return allocate32BitGraphicsMemory(size, ptr, AllocationOrigin::EXTERNAL_ALLOCATION);
-        } else {
-            if (ptr) {
-                return allocateGraphicsMemory(size, ptr, forcePin);
-            }
-            if (enable64kbpages) {
-                return allocateGraphicsMemory64kb(size, MemoryConstants::pageSize64k, forcePin);
-            } else {
-                return allocateGraphicsMemory(size, MemoryConstants::pageSize, forcePin, false);
-            }
-        }
-    }
-
     std::unique_ptr<Allocator32bit> allocator32Bit;
 
     bool peekVirtualPaddingSupport() { return virtualPaddingAvailable; }
@@ -235,6 +235,9 @@ class MemoryManager {
     }
 
   protected:
+    static bool getAllocationData(AllocationData &allocationData, bool mustBeZeroCopy, bool allocateMemory, bool forcePin, bool uncacheable, const void *hostPtr, size_t size, GraphicsAllocation::AllocationType type);
+
+    GraphicsAllocation *allocateGraphicsMemory(const AllocationData &allocationData);
     std::recursive_mutex mtx;
     std::unique_ptr<TagAllocator<HwTimeStamps>> profilingTimeStampAllocator;
     std::unique_ptr<TagAllocator<HwPerfCounter>> perfCounterAllocator;

runtime/memory_manager/os_agnostic_memory_manager.cpp

@@ -67,15 +67,15 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemory64kb(size_t s
     return allocateGraphicsMemory(alignUp(size, MemoryConstants::pageSize64k), MemoryConstants::pageSize64k, forcePin, false);
 }
 
-GraphicsAllocation *OsAgnosticMemoryManager::allocate32BitGraphicsMemory(size_t size, void *ptr, AllocationOrigin allocationOrigin) {
+GraphicsAllocation *OsAgnosticMemoryManager::allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) {
     if (ptr) {
-        auto allocationSize = alignSizeWholePage(reinterpret_cast<void *>(ptr), size);
+        auto allocationSize = alignSizeWholePage(ptr, size);
         auto gpuVirtualAddress = allocator32Bit->allocate(allocationSize);
         if (!gpuVirtualAddress) {
             return nullptr;
         }
         uint64_t offset = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(ptr) & MemoryConstants::pageMask);
-        MemoryAllocation *memAlloc = new MemoryAllocation(false, reinterpret_cast<void *>(ptr), GmmHelper::canonize(gpuVirtualAddress + offset), size, counter);
+        MemoryAllocation *memAlloc = new MemoryAllocation(false, const_cast<void *>(ptr), GmmHelper::canonize(gpuVirtualAddress + offset), size, counter);
         memAlloc->is32BitAllocation = true;
         memAlloc->gpuBaseAddress = GmmHelper::canonize(allocator32Bit->getBase());
         memAlloc->sizeToFree = allocationSize;
@@ -205,7 +205,7 @@ void OsAgnosticMemoryManager::cleanOsHandles(OsHandleStorage &handleStorage) {
     }
 }
 GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryForImage(ImageInfo &imgInfo, Gmm *gmm) {
-    auto alloc = allocateGraphicsMemory(imgInfo.size, MemoryConstants::preferredAlignment);
+    auto alloc = allocateGraphicsMemory(imgInfo.size);
     if (alloc) {
         alloc->gmm = gmm;
     }

runtime/memory_manager/os_agnostic_memory_manager.h

@@ -60,7 +60,7 @@ class OsAgnosticMemoryManager : public MemoryManager {
     ~OsAgnosticMemoryManager() override;
     GraphicsAllocation *allocateGraphicsMemory(size_t size, size_t alignment, bool forcePin, bool uncacheable) override;
     GraphicsAllocation *allocateGraphicsMemory64kb(size_t size, size_t alignment, bool forcePin) override;
-    GraphicsAllocation *allocate32BitGraphicsMemory(size_t size, void *ptr, AllocationOrigin allocationOrigin) override;
+    GraphicsAllocation *allocate32BitGraphicsMemory(size_t size, const void *ptr, AllocationOrigin allocationOrigin) override;
     GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, bool requireSpecificBitness, bool reuseBO) override;
     GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle) override { return nullptr; }
     GraphicsAllocation *allocateGraphicsMemoryForImage(ImageInfo &imgInfo, Gmm *gmm) override;